From 17d7145e3c01bddf7288f90e7fe0a7c577d0af03 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 23 Aug 2026 13:24:44 +0800 Subject: [PATCH 01/41] test(scripts): add reset-safe internode metric sampling (#6437) Co-authored-by: heihutu --- scripts/run_internode_transport_baseline.sh | 127 +++++++++++++++++++- 1 file changed, 121 insertions(+), 6 deletions(-) diff --git a/scripts/run_internode_transport_baseline.sh b/scripts/run_internode_transport_baseline.sh index 93166268c..87bfe1aaf 100755 --- a/scripts/run_internode_transport_baseline.sh +++ b/scripts/run_internode_transport_baseline.sh @@ -5,7 +5,8 @@ set -euo pipefail # Reuses scripts/run_object_batch_bench.sh and exports reproducible artifacts: # - run manifest with scenario/tool metadata and git revision # - object benchmark summaries per scenario/workload/concurrency -# - optional internode operation metric deltas from a Prometheus text endpoint +# - optional internode operation metric deltas from Prometheus text snapshots or +# PromQL increase() queries SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" @@ -32,6 +33,8 @@ SCENARIOS="local=http://127.0.0.1:9000,distributed=http://127.0.0.1:9001" # Optional Prometheus text exposition URL. # If empty, metrics delta collection is skipped. INTERNODE_METRICS_URL="" +INTERNODE_METRICS_MODE="text-delta" +INTERNODE_METRICS_RANGE="" usage() { cat <<'USAGE' @@ -53,7 +56,9 @@ Optional: --samples Default: 20000 (for s3bench) --warp-bin Default: warp --s3bench-bin Default: s3bench - --metrics-url Prometheus text endpoint for internode metrics delta + --metrics-url Metrics URL for internode deltas. Use a Prometheus text endpoint with text-delta mode, or /api/v1/query with prometheus-increase mode + --metrics-mode text-delta or prometheus-increase. Default: text-delta + --metrics-range PromQL range for prometheus-increase mode. Default: --duration --out-dir Default: target/bench/internode-transport- --extra-args "" Passed to run_object_batch_bench.sh --extra-args --insecure TLS insecure (self-signed) @@ -62,6 +67,7 @@ Optional: Notes: - This baseline covers S3 PUT/GET workloads and records internode metric deltas when --metrics-url is set. + - Use --metrics-mode prometheus-increase with a Prometheus /api/v1/query URL when service restarts may reset counters between runs. - The run manifest intentionally omits access keys, secret keys, and extra args to avoid writing credentials to artifacts. - Healing/replication-specific workloads should be run separately and appended to the same artifact directory. USAGE @@ -91,6 +97,8 @@ parse_args() { --warp-bin) WARP_BIN="$2"; shift 2 ;; --s3bench-bin) S3BENCH_BIN="$2"; shift 2 ;; --metrics-url) INTERNODE_METRICS_URL="$2"; shift 2 ;; + --metrics-mode) INTERNODE_METRICS_MODE="$2"; shift 2 ;; + --metrics-range) INTERNODE_METRICS_RANGE="$2"; shift 2 ;; --out-dir) OUT_DIR="$2"; shift 2 ;; --extra-args) EXTRA_ARGS="$2"; shift 2 ;; --insecure) INSECURE=true; shift ;; @@ -118,6 +126,13 @@ validate_args() { echo "ERROR: --scenarios cannot be empty" >&2 exit 1 fi + if [[ "${INTERNODE_METRICS_MODE}" != "text-delta" && "${INTERNODE_METRICS_MODE}" != "prometheus-increase" ]]; then + echo "ERROR: --metrics-mode must be text-delta or prometheus-increase" >&2 + exit 1 + fi + if [[ "${INTERNODE_METRICS_MODE}" == "prometheus-increase" && -n "${INTERNODE_METRICS_URL}" && -z "${INTERNODE_METRICS_RANGE}" ]]; then + INTERNODE_METRICS_RANGE="${DURATION}" + fi } setup_output() { @@ -159,6 +174,8 @@ write_run_manifest() { echo "samples=${SAMPLES}" echo "insecure=${INSECURE}" echo "metrics_url=${INTERNODE_METRICS_URL:-N/A}" + echo "metrics_mode=${INTERNODE_METRICS_MODE}" + echo "metrics_range=${INTERNODE_METRICS_RANGE:-N/A}" echo "out_dir=${OUT_DIR}" echo "extra_args_present=$([[ -n "${EXTRA_ARGS}" ]] && echo true || echo false)" echo "access_key=REDACTED" @@ -166,6 +183,72 @@ write_run_manifest() { } > "${manifest}" } +collect_internode_increases() { + local snapshot_file="$1" + if [[ -z "${INTERNODE_METRICS_URL}" ]]; then + : > "${snapshot_file}" + return 0 + fi + if [[ "${DRY_RUN}" == "true" ]]; then + : > "${snapshot_file}" + return 0 + fi + + python3 - "${INTERNODE_METRICS_URL}" "${INTERNODE_METRICS_RANGE}" "${snapshot_file}" <<'PY' +import json +import pathlib +import sys +import urllib.parse +import urllib.request + +query_url, prom_range, snapshot_raw = sys.argv[1:] +snapshot_path = pathlib.Path(snapshot_raw) +metrics = [ + "rustfs_system_network_internode_operation_requests_outgoing_total", + "rustfs_system_network_internode_operation_requests_incoming_total", + "rustfs_system_network_internode_operation_errors_total", + "rustfs_system_network_internode_operation_classified_errors_total", +] + + +def escape_label(value): + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +def query_prometheus(query): + separator = "&" if "?" in query_url else "?" + url = f"{query_url}{separator}{urllib.parse.urlencode({'query': query})}" + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=30) as response: + payload = json.loads(response.read().decode("utf-8")) + if payload.get("status") != "success": + raise RuntimeError(f"Prometheus query failed: {payload!r}") + return payload.get("data", {}).get("result", []) + + +lines = [] +try: + for metric_name in metrics: + query = f"sum by(operation, backend) (increase({metric_name}[{prom_range}]))" + for sample in query_prometheus(query): + labels = { + key: value + for key, value in sample.get("metric", {}).items() + if key in {"operation", "backend"} + } + value = sample.get("value", [None, None])[1] + if value is None: + continue + label_text = ",".join(f'{key}="{escape_label(value)}"' for key, value in sorted(labels.items())) + lines.append(f"{metric_name}{{{label_text}}} {value}") +except Exception as err: # noqa: BLE001 - shell harness reports and skips metrics on query errors. + snapshot_path.write_text(f"# prometheus_increase_failed error={err}\n", encoding="utf-8") + raise SystemExit(0) + +snapshot_path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8") +PY +} + collect_internode_snapshot() { local snapshot_file="$1" if [[ -z "${INTERNODE_METRICS_URL}" ]]; then @@ -202,7 +285,7 @@ extract_internode_rows() { n = split($0, parts, " ") value = parts[n] gsub(/[[:space:]]+/, "", value) - if (value ~ /^[0-9]+([.][0-9]+)?$/) { + if (value ~ /^[-+]?[0-9]+([.][0-9]+)?([eE][-+]?[0-9]+)?$/) { print metric "," op "," backend "," value } }' "${src}" @@ -243,6 +326,30 @@ append_metric_deltas() { rm -f "${before_rows}" "${after_rows}" } +append_metric_increases() { + local scenario="$1" + local workload="$2" + local conc="$3" + local size="$4" + local increase_file="$5" + + local increase_rows + increase_rows="$(mktemp)" + extract_internode_rows "${increase_file}" > "${increase_rows}" + + awk -F',' -v scenario="${scenario}" -v workload="${workload}" -v conc="${conc}" -v size="${size}" ' + { + metric = $1 + operation = $2 + backend = $3 + delta = $4 + 0 + printf "%s,%s,%s,%s,%s,%s,%s,%.0f,%.0f,%.0f\n", scenario, workload, conc, size, metric, operation, backend, 0, delta, delta + } + ' "${increase_rows}" >> "${OUT_DIR}/internode_metric_deltas.csv" + + rm -f "${increase_rows}" +} + append_object_summary() { local scenario="$1" local endpoint="$2" @@ -274,9 +381,10 @@ run_workload() { local bucket="${BUCKET_PREFIX}-${scenario}-${workload}-c${conc}" local before_metrics="${run_dir}/metrics_before.prom" local after_metrics="${run_dir}/metrics_after.prom" + local increase_metrics="${run_dir}/metrics_increase.prom" mkdir -p "${run_dir}" - if [[ -n "${INTERNODE_METRICS_URL}" ]]; then + if [[ -n "${INTERNODE_METRICS_URL}" && "${INTERNODE_METRICS_MODE}" == "text-delta" ]]; then collect_internode_snapshot "${before_metrics}" fi @@ -311,9 +419,12 @@ run_workload() { "${cmd[@]}" append_object_summary "${scenario}" "${endpoint}" "${workload}" "${conc}" "${run_dir}" - if [[ -n "${INTERNODE_METRICS_URL}" ]]; then + if [[ -n "${INTERNODE_METRICS_URL}" && "${INTERNODE_METRICS_MODE}" == "text-delta" ]]; then collect_internode_snapshot "${after_metrics}" append_metric_deltas "${scenario}" "${workload}" "${conc}" "all_sizes" "${before_metrics}" "${after_metrics}" + elif [[ -n "${INTERNODE_METRICS_URL}" && "${INTERNODE_METRICS_MODE}" == "prometheus-increase" ]]; then + collect_internode_increases "${increase_metrics}" + append_metric_increases "${scenario}" "${workload}" "${conc}" "all_sizes" "${increase_metrics}" fi } @@ -350,7 +461,11 @@ main() { validate_args require_cmd awk if [[ -n "${INTERNODE_METRICS_URL}" && "${DRY_RUN}" != "true" ]]; then - require_cmd curl + if [[ "${INTERNODE_METRICS_MODE}" == "text-delta" ]]; then + require_cmd curl + else + require_cmd python3 + fi fi if [[ ! -x "${OBJECT_BENCH_SCRIPT}" ]]; then echo "ERROR: benchmark script not executable: ${OBJECT_BENCH_SCRIPT}" >&2 From 66da8565c96b5a96c8bf86206b4b8d598e7caea2 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 23 Aug 2026 13:31:02 +0800 Subject: [PATCH 02/41] chore(deps): update flake.lock (#6436) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/8be7bd0' (2026-08-14) → 'github:NixOS/nixpkgs/391b592' (2026-08-20) • Updated input 'rust-overlay': 'github:oxalica/rust-overlay/b211ead' (2026-08-16) → 'github:oxalica/rust-overlay/f60c1b5' (2026-08-23) --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 27fdbf27e..320e9d054 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1786719841, - "narHash": "sha256-QcpQOT0NQEFkI77t+YXPZqDJc35iIodG7zinieOwFUg=", + "lastModified": 1787209939, + "narHash": "sha256-WvvHR4kSQLAbtouMC/ruZ5UpLwlUcY3K4FAllMN+yGk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8be7bd0c83f12e2e3bbba07c9044d6fed9e66f7f", + "rev": "391b592eb44808b3bd0cb80bb71b63a5a118b8bb", "type": "github" }, "original": { @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1786849542, - "narHash": "sha256-MS8cOa/ii1+QA8R3JWVzqEIoXimu9n50GwQDiBVTFOM=", + "lastModified": 1787454509, + "narHash": "sha256-r4LDUF+zmJnkftvCVkCrUhSJazsf6EVJF+V2l4/MYbI=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "b211eadeba8b180da9453ec3413a8a3535c85b3f", + "rev": "f60c1b57ff805a46b5175c76fc981fb4f81efbcc", "type": "github" }, "original": { From ab8f8b94dcfa4984d95c91a10c4322ec5fde8a74 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 23 Aug 2026 13:47:33 +0800 Subject: [PATCH 03/41] perf(runtime): enable fsync thread isolation by default (#6438) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change DEFAULT_FSYNC_BLOCKING_THREADS from 0 to 64 to isolate fsync/fdatasync operations into a dedicated blocking thread pool. A/B validation on 4-node EC cluster (testing, 10.0.0.5/8/9/11:9000): PUT 256KiB c16: p99 226ms → 135ms (−40%), p50 60ms → 19ms (−68%) GET 256KiB c16: p99 3.97ms → 3.63ms (−9%), throughput +1.8% GET 4KiB c64: neutral (pure read, no fsync involvement) Without isolation, fsync operations contend with read I/O (pread/stat/open) on the main blocking pool, causing device-bound fsync to starve read operations under mixed PUT+GET workloads. Co-authored-by: heihutu --- crates/config/src/constants/runtime.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/config/src/constants/runtime.rs b/crates/config/src/constants/runtime.rs index 783ad6993..36fc33141 100644 --- a/crates/config/src/constants/runtime.rs +++ b/crates/config/src/constants/runtime.rs @@ -60,9 +60,9 @@ pub const DEFAULT_RNG_SEED: Option = None; // None means random /// Dedicated blocking thread pool for fsync/fdatasync operations. /// When > 1, fsync operations are isolated from the main blocking pool to /// prevent device-bound fsync from starving read operations (pread/stat/open). -/// Default 0 means auto (no isolation, use main runtime). +/// Default 64 isolates fsync from the main blocking pool to prevent device-bound fsync from starving read I/O. pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS"; -pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0; +pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 64; // Dial9 Tokio Telemetry Default values pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default From ba4cd694389e0e48c718da7f1267c21d1c124489 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 23 Aug 2026 15:41:25 +0800 Subject: [PATCH 04/41] fix(ecstore): default rename fanout to parallel early-ack path (#6443) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(allocator): replace mimalloc/libmimalloc-sys with rustfs-mimalloc/rustfs-mimalloc-sys Replace the upstream xonatius/mimalloc_rust.git fork (mimalloc + libmimalloc-sys) with the published rustfs-mimalloc (v0.5.0) and rustfs-mimalloc-sys (v0.5.0) crates from crates.io. The new crates are based on mimalloc V3 (v3.5.0) and provide: - MiMalloc global allocator with safe API (collect, stats_json, process_info) - Heap management and arena operations (heap module) - Full FFI bindings to mimalloc V3 Changes: - Workspace deps: mimalloc + libmimalloc-sys (git) → rustfs-mimalloc + rustfs-mimalloc-sys (crates.io) - allocator_reclaim.rs: libmimalloc_sys::mi_collect → rustfs_mimalloc::MiMalloc::collect - memory_observability.rs: raw FFI mi_stats_get_json → MiMalloc::stats_json() - main.rs: heap ownership tests use Heap::contains() (V3 API) - deny.toml: remove xonatius/mimalloc_rust.git from allow-git Co-Authored-By: heihutu * fix(ecstore): default rename fanout to parallel early-ack path Switch the default rename_data commit fanout from serial join_all to the parallel JoinSet early-ack path. The serial path (#5987) was the primary cause of the 1MiB PUT regression (-71.7%) observed in rc.3 benchmarks. A/B verification on testing 4-node cluster (c=64, 1MiB PUT, 2min): - Serial (join_all): 96.99 MiB/s, P50=644ms - Early ack (JoinSet): 177.46 MiB/s, P50=407ms (+83%) Also: - Update rename_data_reclaims_synthetic_inline_rollback_dir_after_commit to use rename_data_owned and await tail_drain for proper cleanup. - Update rename_data_waits_for_tail_disk_after_write_quorum to explicitly test the serial path (now non-default) via env override. - Add error source chain to HTTP Body stream transport error log (backlog#2005) so the underlying cause is visible. Ref: rustfs/backlog#2005 Ref: rustfs/backlog#1792#issuecomment-5384346238 Ref: rustfs/backlog#1792#issuecomment-5384370938 Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- .../src/set_disk/core/io_primitives.rs | 98 +++++++++++-------- rustfs/src/server/http.rs | 8 ++ 2 files changed, 63 insertions(+), 43 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 1a77ba267..88125cbac 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3472,7 +3472,7 @@ type RenameDataLegacyTuple = ( ); fn put_rename_early_ack_enabled() -> bool { - rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, false) + rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true) } impl RenameDataCommit { @@ -8631,17 +8631,21 @@ mod tests { inline_fi.mod_time = Some(OffsetDateTime::now_utc()); std::fs::create_dir_all(disk_root.join(RUSTFS_META_TMP_BUCKET).join("tmp-inline")) .expect("inline staging dir should be created"); - SetDisks::rename_data( + // Use rename_data_owned so we can await the tail_drain for cleanup. + let commit = SetDisks::rename_data_owned( std::slice::from_ref(&online_disk), RUSTFS_META_TMP_BUCKET, "tmp-inline", - std::slice::from_ref(&inline_fi), + vec![inline_fi], bucket, object, 1, ) .await .expect("inline version should commit"); + if let Some(td) = commit.tail_drain { + td.await.expect("inline commit tail drain must succeed"); + } // Overwrite the same (nil) version with a non-inline one. let new_data_dir = Uuid::new_v4(); @@ -8654,17 +8658,20 @@ mod tests { .join(new_data_dir.to_string()); std::fs::create_dir_all(&staged_data_dir).expect("streaming staging dir should be created"); std::fs::write(staged_data_dir.join("part.1"), b"streamed-body").expect("staged part should be written"); - SetDisks::rename_data( + let commit = SetDisks::rename_data_owned( std::slice::from_ref(&online_disk), RUSTFS_META_TMP_BUCKET, "tmp-streaming", - std::slice::from_ref(&streaming_fi), + vec![streaming_fi], bucket, object, 1, ) .await .expect("non-inline overwrite should commit"); + if let Some(td) = commit.tail_drain { + td.await.expect("non-inline overwrite tail drain must succeed"); + } let mut leftovers: Vec = std::fs::read_dir(disk_root.join(bucket).join(object)) .expect("committed object dir should be readable") @@ -8825,49 +8832,54 @@ mod tests { #[tokio::test] #[serial_test::serial(rename_quorum_ack)] async fn rename_data_waits_for_tail_disk_after_write_quorum() { - const DISKS: usize = 4; - let bucket = "rename-tail-success-bucket"; - let object = "rename-tail-success-object"; - let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; - prepare_rename_source_dirs(&dirs, &disks, "source").await; - let file_infos = rename_commit_fileinfos(object, DISKS, "tail-success-etag"); - let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); + // Explicitly test the serial (join_all) path: early ack is now the + // default, so disable it to verify the legacy behaviour still works. + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async { + const DISKS: usize = 4; + let bucket = "rename-tail-success-bucket"; + let object = "rename-tail-success-object"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let file_infos = rename_commit_fileinfos(object, DISKS, "tail-success-etag"); + let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME); - let rename = SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 3); - tokio::pin!(rename); - tokio::time::timeout(BARRIER_PAUSE_GUARD, async { - tokio::select! { - () = barrier.wait_until_paused() => {} - result = &mut rename => panic!("rename_data returned before the armed fan-out barrier: {result:?}"), - } - }) - .await - .expect("paused disk must reach the armed rename barrier"); - - assert!( - tokio::time::timeout(Duration::from_millis(50), &mut rename).await.is_err(), - "current rename_data waits for the paused fan-out disk even after the other three disks can reach write quorum" - ); - - barrier.release(); - rename + let rename = SetDisks::rename_data(&disks, RUSTFS_META_TMP_BUCKET, "source", &file_infos, bucket, object, 3); + tokio::pin!(rename); + tokio::time::timeout(BARRIER_PAUSE_GUARD, async { + tokio::select! { + () = barrier.wait_until_paused() => {} + result = &mut rename => panic!("rename_data returned before the armed fan-out barrier: {result:?}"), + } + }) .await - .expect("tail success must complete the rename after the barrier is released"); + .expect("paused disk must reach the armed rename barrier"); - for (idx, dir) in dirs.iter().enumerate() { - let reopened = reopen_local_disk(dir).await; - let stored = reopened - .read_version("", bucket, object, "", &ReadOptions::default()) - .await - .unwrap_or_else(|err| panic!("disk {idx} must contain the tail-success commit after reopen: {err:?}")); - assert_eq!( - stored.metadata.get("etag").map(String::as_str), - Some("tail-success-etag"), - "disk {idx} must expose the same committed metadata after tail success and reopen" + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut rename).await.is_err(), + "serial rename_data waits for the paused fan-out disk even after write quorum" ); - } - drop(dirs); + barrier.release(); + rename + .await + .expect("tail success must complete the rename after the barrier is released"); + + for (idx, dir) in dirs.iter().enumerate() { + let reopened = reopen_local_disk(dir).await; + let stored = reopened + .read_version("", bucket, object, "", &ReadOptions::default()) + .await + .unwrap_or_else(|err| panic!("disk {idx} must contain the tail-success commit after reopen: {err:?}")); + assert_eq!( + stored.metadata.get("etag").map(String::as_str), + Some("tail-success-etag"), + "disk {idx} must expose the same committed metadata after tail success and reopen" + ); + } + + drop(dirs); + }) + .await; } #[tokio::test] diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index ca421962f..20253d2df 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -1831,6 +1831,13 @@ fn handle_connection_error(peer_addr: Option<&str>, err: &(dyn std::error::Error } else if hyper_err.is_parse() { log_transport_failed(peer_addr, "parse_failure", &hyper_err.to_string()); } else if hyper_err.is_user() { + // is_user() = "error from user's Body stream": the application + // returned a streaming body that failed mid-flight. Log the full + // error source chain so the underlying cause (disk read failure, + // upstream RPC error, deleted object, etc.) is visible. + let cause = std::error::Error::source(hyper_err) + .map(|e| e.to_string()) + .unwrap_or_default(); error!( event = EVENT_HTTP_TRANSPORT_FAILED, component = LOG_COMPONENT_SERVER, @@ -1838,6 +1845,7 @@ fn handle_connection_error(peer_addr: Option<&str>, err: &(dyn std::error::Error peer_addr = %peer_addr, error_kind = "service_error", error = %hyper_err, + cause = %cause, result = "transport_error", "HTTP transport failed" ); From 0d30c69e5f4367dd000f22d27dd03caee1169a47 Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 23 Aug 2026 15:41:37 +0800 Subject: [PATCH 05/41] perf(ecstore): reduce batch read identity cloning (#6441) Co-authored-by: heihutu --- .../src/set_disk/core/io_primitives.rs | 26 +++++++++++++++++-- crates/ecstore/src/store/object.rs | 22 +++++++++++----- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 88125cbac..853eff9fd 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -140,6 +140,21 @@ struct CoalescedReadVersionRequest { tx: oneshot::Sender>, } +#[derive(Clone, Debug, Eq, PartialEq)] +struct ExpectedBatchReadVersionItem { + path: String, + version_id: String, +} + +impl From<&BatchReadVersionItem> for ExpectedBatchReadVersionItem { + fn from(item: &BatchReadVersionItem) -> Self { + Self { + path: item.path.clone(), + version_id: item.version_id.clone(), + } + } +} + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] struct ReadVersionCoalescerKey { disk: usize, @@ -266,7 +281,7 @@ async fn flush_read_version_coalescer_pending( items.push(request.item); } - let expected_items = items.clone(); + let expected_items = items.iter().map(ExpectedBatchReadVersionItem::from).collect::>(); record_read_version_coalescer_event("attempted_batch", items.len()); let result = match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts })) @@ -292,7 +307,7 @@ async fn flush_read_version_coalescer_pending( } fn map_batch_read_version_responses( - expected_items: &[BatchReadVersionItem], + expected_items: &[ExpectedBatchReadVersionItem], responses: Vec, ) -> Vec> { let mut results = (0..expected_items.len()) @@ -6878,6 +6893,7 @@ mod tests { }, ]; + let expected_items = expected_batch_read_version_items(&expected_items); let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter(); let first = results .next() @@ -6918,6 +6934,7 @@ mod tests { version_id: "v-b".to_string(), }, ]; + let expected_items = expected_batch_read_version_items(&expected_items); let results = map_batch_read_version_responses( &expected_items, vec![ @@ -6957,6 +6974,7 @@ mod tests { path: "object-a".to_string(), version_id: "v-a".to_string(), }]; + let expected_items = expected_batch_read_version_items(&expected_items); let mismatched = map_batch_read_version_responses( &expected_items, vec![BatchReadVersionResp { @@ -7018,6 +7036,10 @@ mod tests { ); } + fn expected_batch_read_version_items(items: &[BatchReadVersionItem]) -> Vec { + items.iter().map(ExpectedBatchReadVersionItem::from).collect() + } + /// Isolation guard: unobserved objects record nothing (so parallel tests do /// not inflate one another), and a scope clears its own counts on drop. #[tokio::test] diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 7a8653415..e44e8dedf 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -1203,7 +1203,8 @@ fn inject_batch_delete_pool_errors( bucket: &str, pool_idx: usize, object_names: &[String], - result: &mut (Vec, Vec>), + deleted: &[DeletedObject], + errors: &mut [Option], ) { let state = BATCH_DELETE_POOL_ERROR_INJECTION .get_or_init(|| std::sync::Mutex::new(None)) @@ -1220,8 +1221,8 @@ fn inject_batch_delete_pool_errors( let Some(error) = state.errors.get(object_name) else { continue; }; - if result.1[idx].is_none() && result.0[idx].found { - result.1[idx] = Some(error.clone()); + if errors[idx].is_none() && deleted[idx].found { + errors[idx] = Some(error.clone()); state.observed.fetch_add(1, Ordering::AcqRel); } } @@ -3194,7 +3195,7 @@ impl ECStore { // Default return value let mut del_objects = vec![DeletedObject::default(); objects.len()]; - let accounting = vec![None; objects.len()]; + let mut accounting = vec![None; objects.len()]; let mut del_errs = Vec::with_capacity(objects.len()); for _ in 0..objects.len() { @@ -3333,11 +3334,12 @@ impl ECStore { .iter() .map(|object| object.object_name.clone()) .collect::>(); - let result = pool.delete_objects(bucket, pool_objects, pool_opts).await; + let result = pool.delete_objects_with_accounting(bucket, pool_objects, pool_opts).await; #[cfg(test)] let result = { let mut result = result; - inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, &mut result); + let (deleted, errors, _) = &mut result; + inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, deleted, errors); result }; (object_indices, result) @@ -3347,7 +3349,7 @@ impl ECStore { let results = join_all(futures).await; for idx in 0..del_objects.len() { - let pool_results = results.iter().filter_map(|(object_indices, (dels, errs))| { + let pool_results = results.iter().filter_map(|(object_indices, (dels, errs, _))| { let pool_object_idx = object_indices.binary_search(&idx).ok()?; Some((&dels[pool_object_idx], &errs[pool_object_idx])) }); @@ -3367,6 +3369,12 @@ impl ECStore { } } + for (object_indices, (_, _, pool_accounting)) in &results { + for (pool_object_idx, object_idx) in object_indices.iter().enumerate() { + accounting[*object_idx] = pool_accounting.get(pool_object_idx).cloned().flatten(); + } + } + #[cfg(test)] for (idx, object) in objects.iter().enumerate() { if del_errs[idx].is_none() && del_objects[idx].delete_marker { From c442c543d36da2cac9517dae7f51ac8d64d7d97f Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 15:55:06 +0800 Subject: [PATCH 06/41] fix(ecstore): merge peer pool meta reload monotonically (#6392) The peer reload_pool_meta handler blindly replaced in-memory pool metadata with the persisted snapshot, so a delayed or out-of-order reload could roll back newer local queued/canceled/failed/complete decommission state, and a missing pool.bin wiped local state to an empty default. Route peer reload through the same monotonic merge used by the admin status refresh (merge_pool_status_refresh): entries are replaced only when strictly newer and no local worker is active; missing snapshots fail closed. The helper now reports whether any entry was replaced or appended, and rejected stale/missing reloads are logged. The RPC handler spawns missing decommission workers only after a reload actually merged newer state, so duplicate deliveries cannot start workers for an older generation. Fixes rustfs/backlog#1917 --- crates/ecstore/src/core/pools.rs | 64 ++++- crates/ecstore/src/store/rebalance.rs | 327 ++++++++++++++++++++++++- rustfs/src/storage/rpc/node_service.rs | 8 +- 3 files changed, 389 insertions(+), 10 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 4ca460c20..0c39d1f57 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -1056,16 +1056,22 @@ fn should_replace_pool_status_for_status_refresh( !has_active_worker && persisted.last_update > current.last_update } -fn merge_pool_status_refresh(current: &mut PoolMeta, persisted: PoolMeta, active_workers: &[bool]) { +/// Merges a persisted pool metadata snapshot into `current` monotonically: +/// a pool entry is replaced only when no active worker covers it and the +/// snapshot is strictly newer, so delayed snapshots never roll back local +/// queued/terminal progressions. Returns whether any entry was replaced or +/// appended. +pub(crate) fn merge_pool_status_refresh(current: &mut PoolMeta, persisted: PoolMeta, active_workers: &[bool]) -> bool { if persisted.pools.is_empty() { - return; + return false; } if current.pools.is_empty() { *current = persisted; - return; + return true; } + let mut merged_newer = false; for (idx, persisted_pool) in persisted.pools.into_iter().enumerate() { if persisted_pool.id != idx { continue; @@ -1075,11 +1081,14 @@ fn merge_pool_status_refresh(current: &mut PoolMeta, persisted: PoolMeta, active if idx < current.pools.len() { if should_replace_pool_status_for_status_refresh(current.pools.get(idx), &persisted_pool, has_active_worker) { current.pools[idx] = persisted_pool; + merged_newer = true; } } else if idx == current.pools.len() && !has_active_worker { current.pools.push(persisted_pool); + merged_newer = true; } } + merged_newer } fn resolve_start_decommission_pool_meta_reload_result(result: Result<()>) -> Result<()> { @@ -6621,6 +6630,55 @@ mod pools_tests { assert_eq!(info.bytes_done, 1_024); } + #[test] + fn test_merge_pool_status_refresh_fails_closed_on_missing_persisted_pools() { + let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid"); + let mut current = PoolMeta { + pools: vec![decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + )], + ..Default::default() + }; + current.pools[0].last_update = newer; + + assert!( + !merge_pool_status_refresh(&mut current, PoolMeta::default(), &[false]), + "an empty persisted snapshot must fail closed instead of replacing local state" + ); + + let info = current.pools[0] + .decommission + .as_ref() + .expect("local decommission info should survive a missing snapshot"); + assert!(info.complete); + assert_eq!(current.pools[0].last_update, newer); + } + + #[test] + fn test_merge_pool_status_refresh_ignores_mislabeled_pool_entries() { + let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid"); + let mut current = PoolMeta { + pools: vec![decommission_test_pool_status(0, None)], + ..Default::default() + }; + let mut persisted = PoolMeta { + pools: vec![decommission_test_pool_status(0, Some(PoolDecommissionInfo::default()))], + ..Default::default() + }; + persisted.pools[0].id = 7; + persisted.pools[0].last_update = older; + + assert!( + !merge_pool_status_refresh(&mut current, persisted, &[false]), + "a pool entry whose id does not match its index must be ignored" + ); + assert!(current.pools[0].decommission.is_none()); + } + #[test] fn test_dedup_indices_removes_duplicates_preserving_order() { assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]); diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index dc2fa2ee5..c7a4e35c6 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -14,10 +14,15 @@ use super::*; use crate::config::storageclass; +use crate::core::pools::merge_pool_status_refresh; use crate::layout::pool_space::{ServerPoolsAvailableSpace, build_server_pools_available_space}; use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, object::ObjectOperations as _}; pub(in crate::store) mod support; + +const LOG_COMPONENT_ECSTORE: &str = "ecstore"; +const LOG_SUBSYSTEM_POOLS: &str = "pools"; +const EVENT_POOL_META_RELOAD: &str = "pool_meta_reload"; use support::{ LatestObjectInfoCandidate, PoolErr, PoolObjInfo, RebalanceDeletePoolResult, pool_lookup_not_found_error, rebalance_disk_set_lookup_error, resolve_latest_object_info_candidates, resolve_rebalance_delete_from_all_pools_result, @@ -684,17 +689,49 @@ impl ECStore { ) } - pub async fn reload_pool_meta(&self) -> Result<()> { - let mut meta = PoolMeta::default(); + /// Peer reload entry: refreshes in-memory pool metadata from the shared + /// persisted snapshot. Returns whether newer state was actually merged so + /// callers only trigger missing-worker recovery after a real state change; + /// delayed snapshots are merged monotonically and never blind-assigned. + pub async fn reload_pool_meta(&self) -> Result { + let mut reloaded = PoolMeta::default(); resolve_store_rebalance_pool_meta_reload_result( - meta.load(self.pools[0].clone(), self.pools.clone()).await, + reloaded.load(self.pools[0].clone(), self.pools.clone()).await, "reload_pool_meta", )?; + // Lock order: release the decommission_cancelers guard before taking + // the pool_meta write guard; neither is held across the disk read. + let active_workers = { + let cancelers = self.decommission_cancelers.read().await; + cancelers.iter().map(Option::is_some).collect::>() + }; + + let incoming_has_pools = !reloaded.pools.is_empty(); let mut pool_meta = self.pool_meta.write().await; - *pool_meta = meta; - // *self.pool_meta.write().expect("operation should succeed") = meta; - Ok(()) + let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers); + + if !merged_newer && !incoming_has_pools { + warn!( + event = EVENT_POOL_META_RELOAD, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + result = "ignored", + reason = "missing_metadata", + "Peer pool meta reload ignored because persisted metadata is missing" + ); + } else if !merged_newer { + debug!( + event = EVENT_POOL_META_RELOAD, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + result = "ignored", + reason = "stale_snapshot", + "Peer pool meta reload ignored as a stale snapshot" + ); + } + + Ok(merged_newer) } /// Disk information deduplication function @@ -861,6 +898,7 @@ mod tests { use super::*; use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType}; use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env}; + use crate::core::pools::{POOL_META_VERSION, PoolDecommissionInfo, PoolStatus}; use crate::disk::error::DiskError; use crate::layout::endpoint::Endpoint; use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; @@ -871,6 +909,7 @@ mod tests { use rustfs_config::server_config::KVS; use rustfs_filemeta::FileInfo; use std::sync::Arc; + use time::{Duration as TimeDuration, OffsetDateTime}; use tokio_util::sync::CancellationToken; async fn setup_multi_pool_test_store( @@ -2097,4 +2136,280 @@ mod tests { .contains("failed to resolve rebalance disk set: pool index 2, set index 7, pool count 3") ); } + + fn reload_test_pool_status(decommission: Option, last_update: time::OffsetDateTime) -> PoolStatus { + PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update, + decommission, + } + } + + fn reload_test_pool_meta(pool: PoolStatus) -> PoolMeta { + PoolMeta { + version: POOL_META_VERSION, + pools: vec![pool], + dont_save: false, + } + } + + async fn persist_reload_snapshot(store: &ECStore, snapshot: &PoolMeta) { + snapshot + .save(store.pools.clone()) + .await + .expect("pool meta snapshot should persist to every pool"); + } + + #[tokio::test] + #[serial_test::serial] + async fn peer_pool_meta_reload_does_not_rollback_newer_local_states() { + let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-stale", &[2]).await; + + let stale_time = OffsetDateTime::now_utc(); + let newer_time = stale_time + TimeDuration::seconds(30); + let progressed_states: [(&str, PoolDecommissionInfo); 4] = [ + ( + "queued", + PoolDecommissionInfo { + queued: true, + start_time: Some(stale_time), + ..Default::default() + }, + ), + ( + "canceled", + PoolDecommissionInfo { + canceled: true, + start_time: Some(stale_time), + ..Default::default() + }, + ), + ( + "failed", + PoolDecommissionInfo { + failed: true, + start_time: Some(stale_time), + ..Default::default() + }, + ), + ( + "complete", + PoolDecommissionInfo { + complete: true, + start_time: Some(stale_time), + ..Default::default() + }, + ), + ]; + + for (state_label, local_state) in progressed_states { + { + let mut pool_meta = store.pool_meta.write().await; + *pool_meta = reload_test_pool_meta(reload_test_pool_status(Some(local_state.clone()), newer_time)); + } + + // A delayed peer message carries a snapshot that predates the local progression. + let stale_snapshot = reload_test_pool_meta(reload_test_pool_status( + Some(PoolDecommissionInfo { + start_time: Some(stale_time), + ..Default::default() + }), + stale_time, + )); + persist_reload_snapshot(&store, &stale_snapshot).await; + + let merged_newer = store.reload_pool_meta().await.expect("stale reload should succeed"); + assert!( + !merged_newer, + "a delayed reload must not report merged newer state for the {state_label} progression" + ); + + let pool_meta = store.pool_meta.read().await; + let info = pool_meta.pools[0] + .decommission + .as_ref() + .expect("local decommission state should survive a stale reload"); + assert_eq!(info.queued, local_state.queued, "{state_label} queued flag must not roll back"); + assert_eq!(info.canceled, local_state.canceled, "{state_label} canceled flag must not roll back"); + assert_eq!(info.failed, local_state.failed, "{state_label} failed flag must not roll back"); + assert_eq!(info.complete, local_state.complete, "{state_label} complete flag must not roll back"); + assert_eq!( + pool_meta.pools[0].last_update, newer_time, + "{state_label} progress timestamp must be kept" + ); + } + + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial] + async fn peer_pool_meta_reload_merges_newer_state_and_is_idempotent_on_duplicate_delivery() { + let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-duplicate", &[2]).await; + + let older_time = OffsetDateTime::now_utc(); + let newer_time = older_time + TimeDuration::seconds(30); + + { + let mut pool_meta = store.pool_meta.write().await; + *pool_meta = reload_test_pool_meta(reload_test_pool_status( + Some(PoolDecommissionInfo { + items_decommissioned: 1, + ..Default::default() + }), + older_time, + )); + } + let newer_snapshot = reload_test_pool_meta(reload_test_pool_status( + Some(PoolDecommissionInfo { + complete: true, + items_decommissioned: 10, + ..Default::default() + }), + newer_time, + )); + persist_reload_snapshot(&store, &newer_snapshot).await; + + let merged_newer = store.reload_pool_meta().await.expect("first reload should succeed"); + assert!(merged_newer, "a strictly newer persisted snapshot must merge"); + + { + let pool_meta = store.pool_meta.read().await; + let info = pool_meta.pools[0].decommission.as_ref().expect("merged decommission state"); + assert!(info.complete); + assert_eq!(info.items_decommissioned, 10); + assert_eq!(pool_meta.pools[0].last_update, newer_time); + } + + // Redelivering the same generation must be a no-op. + let duplicate_merged = store.reload_pool_meta().await.expect("duplicate reload should succeed"); + assert!(!duplicate_merged, "a duplicate delivery must not re-apply merged state"); + + { + let pool_meta = store.pool_meta.read().await; + let info = pool_meta.pools[0].decommission.as_ref().expect("merged decommission state"); + assert!(info.complete); + assert_eq!(info.items_decommissioned, 10); + assert_eq!(pool_meta.pools[0].last_update, newer_time); + } + + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial] + async fn peer_pool_meta_reload_keeps_active_worker_progress_over_newer_snapshot() { + let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-worker", &[2]).await; + *store.decommission_cancelers.write().await = vec![Some(CancellationToken::new())]; + + let worker_time = OffsetDateTime::now_utc(); + let newer_time = worker_time + TimeDuration::seconds(30); + + { + let mut pool_meta = store.pool_meta.write().await; + *pool_meta = reload_test_pool_meta(reload_test_pool_status( + Some(PoolDecommissionInfo { + start_time: Some(worker_time), + items_decommissioned: 10, + bytes_done: 1_024, + ..Default::default() + }), + worker_time, + )); + } + // Even a strictly newer terminal snapshot must not override a live worker. + let newer_terminal_snapshot = reload_test_pool_meta(reload_test_pool_status( + Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + newer_time, + )); + persist_reload_snapshot(&store, &newer_terminal_snapshot).await; + + let merged_newer = store + .reload_pool_meta() + .await + .expect("reload under an active worker should succeed"); + assert!(!merged_newer, "an active local worker must block snapshot replacement"); + + let pool_meta = store.pool_meta.read().await; + let info = pool_meta.pools[0] + .decommission + .as_ref() + .expect("worker progress should remain"); + assert!(!info.complete); + assert_eq!(info.items_decommissioned, 10); + assert_eq!(info.bytes_done, 1_024); + assert_eq!(pool_meta.pools[0].last_update, worker_time); + + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial] + async fn peer_pool_meta_reload_fails_closed_when_persisted_metadata_is_missing() { + let (temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-missing", &[2]).await; + + let kept_time = OffsetDateTime::now_utc(); + { + let mut pool_meta = store.pool_meta.write().await; + *pool_meta = reload_test_pool_meta(reload_test_pool_status( + Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + kept_time, + )); + } + // Persist first so the test controls exactly what exists on disk. + persist_reload_snapshot( + &store, + &reload_test_pool_meta(reload_test_pool_status( + Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + kept_time, + )), + ) + .await; + + let mut deleted_any = false; + for disk_index in 0..2 { + let pool_bin_dir = temp_dir + .path() + .join(format!("pool0-disk{disk_index}")) + .join(crate::disk::RUSTFS_META_BUCKET) + .join(crate::core::pools::POOL_META_NAME); + if pool_bin_dir.exists() { + tokio::fs::remove_dir_all(&pool_bin_dir) + .await + .expect("persisted pool metadata object dir should be removable"); + deleted_any = true; + } + } + // The meta-bucket layout may nest objects per pool; fall back to removing + // every pool.bin object directory below the temp root. + if !deleted_any { + panic!("no pool.bin found under {:?}", temp_dir.path()); + } + + let merged_newer = store + .reload_pool_meta() + .await + .expect("reload with missing metadata should fail closed, not error"); + assert!(!merged_newer, "missing persisted metadata must not count as merged state"); + + let pool_meta = store.pool_meta.read().await; + let info = pool_meta.pools[0] + .decommission + .as_ref() + .expect("missing persisted metadata must not default local state away"); + assert!(info.complete); + assert_eq!(pool_meta.pools[0].last_update, kept_time); + + shutdown.cancel(); + } } diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 2503e25c7..d8eaadcb4 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -1987,8 +1987,10 @@ impl Node for NodeService { error_info: Some("errServerNotInitialized".to_string()), })); }; + // Recover missing workers only after the reload merged newer state; a + // stale or duplicate reload must not spawn workers for an older generation. match store.reload_pool_meta().await { - Ok(_) => match store.spawn_missing_local_decommission_routines().await { + Ok(true) => match store.spawn_missing_local_decommission_routines().await { Ok(_) => Ok(Response::new(ReloadPoolMetaResponse { success: true, error_info: None, @@ -1998,6 +2000,10 @@ impl Node for NodeService { error_info: Some(err.to_string()), })), }, + Ok(false) => Ok(Response::new(ReloadPoolMetaResponse { + success: true, + error_info: None, + })), Err(err) => Ok(Response::new(ReloadPoolMetaResponse { success: false, error_info: Some(err.to_string()), From dc8177c2b87712a0a2f51f3642772ce94511b007 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 16:42:07 +0800 Subject: [PATCH 07/41] fix(heal): make resume checkpoints crash consistent (#6340) * fix(heal): make resume checkpoints crash consistent * fix(heal): fail closed on tampered resume checkpoints * fix(heal): atomically authenticate checkpoints * fix(heal): canonicalize checkpoint integrity digest * fix(storage): bound conditional file lock artifacts * fix(heal): require current checkpoint digest * fix(heal): reset unverified checkpoint progress * fix(ecstore): support Windows checkpoint CAS --------- Signed-off-by: houseme Co-authored-by: overtrue Co-authored-by: houseme --- Cargo.lock | 1 + crates/ecstore/src/disk/local.rs | 82 ++++- crates/ecstore/src/disk/os.rs | 85 +++++ crates/heal/Cargo.toml | 1 + crates/heal/src/heal/erasure_healer.rs | 5 + crates/heal/src/heal/resume.rs | 1 + crates/heal/src/heal/resume/checkpoint.rs | 338 +++++++++++++++++-- crates/heal/src/heal/resume/tests.rs | 389 ++++++++++++++++++++++ crates/heal/src/heal/resume/utils.rs | 3 +- 9 files changed, 877 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 517983884..d00968f01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9564,6 +9564,7 @@ dependencies = [ "serde", "serde_json", "serial_test", + "sha2 0.11.0", "temp-env", "tempfile", "thiserror 2.0.20", diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 9994f8612..5217e819b 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -8000,10 +8000,15 @@ impl DiskAPI for LocalDisk { use std::io::Write as _; let file_path = self.io_get_object_path(volume, path)?; - let lock_path = file_path.with_extension("rustfs-cas.lock"); let path = path.to_string(); let sync_metadata = effective_durability(volume).syncs_commit_metadata(); return Ok(tokio::task::spawn_blocking(move || { + // A persistent directory lock bounds metadata growth. Removing + // per-target lock files can split flock ownership across inodes. + let lock_path = file_path + .parent() + .ok_or_else(|| std::io::Error::new(ErrorKind::InvalidInput, "conditional file has no parent"))? + .join(".rustfs-cas.lock"); let lock = std::fs::OpenOptions::new() .create(true) .truncate(false) @@ -8068,7 +8073,25 @@ impl DiskAPI for LocalDisk { .map_err(DiskError::from)??); } - #[cfg(not(unix))] + #[cfg(windows)] + { + let file_path = self.io_get_object_path(volume, path)?; + let sync_metadata = effective_durability(volume).syncs_commit_metadata(); + let publication_root = self.publication_root.clone(); + return Ok(tokio::task::spawn_blocking(move || { + os::compare_and_update_control_file( + &file_path, + expected.as_deref(), + replacement.as_deref(), + sync_metadata, + &publication_root, + ) + }) + .await + .map_err(DiskError::from)??); + } + + #[cfg(not(any(unix, windows)))] { let _ = (volume, path, expected, replacement); Err(DiskError::MethodNotAllowed) @@ -21823,9 +21846,9 @@ mod test { assert!(matches!(results[1].as_ref().unwrap_err(), DiskError::Io(_))); } - #[cfg(unix)] + #[cfg(any(unix, windows))] #[tokio::test] - async fn conditional_file_update_never_deletes_a_new_owner() { + async fn windows_and_unix_conditional_file_update_never_deletes_a_new_owner() { use tempfile::tempdir; let dir = tempdir().expect("temp dir should be created"); @@ -21856,8 +21879,18 @@ mod test { disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH) .await .expect("new owner marker should remain"), - owner_b + owner_b.clone() ); + assert_eq!( + disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Some(owner_b), None) + .await + .expect("current owner should remove marker"), + ConditionalFileUpdate::Updated + ); + assert!(matches!( + disk.read_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH).await, + Err(DiskError::FileNotFound) + )); } #[cfg(unix)] @@ -21872,7 +21905,10 @@ mod test { let marker_path = disk .get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH) .expect("marker path should resolve"); - let lock_path = marker_path.with_extension("rustfs-cas.lock"); + let lock_path = marker_path + .parent() + .expect("marker path should have a parent") + .join(".rustfs-cas.lock"); let lock = std::fs::OpenOptions::new() .create(true) .truncate(false) @@ -21893,6 +21929,40 @@ mod test { assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock)); } + #[cfg(windows)] + #[tokio::test] + async fn windows_conditional_file_update_returns_would_block_when_marker_lock_is_contended() { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse"); + let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created"); + ensure_test_volume(&disk, RUSTFS_META_BUCKET).await; + let marker_path = disk + .get_object_path(RUSTFS_META_BUCKET, HEALING_MARKER_PATH) + .expect("marker path should resolve"); + let lock_path = marker_path + .parent() + .expect("marker path should have a parent") + .join(".rustfs-cas.lock"); + let lock = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(lock_path) + .expect("marker lock should open"); + lock.try_lock().expect("marker lock should be held"); + + let err = tokio::time::timeout( + Duration::from_secs(1), + disk.compare_and_update_file(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, None, Some(Bytes::from_static(b"owner"))), + ) + .await + .expect("contended conditional update must not block") + .expect_err("contended conditional update must retry"); + + assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::WouldBlock)); + } + #[cfg(target_os = "linux")] #[tokio::test] async fn replacement_io_paths_stay_under_the_mount_lease() { diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index 77ea01991..7e6a92557 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(windows)] +use crate::disk::ConditionalFileUpdate; use crate::disk::error::DiskError; use crate::disk::error::Result; use crate::disk::error_conv::to_file_error; @@ -3458,6 +3460,89 @@ fn read_windows_relative_file(file_path: &Path, parent_guard: &ExistingBaseDirec Ok(Some(data)) } +#[cfg(windows)] +pub(crate) fn compare_and_update_control_file( + file_path: &Path, + expected: Option<&[u8]>, + replacement: Option<&[u8]>, + sync_metadata: bool, + publication_root: &PublicationRoot, +) -> io::Result { + use windows_sys::{ + Wdk::Storage::FileSystem::{ + FILE_NON_DIRECTORY_FILE, FILE_OPEN, FILE_OPEN_IF, FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT, + }, + Win32::Storage::FileSystem::{ + DELETE, FILE_ATTRIBUTE_NORMAL, FILE_READ_ATTRIBUTES, FILE_SHARE_READ, FILE_SHARE_WRITE, FILE_WRITE_DATA, SYNCHRONIZE, + }, + }; + + let parent = file_path + .parent() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file has no parent"))?; + let parent_guard = lock_windows_directory_tree(parent, Some(parent), publication_root)?; + let lock = open_windows_relative( + parent_guard.last_handle()?, + std::ffi::OsStr::new(".rustfs-cas.lock"), + SYNCHRONIZE | FILE_READ_ATTRIBUTES | FILE_WRITE_DATA, + FILE_SHARE_READ | FILE_SHARE_WRITE, + FILE_OPEN_IF, + FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT, + FILE_ATTRIBUTE_NORMAL, + true, + )?; + validate_windows_owned_file(&lock)?; + match lock.as_file().try_lock() { + Ok(()) => {} + Err(std::fs::TryLockError::WouldBlock) => return Err(io::Error::from(io::ErrorKind::WouldBlock)), + Err(std::fs::TryLockError::Error(err)) => return Err(err), + } + + let current = read_windows_relative_file(file_path, &parent_guard)?; + let matches = match (¤t, expected) { + (None, None) => true, + (Some(current), Some(expected)) => current.as_slice() == expected, + _ => false, + }; + if !matches { + return Ok(match current { + None => ConditionalFileUpdate::Missing, + Some(_) => ConditionalFileUpdate::Mismatch, + }); + } + + match replacement { + Some(replacement) => RenameDestinationPathGuard { + directory: parent.to_path_buf(), + _directory_guard: parent_guard, + } + .write_file_for_path_access(file_path, replacement, sync_metadata, sync_metadata)?, + None => { + let file_name = file_path + .file_name() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "conditional file must have a name"))?; + let file = open_windows_relative( + parent_guard.last_handle()?, + file_name, + DELETE | SYNCHRONIZE | FILE_READ_ATTRIBUTES, + FILE_SHARE_READ, + FILE_OPEN, + FILE_NON_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT, + 0, + true, + )?; + validate_windows_owned_file(&file)?; + set_windows_file_delete_on_close(&file, true)?; + drop(file); + if sync_metadata { + fsync_dir_std(parent)?; + } + } + } + + Ok(ConditionalFileUpdate::Updated) +} + #[cfg(windows)] fn open_windows_directory_component( parent: &WindowsDirectoryHandle, diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index 4d4fb355b..7ea3b1854 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -91,6 +91,7 @@ metrics = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } crc-fast = { workspace = true } +sha2 = { workspace = true } [dev-dependencies] serde_json = { workspace = true, features = ["raw_value"] } diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 920fffcf5..a0ace01b4 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -373,6 +373,11 @@ impl ErasureSetHealer { set_disk_id: &str, buckets: &[String], ) -> Result<(ResumeManager, CheckpointManager)> { + if self.replacement_task_id.is_none() && CheckpointManager::is_blocked(&self.disk, task_id).await { + return Err(Error::TaskExecutionFailed { + message: format!("Resume task {task_id} has a blocked checkpoint"), + }); + } // check if resume state exists let has_resume_state = if self.replacement_task_id.is_some() { ResumeManager::has_replacement_intent(&self.disk, task_id).await diff --git a/crates/heal/src/heal/resume.rs b/crates/heal/src/heal/resume.rs index 291132c12..239713d3a 100644 --- a/crates/heal/src/heal/resume.rs +++ b/crates/heal/src/heal/resume.rs @@ -51,6 +51,7 @@ const RESUME_STATE_FILE: &str = "ahm_resume_state.json"; const REPLACEMENT_INTENT_FILE: &str = "ahm_replacement_intent.json"; const RESUME_PROGRESS_FILE: &str = "ahm_progress.json"; pub(super) const RESUME_CHECKPOINT_FILE: &str = "ahm_checkpoint.json"; +pub(super) const RESUME_CHECKPOINT_BLOCKED_FILE: &str = "ahm_checkpoint.blocked"; const REPLACEMENT_COMPLETION_PROOF_FILE: &str = "ahm_replacement_completion_proof.json"; const REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement"; const REPLACEMENT_INTENT_SEAL_FILE: &str = "ahm_replacement_intent_seal"; diff --git a/crates/heal/src/heal/resume/checkpoint.rs b/crates/heal/src/heal/resume/checkpoint.rs index 1b4b7ece3..18e159386 100644 --- a/crates/heal/src/heal/resume/checkpoint.rs +++ b/crates/heal/src/heal/resume/checkpoint.rs @@ -13,26 +13,31 @@ // limitations under the License. use crate::{Error, Result}; +use base64::Engine as _; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::HashSet; use std::path::Path; use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; -use tokio::sync::RwLock; +use tokio::sync::{Mutex as AsyncMutex, RwLock}; use tracing::{debug, warn}; -use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET}; +use super::super::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes}; +use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt, RUSTFS_META_BUCKET}; use super::{ - LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_FILE, delete_resume_file, path_to_str, - validate_resume_task_id, + LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_BLOCKED_FILE, RESUME_CHECKPOINT_FILE, + delete_resume_file, path_to_str, validate_resume_task_id, }; const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state"; +const RESUME_CHECKPOINT_DIGEST_FILE: &str = "ahm_checkpoint.sha256"; +const CHECKPOINT_PER_VERSION_SCHEMA: u32 = 5; /// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as /// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable /// to the new `compose_key` identities, so a stale checkpoint is discarded. -pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 5; +pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 6; /// resume checkpoint #[derive(Debug, Clone, Serialize, Deserialize)] @@ -57,6 +62,11 @@ pub struct ResumeCheckpoint { pub failed_objects: HashSet, /// skipped objects pub skipped_objects: HashSet, + /// Integrity digest over the checkpoint with this field set to `None`. + /// Keeping it in the checkpoint makes the payload and its authentication + /// record one CAS generation instead of two independently-written files. + #[serde(default)] + pub integrity_digest: Option, } impl ResumeCheckpoint { @@ -70,6 +80,7 @@ impl ResumeCheckpoint { processed_objects: HashSet::new(), failed_objects: HashSet::new(), skipped_objects: HashSet::new(), + integrity_digest: None, } } @@ -116,17 +127,111 @@ pub struct CheckpointManager { disk: DiskStore, checkpoint: Arc>, throttle: Mutex, + save_lock: AsyncMutex<()>, + last_saved: Mutex>, } impl CheckpointManager { + fn blocked_path(task_id: &str) -> std::path::PathBuf { + Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}")) + } + + /// Return whether a checkpoint was permanently isolated after a malformed + /// or unsupported snapshot was observed. + pub(crate) async fn is_blocked(disk: &DiskStore, task_id: &str) -> bool { + if validate_resume_task_id(task_id).is_err() { + return false; + } + let blocked_path = Self::blocked_path(task_id); + let Ok(path) = path_to_str(&blocked_path) else { + return false; + }; + match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await { + Ok(_) => true, + Err(crate::heal::DiskError::FileNotFound) => false, + Err(_) => true, + } + } + + /// Validate the checkpoint while enumerating resumable state. This reads + /// the checkpoint once and also isolates malformed or unsupported data. + pub(crate) async fn is_resumable(disk: &DiskStore, task_id: &str) -> Result { + validate_resume_task_id(task_id)?; + if Self::is_blocked(disk, task_id).await { + return Err(Error::InvalidCheckpoint(format!("Resume task {task_id} has a blocked checkpoint"))); + } + let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); + let Ok(path) = path_to_str(&file_path) else { + return Err(Error::InvalidCheckpoint("Resume checkpoint path is not valid UTF-8".to_string())); + }; + match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path).await { + Ok(bytes) if bytes.is_empty() => Ok(true), + Ok(bytes) => Self::load_from_data(disk.clone(), task_id, bytes.to_vec()) + .await + .map(|_| true), + Err(crate::heal::DiskError::FileNotFound) => Ok(true), + Err(error) => Err(error.into()), + } + } + + async fn block_invalid_snapshot(disk: &DiskStore, task_id: &str) { + // This marker is intentionally version-agnostic: an unsupported reader + // must stop selector retries until an operator cleans up the snapshot. + let blocked_path = Self::blocked_path(task_id); + let Ok(path) = path_to_str(&blocked_path) else { + return; + }; + let result = EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + path, + None, + Some(EcstoreDiskBytes::from_static(b"blocked")), + ) + .await; + match result { + Ok(EcstoreConditionalFileUpdate::Updated | EcstoreConditionalFileUpdate::Mismatch) => {} + Ok(EcstoreConditionalFileUpdate::Missing) => warn!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_CHECKPOINT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_id, + state = "blocked_marker_write_failed", + error = "marker target disappeared", + "Heal checkpoint could not persist its blocked marker" + ), + Err(error) => warn!( + target: "rustfs::heal::resume", + event = EVENT_HEAL_CHECKPOINT_STATE, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_RESUME, + task_id, + state = "blocked_marker_write_failed", + error = %error, + "Heal checkpoint could not persist its blocked marker" + ), + } + } + /// create new checkpoint manager pub async fn new(disk: DiskStore, task_id: String) -> Result { validate_resume_task_id(&task_id)?; + let checkpoint_volume = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}"); + if let Err(error) = EcstoreDiskAPI::make_volume(disk.as_ref(), &checkpoint_volume).await + && error != crate::heal::DiskError::VolumeExists + { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to create checkpoint volume: {error}"), + }); + } let checkpoint = ResumeCheckpoint::new(task_id); let manager = Self { disk, checkpoint: Arc::new(RwLock::new(checkpoint)), throttle: Mutex::new(PersistThrottle::new()), + save_lock: AsyncMutex::new(()), + last_saved: Mutex::new(None), }; // save initial checkpoint @@ -140,6 +245,7 @@ impl CheckpointManager { error = %e, "Heal checkpoint persistence failed" ); + return Err(e); } Ok(manager) } @@ -148,11 +254,22 @@ impl CheckpointManager { pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result { validate_resume_task_id(task_id)?; let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?; - let mut checkpoint: ResumeCheckpoint = - serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed { - message: format!("Failed to deserialize checkpoint: {e}"), - })?; + Self::load_from_data(disk, task_id, checkpoint_data).await + } + + async fn load_from_data(disk: DiskStore, task_id: &str, checkpoint_data: Vec) -> Result { + validate_resume_task_id(task_id)?; + let mut checkpoint: ResumeCheckpoint = match serde_json::from_slice(&checkpoint_data) { + Ok(checkpoint) => checkpoint, + Err(error) => { + Self::block_invalid_snapshot(&disk, task_id).await; + return Err(Error::TaskExecutionFailed { + message: format!("Failed to deserialize checkpoint: {error}"), + }); + } + }; if checkpoint.task_id != task_id { + Self::block_invalid_snapshot(&disk, task_id).await; return Err(Error::TaskExecutionFailed { message: "Resume checkpoint task id does not match filename".to_string(), }); @@ -163,6 +280,7 @@ impl CheckpointManager { // identities. Discard the stale sets and position, then stamp the // current schema so the scan restarts cleanly. if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA { + Self::block_invalid_snapshot(&disk, task_id).await; return Err(Error::TaskExecutionFailed { message: format!( "Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}", @@ -170,7 +288,45 @@ impl CheckpointManager { ), }); } - if checkpoint.schema_version < CURRENT_CHECKPOINT_SCHEMA { + + let integrity_verified = if let Some(expected) = checkpoint.integrity_digest.as_deref() { + let actual = Self::checkpoint_digest(&Self::serialize_without_digest(&checkpoint)?); + if expected != actual { + Self::block_invalid_snapshot(&disk, task_id).await; + return Err(Error::InvalidCheckpoint(format!( + "Resume checkpoint digest does not match task {task_id}" + ))); + } + true + } else if checkpoint.schema_version >= CURRENT_CHECKPOINT_SCHEMA { + Self::block_invalid_snapshot(&disk, task_id).await; + return Err(Error::InvalidCheckpoint(format!( + "Resume checkpoint digest is missing for task {task_id}" + ))); + } else { + let digest_path = Self::digest_path(task_id); + let digest_path = path_to_str(&digest_path)?; + match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, digest_path).await { + Ok(expected) => { + let actual = Self::checkpoint_digest(&checkpoint_data); + if expected.as_ref() != actual.as_bytes() { + Self::block_invalid_snapshot(&disk, task_id).await; + return Err(Error::InvalidCheckpoint(format!( + "Resume checkpoint digest does not match task {task_id}" + ))); + } + true + } + Err(crate::heal::DiskError::FileNotFound) => false, + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to read checkpoint digest: {error}"), + }); + } + } + }; + + if checkpoint.schema_version < CHECKPOINT_PER_VERSION_SCHEMA || !integrity_verified { warn!( target: "rustfs::heal::resume", event = EVENT_HEAL_CHECKPOINT_STATE, @@ -187,13 +343,15 @@ impl CheckpointManager { checkpoint.skipped_objects.clear(); checkpoint.current_bucket_index = 0; checkpoint.current_object_index = 0; - checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA; } + checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA; Ok(Self { disk, checkpoint: Arc::new(RwLock::new(checkpoint)), throttle: Mutex::new(PersistThrottle::new()), + save_lock: AsyncMutex::new(()), + last_saved: Mutex::new(Some(EcstoreDiskBytes::from(checkpoint_data))), }) } @@ -204,7 +362,7 @@ impl CheckpointManager { } let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); match path_to_str(&file_path) { - Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await { + Ok(path_str) => match HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str).await { Ok(data) => !data.is_empty(), Err(_) => false, }, @@ -292,6 +450,8 @@ impl CheckpointManager { let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); delete_resume_file(&self.disk, &checkpoint_file).await?; + delete_resume_file(&self.disk, &Self::digest_path(&task_id)).await?; + delete_resume_file(&self.disk, &Self::blocked_path(&task_id)).await?; debug!( target: "rustfs::heal::resume", @@ -307,21 +467,130 @@ impl CheckpointManager { /// save checkpoint to disk async fn save_checkpoint(&self) -> Result<()> { - let checkpoint = self.checkpoint.read().await; + // Serialize saves and take the snapshot only after acquiring the lock: + // a slower writer must not publish a snapshot taken before a newer one. + let _save_guard = self.save_lock.lock().await; + let checkpoint = self.checkpoint.read().await.clone(); validate_resume_task_id(&checkpoint.task_id)?; - let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed { - message: format!("Failed to serialize checkpoint: {e}"), - })?; + let unsigned_checkpoint_data = Self::serialize_without_digest(&checkpoint)?; + let digest = Self::checkpoint_digest(&unsigned_checkpoint_data); + let mut persisted_checkpoint = checkpoint.clone(); + persisted_checkpoint.integrity_digest = Some(digest); + let checkpoint_data = + EcstoreDiskBytes::from(serde_json::to_vec(&persisted_checkpoint).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to serialize checkpoint: {e}"), + })?); let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE)); let path_str = path_to_str(&file_path)?; - self.disk - .write_all(RUSTFS_META_BUCKET, path_str, checkpoint_data.into()) + let last_saved = self + .last_saved + .lock() + .map_err(|_| Error::TaskExecutionFailed { + message: "Checkpoint save state lock is poisoned; refusing to save".to_string(), + })? + .clone(); + let update = EcstoreDiskAPI::compare_and_update_file( + self.disk.as_ref(), + RUSTFS_META_BUCKET, + path_str, + last_saved.clone(), + Some(checkpoint_data.clone()), + ) + .await + .map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to save checkpoint: {e}"), + })?; + + let expected = match update { + EcstoreConditionalFileUpdate::Updated => None, + EcstoreConditionalFileUpdate::Missing => { + return Err(Error::TaskExecutionFailed { + message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(), + }); + } + EcstoreConditionalFileUpdate::Mismatch => { + // A healthy manager normally completes the CAS above without + // another read or JSON parse. Inspect only after a mismatch so + // corruption and future schemas cannot be overwritten blindly. + let existing = match HealDiskExt::read_all(self.disk.as_ref(), RUSTFS_META_BUCKET, path_str).await { + Ok(existing) => existing, + Err(crate::heal::DiskError::FileNotFound) => { + return Err(Error::TaskExecutionFailed { + message: "Checkpoint was removed after this manager saved it; refusing to recreate it".to_string(), + }); + } + Err(error) => { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to inspect checkpoint after CAS mismatch: {error}"), + }); + } + }; + + if existing.is_empty() && last_saved.is_none() { + Some(existing) + } else { + let current: ResumeCheckpoint = match serde_json::from_slice(&existing) { + Ok(current) => current, + Err(error) => { + Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await; + return Err(Error::TaskExecutionFailed { + message: format!("Existing checkpoint is corrupt: {error}"), + }); + } + }; + if current.task_id != checkpoint.task_id { + Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await; + return Err(Error::TaskExecutionFailed { + message: "Existing checkpoint task id does not match filename".to_string(), + }); + } + if current.schema_version > CURRENT_CHECKPOINT_SCHEMA { + Self::block_invalid_snapshot(&self.disk, &checkpoint.task_id).await; + return Err(Error::TaskExecutionFailed { + message: format!( + "Existing checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}", + current.schema_version + ), + }); + } + if last_saved.as_ref().is_none_or(|saved| saved.as_ref() != existing.as_ref()) { + return Err(Error::TaskExecutionFailed { + message: "Checkpoint changed since this manager loaded it; refusing to overwrite newer progress" + .to_string(), + }); + } + Some(existing) + } + } + }; + + if let Some(expected) = expected { + match EcstoreDiskAPI::compare_and_update_file( + self.disk.as_ref(), + RUSTFS_META_BUCKET, + path_str, + Some(expected), + Some(checkpoint_data.clone()), + ) .await .map_err(|e| Error::TaskExecutionFailed { - message: format!("Failed to save checkpoint: {e}"), - })?; + message: format!("Failed to save checkpoint after CAS mismatch: {e}"), + })? { + EcstoreConditionalFileUpdate::Updated => {} + EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch => { + return Err(Error::TaskExecutionFailed { + message: "Checkpoint changed while saving; refusing to overwrite newer progress".to_string(), + }); + } + } + } + + let mut last_saved = self.last_saved.lock().map_err(|_| Error::TaskExecutionFailed { + message: "Checkpoint save state lock is poisoned after save".to_string(), + })?; + *last_saved = Some(checkpoint_data); debug!( target: "rustfs::heal::resume", @@ -341,11 +610,38 @@ impl CheckpointManager { let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")); let path_str = path_to_str(&file_path)?; - disk.read_all(RUSTFS_META_BUCKET, path_str) + HealDiskExt::read_all(disk.as_ref(), RUSTFS_META_BUCKET, path_str) .await .map(|bytes| bytes.to_vec()) .map_err(|e| Error::TaskExecutionFailed { message: format!("Failed to read checkpoint file: {e}"), }) } + + fn serialize_without_digest(checkpoint: &ResumeCheckpoint) -> Result> { + let mut unsigned = checkpoint.clone(); + unsigned.integrity_digest = None; + let mut value = serde_json::to_value(&unsigned).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to serialize checkpoint: {e}"), + })?; + for field in ["processed_objects", "failed_objects", "skipped_objects"] { + let Some(values) = value.get_mut(field).and_then(serde_json::Value::as_array_mut) else { + return Err(Error::TaskExecutionFailed { + message: format!("Failed to canonicalize checkpoint field: {field}"), + }); + }; + values.sort_by(|left, right| left.as_str().cmp(&right.as_str())); + } + serde_json::to_vec(&value).map_err(|e| Error::TaskExecutionFailed { + message: format!("Failed to serialize checkpoint: {e}"), + }) + } + + fn checkpoint_digest(checkpoint_data: &[u8]) -> String { + base64::engine::general_purpose::STANDARD.encode(Sha256::digest(checkpoint_data)) + } + + fn digest_path(task_id: &str) -> std::path::PathBuf { + Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_DIGEST_FILE}")) + } } diff --git a/crates/heal/src/heal/resume/tests.rs b/crates/heal/src/heal/resume/tests.rs index 8d8e35cfa..fd76c9924 100644 --- a/crates/heal/src/heal/resume/tests.rs +++ b/crates/heal/src/heal/resume/tests.rs @@ -1600,6 +1600,32 @@ async fn test_checkpoint_schema_v4_discarded_on_load() { temp_dir.close().expect("remove schema test directory"); } +#[tokio::test] +async fn downgraded_unsigned_checkpoint_resets_untrusted_progress() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap(); + manager.add_processed_object("victim-a".to_string()).await.unwrap(); + manager.update_position(2, 500).await.unwrap(); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + let bytes = disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path).await.unwrap(); + let mut downgraded: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + downgraded["schema_version"] = serde_json::json!(CURRENT_CHECKPOINT_SCHEMA - 1); + downgraded.as_object_mut().unwrap().remove("integrity_digest"); + downgraded["processed_objects"] = serde_json::json!(["victim-b"]); + disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, serde_json::to_vec(&downgraded).unwrap().into()) + .await + .expect("write downgraded checkpoint"); + + let manager = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap(); + let checkpoint = manager.get_checkpoint().await; + assert_eq!(checkpoint.schema_version, CURRENT_CHECKPOINT_SCHEMA); + assert_eq!(checkpoint.current_bucket_index, 0); + assert_eq!(checkpoint.current_object_index, 0); + assert!(checkpoint.processed_objects.is_empty()); + temp_dir.close().unwrap(); +} + #[tokio::test] async fn current_normal_resume_schema_preserves_progress() { let (temp_dir, disk) = schema_test_disk().await; @@ -1675,6 +1701,369 @@ async fn future_resume_and_checkpoint_schemas_are_rejected() { temp_dir.close().expect("remove schema test directory"); } +#[tokio::test] +async fn checkpoint_save_does_not_replace_a_non_empty_truncated_snapshot() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("create checkpoint manager"); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + let truncated = b"{\"schema_version\":5,\"task_id\":"; + disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, truncated.as_slice().into()) + .await + .expect("write truncated checkpoint fixture"); + + let error = manager + .update_position(2, 7) + .await + .expect_err("a truncated checkpoint must fail closed during save"); + assert!(error.to_string().contains("Existing checkpoint is corrupt")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path) + .await + .expect("read truncated checkpoint fixture"), + truncated.as_slice() + ); + assert!(CheckpointManager::is_blocked(&disk, &task_id).await); + temp_dir.close().expect("remove checkpoint save test directory"); +} + +#[tokio::test] +async fn checkpoint_save_does_not_replace_a_future_schema_snapshot() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("create checkpoint manager"); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + let mut future = ResumeCheckpoint::new(task_id.clone()); + future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1; + let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture"); + disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, future_bytes.clone().into()) + .await + .expect("write future checkpoint fixture"); + + let error = manager + .update_position(2, 7) + .await + .expect_err("a future schema must fail closed during save"); + assert!(error.to_string().contains("Existing checkpoint schema")); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path) + .await + .expect("read future checkpoint fixture"), + future_bytes + ); + assert!(CheckpointManager::is_blocked(&disk, &task_id).await); + temp_dir.close().expect("remove future schema test directory"); +} + +#[tokio::test] +async fn checkpoint_digest_rejects_same_length_progress_tampering() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("create checkpoint manager"); + manager + .add_processed_object("victim-a".to_string()) + .await + .expect("persist checkpoint progress"); + manager.update_position(1, 1).await.expect("flush checkpoint progress"); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + let original = disk + .read_all(RUSTFS_META_BUCKET, &checkpoint_path) + .await + .expect("read checkpoint fixture"); + let tampered = original + .windows(b"victim-a".len()) + .position(|window| window == b"victim-a") + .map(|index| { + let mut bytes = original.to_vec(); + bytes[index..index + b"victim-a".len()].copy_from_slice(b"victim-b"); + bytes + }) + .expect("checkpoint should contain the processed object"); + disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, tampered.into()) + .await + .expect("write tampered checkpoint fixture"); + + assert!(CheckpointManager::load_from_disk(disk.clone(), &task_id).await.is_err()); + assert!(CheckpointManager::is_blocked(&disk, &task_id).await); + temp_dir.close().expect("remove digest test directory"); +} + +#[tokio::test] +async fn checkpoint_integrity_survives_missing_legacy_sidecar() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap(); + manager.update_position(2, 9).await.unwrap(); + + let digest_path = format!("{BUCKET_META_PREFIX}/{task_id}_ahm_checkpoint.sha256"); + delete_resume_file(&disk, Path::new(&digest_path)).await.unwrap(); + + let restored = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap(); + let checkpoint = restored.get_checkpoint().await; + assert_eq!(checkpoint.current_bucket_index, 2); + assert_eq!(checkpoint.current_object_index, 9); + assert!(checkpoint.integrity_digest.is_some()); + temp_dir.close().unwrap(); +} + +#[tokio::test] +async fn checkpoint_integrity_survives_multi_object_reload() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap(); + for index in 0..32 { + manager.add_processed_object(format!("processed-{index}")).await.unwrap(); + manager.add_failed_object(format!("failed-{index}")).await.unwrap(); + manager.add_skipped_object(format!("skipped-{index}")).await.unwrap(); + } + manager.update_position(2, 9).await.unwrap(); + + CheckpointManager::load_from_disk(disk, &task_id) + .await + .expect("a healthy multi-object checkpoint must survive reload"); + temp_dir.close().unwrap(); +} + +#[tokio::test] +async fn checkpoint_integrity_rejects_a_removed_embedded_digest() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap(); + manager.update_position(2, 9).await.unwrap(); + + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + let bytes = disk + .read_all(RUSTFS_META_BUCKET, &checkpoint_path) + .await + .expect("read checkpoint fixture"); + let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + value["current_object_index"] = serde_json::json!(10); + value.as_object_mut().unwrap().remove("integrity_digest"); + disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, serde_json::to_vec(&value).unwrap().into()) + .await + .expect("write tampered checkpoint fixture"); + + assert!( + CheckpointManager::load_from_disk(disk.clone(), &task_id).await.is_err(), + "a current checkpoint without its embedded digest must fail closed" + ); + assert!(CheckpointManager::is_blocked(&disk, &task_id).await); + temp_dir.close().unwrap(); +} + +#[tokio::test] +async fn new_checkpoint_manager_rebuilds_an_empty_snapshot() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + disk.write_all(RUSTFS_META_BUCKET, &checkpoint_path, EcstoreDiskBytes::new()) + .await + .expect("write empty checkpoint fixture"); + + let manager = CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("a new manager must rebuild an empty checkpoint"); + manager + .update_position(3, 11) + .await + .expect("rebuilt checkpoint must remain writable"); + assert!(CheckpointManager::has_checkpoint(&disk, &task_id).await); + temp_dir.close().expect("remove empty checkpoint test directory"); +} + +#[tokio::test] +async fn deleted_checkpoint_is_not_recreated_by_an_old_manager() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("create checkpoint manager"); + manager.cleanup().await.expect("delete checkpoint fixture"); + + let error = manager + .update_position(1, 2) + .await + .expect_err("an old manager must not resurrect a deleted checkpoint"); + assert!(error.to_string().contains("removed after this manager saved it")); + assert!(!CheckpointManager::has_checkpoint(&disk, &task_id).await); + temp_dir.close().expect("remove deleted checkpoint test directory"); +} + +#[cfg(unix)] +#[tokio::test] +async fn checkpoint_cleanup_leaves_no_task_specific_lock_artifact() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("create checkpoint manager"); + let lock_path = Path::new(BUCKET_META_PREFIX) + .join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}")) + .with_extension("rustfs-cas.lock"); + let lock_path = temp_dir.path().join(RUSTFS_META_BUCKET).join(lock_path); + + manager.cleanup().await.expect("delete checkpoint fixture"); + + assert!( + !lock_path.exists(), + "successful checkpoint cleanup must not leave a task-specific lock artifact" + ); +} + +#[tokio::test] +async fn an_empty_blocked_marker_still_blocks_resume_selection() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let manager = CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("create checkpoint manager"); + let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}"); + disk.write_all(RUSTFS_META_BUCKET, &blocked_path, EcstoreDiskBytes::new()) + .await + .expect("write empty blocked marker fixture"); + + assert!(CheckpointManager::is_blocked(&disk, &task_id).await); + assert!(CheckpointManager::is_resumable(&disk, &task_id).await.is_err()); + // Recovery requires replacing/cleaning the snapshot, then removing the + // marker; ordinary selector retries are intentionally not an unlock path. + manager.cleanup().await.expect("clean blocked checkpoint"); + assert!(!CheckpointManager::is_blocked(&disk, &task_id).await); + temp_dir.close().expect("remove empty blocked marker test directory"); +} + +#[tokio::test] +async fn resumable_selector_skips_healthy_tasks_with_blocked_markers() { + let (temp_dir, disk) = schema_test_disk().await; + let tasks = [ + (ResumeUtils::generate_task_id(), EcstoreDiskBytes::new()), + (ResumeUtils::generate_task_id(), EcstoreDiskBytes::from_static(b"blocked")), + ]; + for (task_id, marker) in &tasks { + ResumeManager::new( + disk.clone(), + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ) + .await + .expect("create healthy resume state"); + CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("create healthy checkpoint"); + let checkpoint_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + let checkpoint_bytes = disk + .read_all(RUSTFS_META_BUCKET, &checkpoint_path) + .await + .expect("read healthy checkpoint before blocking"); + let marker_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}"); + disk.write_all(RUSTFS_META_BUCKET, &marker_path, marker.clone()) + .await + .expect("write blocked marker"); + + assert!(ResumeUtils::get_resumable_tasks(&disk).await.is_err()); + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, &checkpoint_path) + .await + .expect("read healthy checkpoint after blocking"), + checkpoint_bytes + ); + } + temp_dir.close().expect("remove blocked selector test directory"); +} + +#[tokio::test] +async fn stale_checkpoint_manager_cannot_overwrite_newer_progress() { + let (temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let first = CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("create first checkpoint manager"); + let second = CheckpointManager::load_from_disk(disk.clone(), &task_id) + .await + .expect("load second checkpoint manager"); + + second + .update_position(4, 20) + .await + .expect("persist newer checkpoint progress"); + let error = first + .update_position(1, 3) + .await + .expect_err("stale checkpoint manager must not overwrite newer progress"); + assert!(error.to_string().contains("newer progress")); + + let persisted = CheckpointManager::load_from_disk(disk.clone(), &task_id) + .await + .expect("load newer checkpoint progress") + .get_checkpoint() + .await; + assert_eq!(persisted.current_bucket_index, 4); + assert_eq!(persisted.current_object_index, 20); + temp_dir.close().expect("remove stale manager test directory"); +} + +#[tokio::test] +async fn resumable_selector_isolates_future_and_corrupt_checkpoints() { + let (temp_dir, disk) = schema_test_disk().await; + let future_task = ResumeUtils::generate_task_id(); + let corrupt_task = ResumeUtils::generate_task_id(); + for task_id in [&future_task, &corrupt_task] { + ResumeManager::new( + disk.clone(), + task_id.to_string(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ) + .await + .expect("create resumable state fixture"); + } + + let future_path = format!("{BUCKET_META_PREFIX}/{future_task}_{RESUME_CHECKPOINT_FILE}"); + let mut future = ResumeCheckpoint::new(future_task.clone()); + future.schema_version = CURRENT_CHECKPOINT_SCHEMA + 1; + let future_bytes = serde_json::to_vec(&future).expect("serialize future checkpoint fixture"); + disk.write_all(RUSTFS_META_BUCKET, &future_path, future_bytes.clone().into()) + .await + .expect("write future checkpoint fixture"); + let corrupt_path = format!("{BUCKET_META_PREFIX}/{corrupt_task}_{RESUME_CHECKPOINT_FILE}"); + let corrupt_bytes = b"{truncated"; + disk.write_all(RUSTFS_META_BUCKET, &corrupt_path, corrupt_bytes.as_slice().into()) + .await + .expect("write corrupt checkpoint fixture"); + + assert!(CheckpointManager::is_resumable(&disk, &future_task).await.is_err()); + assert!(CheckpointManager::is_resumable(&disk, &corrupt_task).await.is_err()); + assert!(ResumeUtils::get_resumable_tasks(&disk).await.is_err()); + for (task_id, path, bytes) in [ + (&future_task, future_path, future_bytes), + (&corrupt_task, corrupt_path, corrupt_bytes.to_vec()), + ] { + assert_eq!( + disk.read_all(RUSTFS_META_BUCKET, &path) + .await + .expect("read isolated checkpoint bytes"), + bytes + ); + let blocked_path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}"); + assert!( + !disk + .read_all(RUSTFS_META_BUCKET, &blocked_path) + .await + .expect("read checkpoint blocked marker") + .is_empty() + ); + } + temp_dir.close().expect("remove selector isolation test directory"); +} + #[test] fn test_persist_throttle_batches_until_threshold() { let mut throttle = PersistThrottle::new(); diff --git a/crates/heal/src/heal/resume/utils.rs b/crates/heal/src/heal/resume/utils.rs index 71507e7c0..557269d6e 100644 --- a/crates/heal/src/heal/resume/utils.rs +++ b/crates/heal/src/heal/resume/utils.rs @@ -21,7 +21,7 @@ use uuid::Uuid; use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET}; use super::replacement::{ReplacementPhase, ReplacementRecoveryRecord}; use super::{ - EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE, + CheckpointManager, EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE, REPLACEMENT_INTENT_FILE, RESUME_STATE_FILE, ResumeManager, ResumeStateFile, is_replacement_intent, path_to_str, replacement_recovery_corruption_for_state_load, replacement_recovery_dir, validate_resume_task_id, }; @@ -67,6 +67,7 @@ impl ResumeUtils { // Extract task ID from filename: {task_id}_ahm_resume_state.json if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}")) && validate_resume_task_id(task_id).is_ok() + && CheckpointManager::is_resumable(disk, task_id).await? { task_ids.push(task_id.to_string()); } From 4ddc728c9da7726aef48781e743cac8e687ba590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 23 Aug 2026 16:42:30 +0800 Subject: [PATCH 08/41] fix(replication): deny non-owner replication config edits under site replication (#6375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(replication): deny non-owner replication config edits under site replication Under site replication a user holding only bucket-scoped s3:PutReplicationConfiguration could rewrite or erase the operator-managed site-repl-* rules, with the change broadcast to every peer (backlog#1948, audit A1/P2-17). - Gate PutBucketReplication/DeleteBucketReplication in the S3 handlers: when site replication is enabled and the requester is not the owner, return MinIO-parity XMinioReplicationDenyEdit (HTTP 400). The gate runs after policy authorization and only on the external S3 path; the reconciler and peer bucket-meta ingestion are unaffected. - Defense in depth in the bucket usecase: PUT merges the incoming config with the stored site-repl-* rules (same merge as peer ingestion) instead of overwriting verbatim; DELETE keeps the site-repl-* rules and never garbage-collects a bucket target a surviving site-replication rule still references. - Move is_site_replication_rule / merge_incoming_replication_config / replication_target_arn_deployment_id from the admin site-replication handler down to rustfs-replication so the app layer can reuse them without new layering violations. * fix(replication): scope site-owned rule detection to reconciler-derived rules The `site-repl-*` prefix alone classified any rule as site-owned, so on a bucket outside site replication an owner's `site-repl-user` rule survived DeleteBucketReplication (rule and target kept, success returned). Rule ids do not reserve that namespace. A rule is reconciler-owned only when it matches what the reconciler derives: id `site-repl-` for a current remote site replication peer and a destination ARN naming that same deployment id. The S3 put/delete path reads the remote peer set (empty when site replication is disabled) and keeps exactly those rules; everything else is operator state the request replaces or deletes. An incoming rule that claims a current peer's id is dropped so the reconciler rule's id stays unique. The peer ingestion path and the reconciler keep their prefix predicate unchanged. * fix(replication): keep operator rule priorities across site rule merges Merging stored site-replication rules into a PutBucketReplication body renumbered every rule 1..n in list order, rewriting the submitted policy: overlapping same-target rules submitted as priority 5 then 1 became 1 then 2, so the delete-marker-disabled rule won the replication decision. The reconciler and the peer-removal prune renumbered the same way. Operator priorities now stay verbatim everywhere; only the reconciler's derived rules move, to the lowest priorities no operator rule uses, via one pure helper shared by the S3 edit merge, the peer ingestion merge, the reconciler pass and the prune. Being a pure function of the rule list it is idempotent, so the reconciler's no-op check still holds after a merged write, and an on-disk config in the historical layout (operator rules 1..k, site rules k+1..n) yields the same bytes, so nothing is rewritten on upgrade. * fix(replication): pass site peer ids into the bucket usecase from the interface layer The review fix made the bucket usecase read the site-replication peer set through the admin handlers, an app->interface import the layer guard rejects. The S3 handlers (interface) now read the peer set and pass it in, so the usecase stays a pure function of its inputs; a state-read failure still fails the edit closed, just one layer up. * fix(replication): classify peer-ingested rules by the derived id/ARN contract The peer ingestion merge still treated every incoming `site-repl-*` id as reconciler-owned, so an owner-authored `site-repl-user` rule that the S3 merge now keeps on the editing site was dropped on every peer and the sites persisted different operator configs. The ingestion merge now classifies by the same derived contract as the S3 merge: a rule is the reconciler's only when its `site-repl-` names the deployment its destination ARN targets and that deployment is a site of the cluster (the receiver's own id included, since the sender's rule towards the receiver names it). The reconciler, the peer-removal prune and the target-online probe switch from the id prefix to the derived shape as well, so the rule survives their passes too; rules in the derived shape that name a removed peer or this site are still rebuilt away. Regression: a PutBucketReplication merged on site A and ingested on site B keeps `site-repl-user` on both and the operator rule sets agree. * fix(replication): keep an operator role target through site rule merges The S3 and peer-ingestion merges cleared `Role` whenever it parsed as a site-replication ARN, which an owner-submitted remote target with an empty region (`arn:minio:replication:::`) also does. The merged config then selected the rule destination ARNs instead of the validated role target. Only a role naming a current site of the cluster is the holder's identity (the reconciler's per-peer target lookup reads it); every other role passed target validation and stays. The reconciler's repair pass applies the same rule. Regression: an owner role target survives both merges and `filter_target_arns` / `replication_target_arns` select it; a role naming a current peer is still cleared. * fix(replication): gate operator priority preservation on a peer contract probe Keeping operator rule priorities verbatim is not rolling-upgrade safe: a peer still running the pre-contract code renumbers every rule 1..n in list order on ingest and on each reconciler pass, so an upgraded site broadcasting `5,1` leaves that peer on `1,2` — which can select the other overlapping rule — and the sites never reconverge. Operator rules now merge under an explicit contract: - `OperatorRuleContract::Derived`: site rules are the derived id/ARN shape, operator priorities stay verbatim (the behavior of the previous commits). - `OperatorRuleContract::Legacy`: byte-for-byte what a pre-contract peer does — `site-repl-*` ids are all site rules, a site-replication-shaped `Role` is dropped, every rule is renumbered 1..n in list order. The S3 merge additionally lists the operator rules in priority order first, so the renumbering keeps their relative order and the winning rule per target is the one the operator submitted. The S3 PutBucketReplication/DeleteBucketReplication path probes every remote peer through the existing `peer/edit-capabilities` endpoint (capability `derived-rule-contract`; pre-contract peers answer `success:false` or 404) and merges under Derived only when every peer supports it; any refusal or probe failure pins that edit to Legacy. Every bucket-meta item this site sends (S3 hooks, bootstrap plan, retry snapshots, tombstones) carries `derivedRuleContract: true`; a receiver merges a payload without the marker the Legacy way, so an item from a pre-contract sender is handled exactly as its own peers handle it. Rolling upgrade: while any site runs the older code every edit is canonicalized cluster-wide (numbers lost, order kept); once the last site is upgraded the next edit keeps its priorities. Configs canonicalized during the mixed period are not renumbered back — the derived priority assignment is a no-op on the canonical layout — so an operator who wants the original values re-submits the config after the upgrade completes. Adding a site that runs the older code after priorities were preserved is not gated and would desynchronize that bucket until the next edit. --------- Co-authored-by: houseme --- crates/ecstore/src/api/mod.rs | 24 +- crates/ecstore/src/bucket/replication/mod.rs | 13 +- .../replication_config_boundary.rs | 13 +- crates/madmin/src/site_replication.rs | 6 + crates/replication/src/config.rs | 484 ++++++++++++++++++ crates/replication/src/lib.rs | 14 +- rustfs/src/admin/handlers/site_replication.rs | 403 +++++++++++---- rustfs/src/admin/storage_api.rs | 6 +- rustfs/src/app/bucket_usecase.rs | 310 ++++++++++- rustfs/src/app/storage_api.rs | 2 + rustfs/src/storage/ecfs.rs | 174 ++++++- 11 files changed, 1291 insertions(+), 158 deletions(-) diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 456d38ac7..e55e9b97b 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -187,22 +187,24 @@ pub mod bucket { pub use crate::bucket::replication::{ BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric, - MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, - REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, - REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, - ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig, + MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, OperatorRuleContract, + REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, + REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, + REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus, - VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent, - delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool, - get_global_replication_stats, get_proxy_targets, init_background_replication, - invalid_replication_config_status_field, persist_force_delete_intent, read_durable_mrf_backlog, - replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns, - resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication, - should_use_existing_delete_replication_info, should_use_existing_delete_replication_source, + VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent, + complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id, + get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication, + invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule, + merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent, + read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, + replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id, + should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info, + should_use_existing_delete_replication_source, site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns, version_purge_status_to_filemeta, }; diff --git a/crates/ecstore/src/bucket/replication/mod.rs b/crates/ecstore/src/bucket/replication/mod.rs index be0de0121..091b3a964 100644 --- a/crates/ecstore/src/bucket/replication/mod.rs +++ b/crates/ecstore/src/bucket/replication/mod.rs @@ -44,11 +44,14 @@ mod replication_versioning_boundary; mod runtime_boundary; pub use replication_config_boundary::{ - ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, - REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, - ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError, - invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target, - unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns, + ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, + REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, + REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError, + assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_role, + is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config, + replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target, + site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure, + validate_replication_config_target_arns, }; pub(crate) use replication_filemeta_boundary::version_purge_statuses_map; pub use replication_filemeta_boundary::{ diff --git a/crates/ecstore/src/bucket/replication/replication_config_boundary.rs b/crates/ecstore/src/bucket/replication/replication_config_boundary.rs index 484fe7a25..fbd9e88b1 100644 --- a/crates/ecstore/src/bucket/replication/replication_config_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_config_boundary.rs @@ -13,9 +13,12 @@ // limitations under the License. pub use rustfs_replication::{ - ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, - REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, - ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError, - invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target, - unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns, + ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, + REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, + REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt, + ReplicationTargetValidationError, assign_site_replication_rule_priorities, invalid_replication_config_status_field, + is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config, + replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target, + site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure, + validate_replication_config_target_arns, }; diff --git a/crates/madmin/src/site_replication.rs b/crates/madmin/src/site_replication.rs index 1dcee9350..ee8269078 100644 --- a/crates/madmin/src/site_replication.rs +++ b/crates/madmin/src/site_replication.rs @@ -431,6 +431,12 @@ pub struct SRBucketMeta { pub cors: Option, #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] pub api_version: Option, + /// Set by a sender that merges replication configs under the derived + /// site-rule contract (operator rule priorities verbatim, `site-repl-*` + /// ids classified by id/ARN). A receiver merges a payload without it the + /// pre-contract way; a pre-contract receiver ignores the field. + #[serde(rename = "derivedRuleContract", default, skip_serializing_if = "std::ops::Not::not")] + pub derived_rule_contract: bool, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] diff --git a/crates/replication/src/config.rs b/crates/replication/src/config.rs index 405799f94..58e87ac1a 100644 --- a/crates/replication/src/config.rs +++ b/crates/replication/src/config.rs @@ -270,6 +270,218 @@ pub fn active_replication_rule_destination_arns(config: &ReplicationConfiguratio arns } +/// Deployment id extracted from a site-replication target ARN +/// (`arn:{rustfs|minio}:replication:::`), or `None` +/// for an operator-authored ARN. +pub fn replication_target_arn_deployment_id(arn: &str) -> Option { + let parts: Vec<_> = arn.split(':').collect(); + if parts.len() == 6 + && parts[0] == "arn" + && matches!(parts[1], "rustfs" | "minio") + && parts[2] == "replication" + && !parts[4].is_empty() + { + return Some(parts[4].to_string()); + } + + None +} + +/// Rule id prefix the site-replication reconciler stamps on the rules it +/// derives (`site-repl-`). +pub const SITE_REPLICATION_RULE_ID_PREFIX: &str = "site-repl-"; + +/// Whether `rule` carries a site-replication rule id (`site-repl-*`). Rule +/// ids are not reserved, so this is only the classification of +/// [`OperatorRuleContract::Legacy`]; every other path classifies by +/// [`site_replication_rule_deployment_id`]. +pub fn is_site_replication_rule(rule: &ReplicationRule) -> bool { + rule.id + .as_deref() + .is_some_and(|id| id.starts_with(SITE_REPLICATION_RULE_ID_PREFIX)) +} + +/// Deployment id of the peer a reconciler-derived rule replicates to, or +/// `None` for any other rule. The reconciler builds each rule from one peer: +/// the id is `site-repl-` and the destination ARN names that +/// same deployment id — an operator-authored `site-repl-user` rule, or a +/// `site-repl-` id pasted onto a foreign ARN, fails the agreement check. +/// Callers that know the current peer set must also confirm the id is one of +/// those peers before treating the rule as reconciler-owned. +pub fn site_replication_rule_deployment_id(rule: &ReplicationRule) -> Option<&str> { + let deployment_id = rule.id.as_deref()?.strip_prefix(SITE_REPLICATION_RULE_ID_PREFIX)?; + (!deployment_id.is_empty() + && replication_target_arn_deployment_id(&rule.destination.bucket).as_deref() == Some(deployment_id)) + .then_some(deployment_id) +} + +/// Whether `rule` is one the local reconciler derived for a current remote +/// site-replication peer in `peer_deployment_ids`. With an empty peer set +/// (site replication disabled) nothing qualifies, so a bucket outside site +/// replication keeps the verbatim S3 put/delete semantics. +pub fn is_reconciler_owned_site_replication_rule(rule: &ReplicationRule, peer_deployment_ids: &HashSet) -> bool { + site_replication_rule_deployment_id(rule).is_some_and(|deployment_id| peer_deployment_ids.contains(deployment_id)) +} + +/// Whether a config's `Role` is a site-replication ARN naming a site in +/// `deployment_ids`. Such a role is the holder's identity, not policy: the +/// reconciler's per-peer target lookup reads it, so carrying it across sites +/// would pin the receiver's targets to the sender's. Any other role — an IAM +/// role, or an operator remote target whose ARN happens to carry an empty +/// region — passed target validation and drives target selection. +pub fn is_site_replication_role(role: &str, deployment_ids: &HashSet) -> bool { + replication_target_arn_deployment_id(role).is_some_and(|deployment_id| deployment_ids.contains(&deployment_id)) +} + +/// How the sites of a cluster treat the operator rules of a replication +/// config merge. Every site must apply the same contract to the same +/// payload or the sites persist different configs, so the S3 edit path +/// probes the peers before merging and a peer payload carries the contract +/// its sender applied. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OperatorRuleContract { + /// Site rules are the derived id/ARN shape; operator rule priorities are + /// kept verbatim. + Derived, + /// Some site still runs the pre-contract code: every `site-repl-*` id is + /// a site rule, a site-replication-shaped `Role` is dropped, and every + /// rule is renumbered 1..n in list order on ingest and on each reconciler + /// pass. Merging the same way keeps a mixed cluster on one config; the + /// operator's priority values are lost for that edit but their order — + /// what decides the winning rule per target — is not, because the S3 + /// merge lists the operator rules in priority order first. + Legacy, +} + +/// Merge a peer's replication config into the local one. +/// +/// Reconciler-derived rules encode the *holder's* outbound direction — their +/// destination ARN names another site — so applying an external rule set +/// verbatim replaces the local reverse rule with one this site can never +/// satisfy (no bucket target backs it) and replication silently stops. Only +/// operator-authored rules travel: the sender's derived rules are dropped +/// and the local site's survive. `site_deployment_ids` is every site of the +/// cluster, the receiver included — the sender's rule towards the receiver +/// names the receiver's own id. Rules are classified by the derived id/ARN +/// contract ([`is_reconciler_owned_site_replication_rule`]), the same one +/// the S3 edit merge applies, so an operator-authored `site-repl-*` id +/// persists on every site. `incoming == None` models a delete of the +/// operator-authored rules. +pub fn merge_incoming_replication_config( + incoming: Option, + local: Option, + site_deployment_ids: &HashSet, + contract: OperatorRuleContract, +) -> Option { + merge_replication_config_keeping_site_rules(incoming, local, site_deployment_ids, contract) +} + +/// [`merge_incoming_replication_config`] for the S3 put/delete-bucket-replication +/// path (issue #1948): only rules the local reconciler derived for a current +/// peer in `peer_deployment_ids` survive as site rules; every other stored +/// rule — including an operator-authored `site-repl-*` id — is operator state +/// that the request replaces or deletes. An incoming rule whose id is a +/// current peer's `site-repl-` is dropped whatever its ARN: accepting it +/// would duplicate the reconciler rule's id. Under +/// [`OperatorRuleContract::Legacy`] the merge instead reproduces what the +/// pre-contract peers will do with the broadcast, listing the operator rules +/// in priority order so their relative order survives the renumbering. +pub fn merge_user_replication_config( + incoming: Option, + local: Option, + peer_deployment_ids: &HashSet, + contract: OperatorRuleContract, +) -> Option { + let incoming = incoming.map(|mut config| { + match contract { + OperatorRuleContract::Derived => config.rules.retain(|rule| { + !rule + .id + .as_deref() + .and_then(|id| id.strip_prefix(SITE_REPLICATION_RULE_ID_PREFIX)) + .is_some_and(|deployment_id| peer_deployment_ids.contains(deployment_id)) + }), + // Pre-contract peers renumber in list order, so listing the + // operator rules in priority order keeps their relative order — + // and the replication decision — through that renumbering. + OperatorRuleContract::Legacy => config.rules.sort_by_key(|rule| rule.priority.unwrap_or(0)), + } + config + }); + merge_replication_config_keeping_site_rules(incoming, local, peer_deployment_ids, contract) +} + +fn merge_replication_config_keeping_site_rules( + incoming: Option, + local: Option, + deployment_ids: &HashSet, + contract: OperatorRuleContract, +) -> Option { + let is_site_rule = |rule: &ReplicationRule| match contract { + OperatorRuleContract::Derived => is_reconciler_owned_site_replication_rule(rule, deployment_ids), + OperatorRuleContract::Legacy => is_site_replication_rule(rule), + }; + let incoming_role = incoming.as_ref().map(|config| config.role.clone()).unwrap_or_default(); + // Operator rules first, then the local site rules — the same order the + // site-replication reconciler produces, so its no-op check matches and + // the bucket metadata is written once per broadcast, not twice. + let mut rules: Vec = incoming + .into_iter() + .flat_map(|config| config.rules) + .filter(|rule| !is_site_rule(rule)) + .collect(); + rules.extend( + local + .into_iter() + .flat_map(|config| config.rules) + .filter(|rule| is_site_rule(rule)), + ); + + if rules.is_empty() { + return None; + } + + let drop_role = match contract { + OperatorRuleContract::Derived => { + assign_site_replication_rule_priorities(&mut rules, is_site_rule); + is_site_replication_role(&incoming_role, deployment_ids) + } + OperatorRuleContract::Legacy => { + for (index, rule) in rules.iter_mut().enumerate() { + rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX)); + } + replication_target_arn_deployment_id(&incoming_role).is_some() + } + }; + let role = if drop_role { String::new() } else { incoming_role }; + + Some(ReplicationConfiguration { role, rules }) +} + +/// Give the site rules in `rules` the lowest priorities no operator rule uses, +/// in rule order, leaving every operator rule's priority untouched. Operator +/// priorities decide which rule wins per target, so they are part of the +/// submitted policy; site rules are derived state and only need to be unique +/// (`validate_replication_config_structure` rejects duplicates). The result +/// is a pure function of the rule list, so the site-replication reconciler, +/// the peer ingestion merge and the S3 edit merge all converge on the same +/// bytes and the reconciler's no-op check holds. +pub fn assign_site_replication_rule_priorities(rules: &mut [ReplicationRule], is_site_rule: impl Fn(&ReplicationRule) -> bool) { + let taken: HashSet = rules + .iter() + .filter(|rule| !is_site_rule(rule)) + .map(|rule| rule.priority.unwrap_or(0)) + .collect(); + let mut next = 1; + for rule in rules.iter_mut().filter(|rule| is_site_rule(rule)) { + while taken.contains(&next) { + next += 1; + } + rule.priority = Some(next); + next = next.saturating_add(1); + } +} + pub fn replication_target_arns(config: &ReplicationConfiguration) -> HashSet { let role = config.role.trim(); if !role.is_empty() { @@ -1544,4 +1756,276 @@ mod tests { "the child rule must win for target A while the overlapping child target B remains eligible" ); } + + #[test] + fn site_replication_rule_deployment_id_requires_id_and_arn_agreement() { + let reconciler_rule = replication_rule("site-repl-peer-dep", "arn:rustfs:replication::peer-dep:bucket"); + assert_eq!(site_replication_rule_deployment_id(&reconciler_rule), Some("peer-dep")); + + // A remote-target ARN carries the remote's deployment id (or a random + // uuid), never the operator's rule id. + let operator_named_rule = replication_rule("site-repl-user", "arn:minio:replication:us-east-1:2f1c-remote:bucket"); + assert_eq!(site_replication_rule_deployment_id(&operator_named_rule), None); + + let foreign_arn = replication_rule("site-repl-peer-dep", "arn:rustfs:replication::other-dep:bucket"); + assert_eq!(site_replication_rule_deployment_id(&foreign_arn), None); + + let empty_id = replication_rule("site-repl-", "arn:rustfs:replication::peer-dep:bucket"); + assert_eq!(site_replication_rule_deployment_id(&empty_id), None); + + let peers = HashSet::from(["peer-dep".to_string()]); + assert!(is_reconciler_owned_site_replication_rule(&reconciler_rule, &peers)); + assert!(!is_reconciler_owned_site_replication_rule(&reconciler_rule, &HashSet::new())); + let removed_peer = replication_rule("site-repl-gone-dep", "arn:rustfs:replication::gone-dep:bucket"); + assert!(!is_reconciler_owned_site_replication_rule(&removed_peer, &peers)); + } + + // The merge must not rewrite the operator's priorities: with the + // priority-5 rule listed first and renumbered 1 then 2, the priority-1 + // delete-marker-disabled rule would win the replication decision. + #[test] + fn merge_keeps_operator_priorities_and_replication_decision() { + let user_arn = "arn:minio:replication:us-east-1:2f1c-remote:bucket"; + let peer_arn = "arn:rustfs:replication::peer-dep:bucket"; + let incoming = ReplicationConfiguration { + role: String::new(), + rules: vec![ + delete_marker_rule("dm-enabled", user_arn, "logs/", 5, true), + delete_marker_rule("dm-disabled", user_arn, "logs/2026/", 1, false), + ], + }; + let mut site_rule = delete_marker_rule("site-repl-peer-dep", peer_arn, "", 7, true); + site_rule.prefix = None; + let local = structure_config(vec![site_rule]); + let opts = ObjectOpts { + name: "logs/2026/app.log".to_string(), + op_type: ReplicationType::Delete, + delete_marker: true, + version_id: None, + ..Default::default() + }; + let submitted: Vec<_> = incoming.filter_target_replication_decisions(&opts); + + let peers = HashSet::from(["peer-dep".to_string()]); + let merged = + merge_user_replication_config(Some(incoming.clone()), Some(local.clone()), &peers, OperatorRuleContract::Derived) + .expect("rules"); + + let priorities: Vec<_> = merged + .rules + .iter() + .map(|rule| (rule.id.as_deref().unwrap(), rule.priority)) + .collect(); + assert_eq!( + priorities, + vec![ + ("dm-enabled", Some(5)), + ("dm-disabled", Some(1)), + ("site-repl-peer-dep", Some(2)) + ], + "operator priorities are kept verbatim; the site rule takes the lowest free slot" + ); + assert!(validate_replication_config_structure(&merged).is_ok()); + let mut decisions = merged.filter_target_replication_decisions(&opts); + decisions.retain(|(arn, _)| arn == user_arn); + assert_eq!(decisions, submitted, "the merged config must replicate exactly as the operator submitted"); + assert_eq!(decisions, vec![(user_arn.to_string(), true)]); + + // The peer ingestion merge follows the same rule. + let merged = + merge_incoming_replication_config(Some(incoming), Some(local), &peers, OperatorRuleContract::Derived).expect("rules"); + let priorities: Vec<_> = merged.rules.iter().map(|rule| rule.priority).collect(); + assert_eq!(priorities, vec![Some(5), Some(1), Some(2)]); + } + + #[test] + fn site_rule_priorities_skip_every_operator_priority() { + let mut rules = vec![ + delete_marker_rule("a", "arn:a", "", 2, true), + delete_marker_rule("site-repl-x", "arn:rustfs:replication::x:b", "", 9, true), + delete_marker_rule("b", "arn:a", "", 1, true), + delete_marker_rule("site-repl-y", "arn:rustfs:replication::y:b", "", 9, true), + delete_marker_rule("c", "arn:a", "", 4, true), + ]; + assign_site_replication_rule_priorities(&mut rules, is_site_replication_rule); + let priorities: Vec<_> = rules.iter().map(|rule| rule.priority).collect(); + assert_eq!(priorities, vec![Some(2), Some(3), Some(1), Some(5), Some(4)]); + assert!(validate_replication_config_structure(&structure_config(rules.clone())).is_ok()); + + // Idempotent, so the reconciler's pass over an already-merged config + // is a byte-stable no-op rather than a rewrite every period. + let settled = rules.clone(); + assign_site_replication_rule_priorities(&mut rules, is_site_replication_rule); + assert_eq!(rules, settled); + } + + fn operator_rule_ids(config: &ReplicationConfiguration) -> Vec<(&str, Option)> { + config + .rules + .iter() + .filter(|rule| site_replication_rule_deployment_id(rule).is_none()) + .map(|rule| (rule.id.as_deref().unwrap(), rule.priority)) + .collect() + } + + // Issue #1948 review: an owner-authored `site-repl-user` rule is operator + // state. Site A's S3 merge keeps it; the broadcast payload must survive + // site B's peer ingestion too, or the sites persist different configs. + #[test] + fn peer_ingestion_keeps_owner_site_repl_user_rule_and_sites_agree() { + let user_arn = "arn:minio:replication:us-east-1:2f1c-remote:bucket"; + let put = structure_config(vec![ + delete_marker_rule("site-repl-user", user_arn, "logs/", 3, true), + delete_marker_rule("nightly", user_arn, "", 1, true), + ]); + let a_local = structure_config(vec![replication_rule("site-repl-b-dep", "arn:rustfs:replication::b-dep:bucket")]); + let a_peers = HashSet::from(["b-dep".to_string()]); + let a_merged = + merge_user_replication_config(Some(put), Some(a_local), &a_peers, OperatorRuleContract::Derived).expect("rules"); + assert_eq!(operator_rule_ids(&a_merged), vec![("site-repl-user", Some(3)), ("nightly", Some(1))]); + + // Site B ingests A's broadcast; its own reverse rule names A. + let b_local = structure_config(vec![replication_rule("site-repl-a-dep", "arn:rustfs:replication::a-dep:bucket")]); + let b_sites = HashSet::from(["a-dep".to_string(), "b-dep".to_string()]); + let b_merged = + merge_incoming_replication_config(Some(a_merged.clone()), Some(b_local), &b_sites, OperatorRuleContract::Derived) + .expect("rules"); + + let ids: Vec<_> = b_merged.rules.iter().map(|rule| rule.id.as_deref().unwrap()).collect(); + assert_eq!(ids, vec!["site-repl-user", "nightly", "site-repl-a-dep"]); + assert_eq!( + operator_rule_ids(&b_merged), + operator_rule_ids(&a_merged), + "both sites must persist the same operator rules" + ); + } + + // Issue #1948 review: `Role` is only the sender's when it names a + // current site-replication peer; an owner-submitted role target has + // already passed target validation and drives target selection. + #[test] + fn merge_keeps_operator_role_target_for_target_selection() { + let role = "arn:minio:replication::operator-dep:bucket"; + let peers = HashSet::from(["peer-dep".to_string()]); + let incoming = ReplicationConfiguration { + role: role.to_string(), + rules: vec![delete_marker_rule("nightly", role, "", 1, true)], + }; + let local = structure_config(vec![replication_rule( + "site-repl-peer-dep", + "arn:rustfs:replication::peer-dep:bucket", + )]); + let opts = ObjectOpts { + name: "logs/app.log".to_string(), + ..Default::default() + }; + + let merged = + merge_user_replication_config(Some(incoming.clone()), Some(local.clone()), &peers, OperatorRuleContract::Derived) + .expect("rules"); + assert_eq!(merged.role, role); + assert_eq!(replication_target_arns(&merged), HashSet::from([role.to_string()])); + assert_eq!(merged.filter_target_arns(&opts), vec![role.to_string()]); + + let ingested = + merge_incoming_replication_config(Some(incoming.clone()), Some(local.clone()), &peers, OperatorRuleContract::Derived) + .expect("rules"); + assert_eq!(ingested.role, role); + assert_eq!(ingested.filter_target_arns(&opts), vec![role.to_string()]); + + // A role naming a current peer is the sender's identity and still goes. + let mut derived_role = incoming; + derived_role.role = "arn:rustfs:replication::peer-dep:bucket".to_string(); + let merged = + merge_user_replication_config(Some(derived_role), Some(local), &peers, OperatorRuleContract::Derived).expect("rules"); + assert!(merged.role.is_empty()); + } + + // Issue #1948 review: while a site still runs the pre-contract code the + // cluster must stay on one config. A new site broadcasting `5,1` would + // be renumbered `1,2` by that peer — selecting the other overlapping + // rule — so the new sites merge the legacy way and list the operator + // rules in priority order first, which keeps the decision. + #[test] + fn legacy_contract_matches_pre_contract_peers_and_keeps_the_decision() { + let user_arn = "arn:minio:replication:us-east-1:2f1c-remote:bucket"; + let put = ReplicationConfiguration { + role: "arn:minio:replication::operator-dep:bucket".to_string(), + rules: vec![ + delete_marker_rule("dm-enabled", user_arn, "logs/", 5, true), + delete_marker_rule("dm-disabled", user_arn, "logs/2026/", 1, false), + delete_marker_rule("site-repl-user", user_arn, "tmp/", 2, true), + ], + }; + let a_local = structure_config(vec![replication_rule("site-repl-b-dep", "arn:rustfs:replication::b-dep:bucket")]); + let opts = ObjectOpts { + name: "logs/2026/app.log".to_string(), + op_type: ReplicationType::Delete, + delete_marker: true, + ..Default::default() + }; + // Decisions per rule destination: the role is dropped by the legacy + // merge, so compare against the rules alone. + let submitted: Vec<_> = structure_config(put.rules.clone()).filter_target_replication_decisions(&opts); + + let a_peers = HashSet::from(["b-dep".to_string()]); + let a_merged = + merge_user_replication_config(Some(put.clone()), Some(a_local.clone()), &a_peers, OperatorRuleContract::Legacy) + .expect("rules"); + let layout: Vec<_> = a_merged + .rules + .iter() + .map(|rule| (rule.id.as_deref().unwrap(), rule.priority)) + .collect(); + assert_eq!( + layout, + vec![ + ("dm-disabled", Some(1)), + ("dm-enabled", Some(2)), + ("site-repl-b-dep", Some(3)) + ], + "legacy: operator rules in priority order, every rule renumbered 1..n, `site-repl-*` ids dropped" + ); + assert!(a_merged.role.is_empty(), "legacy peers drop any site-replication-shaped role"); + let mut decisions = a_merged.filter_target_replication_decisions(&opts); + decisions.retain(|(arn, _)| arn == user_arn); + assert_eq!(decisions, submitted, "the renumbering must not flip the winning rule"); + + // A pre-contract peer renumbers A's payload in list order: same bytes. + let mut pre_contract = a_merged + .rules + .iter() + .filter(|rule| !is_site_replication_rule(rule)) + .cloned() + .collect::>(); + pre_contract.push(replication_rule("site-repl-a-dep", "arn:rustfs:replication::a-dep:bucket")); + for (index, rule) in pre_contract.iter_mut().enumerate() { + rule.priority = Some(index as i32 + 1); + } + // A new peer told the payload is legacy produces the same bytes too. + let b_local = structure_config(vec![replication_rule("site-repl-a-dep", "arn:rustfs:replication::a-dep:bucket")]); + let b_sites = HashSet::from(["a-dep".to_string(), "b-dep".to_string()]); + let b_merged = merge_incoming_replication_config(Some(a_merged), Some(b_local), &b_sites, OperatorRuleContract::Legacy) + .expect("rules"); + assert_eq!(b_merged.rules, pre_contract); + + // Every site on the derived contract: the submitted policy is kept. + let a_merged = + merge_user_replication_config(Some(put), Some(a_local), &a_peers, OperatorRuleContract::Derived).expect("rules"); + let layout: Vec<_> = a_merged + .rules + .iter() + .map(|rule| (rule.id.as_deref().unwrap(), rule.priority)) + .collect(); + assert_eq!( + layout, + vec![ + ("dm-enabled", Some(5)), + ("dm-disabled", Some(1)), + ("site-repl-user", Some(2)), + ("site-repl-b-dep", Some(3)) + ] + ); + assert_eq!(a_merged.role, "arn:minio:replication::operator-dep:bucket"); + } } diff --git a/crates/replication/src/lib.rs b/crates/replication/src/lib.rs index cb0761b5c..def314f95 100644 --- a/crates/replication/src/lib.rs +++ b/crates/replication/src/lib.rs @@ -29,12 +29,14 @@ mod storage_api; pub mod tagging; pub use config::{ - ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, - REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, - ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError, - active_replication_rule_destination_arns, invalid_replication_config_status_field, replication_target_arns, - should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_structure, - validate_replication_config_target_arns, + ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, + REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, + REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError, + active_replication_rule_destination_arns, assign_site_replication_rule_priorities, invalid_replication_config_status_field, + is_reconciler_owned_site_replication_rule, is_site_replication_role, is_site_replication_rule, + merge_incoming_replication_config, merge_user_replication_config, replication_target_arn_deployment_id, + replication_target_arns, should_remove_replication_target, site_replication_rule_deployment_id, + unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns, }; pub use delete::{ DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 23c8b680b..d28033085 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -31,6 +31,10 @@ use crate::admin::storage_api::bucket::metadata::{ use crate::admin::storage_api::bucket::metadata_sys; use crate::admin::storage_api::bucket::quota::BucketQuota; use crate::admin::storage_api::bucket::replication; +use crate::admin::storage_api::bucket::replication::{ + OperatorRuleContract, assign_site_replication_rule_priorities, is_site_replication_role, merge_incoming_replication_config, + replication_target_arn_deployment_id, site_replication_rule_deployment_id, +}; use crate::admin::storage_api::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; use crate::admin::storage_api::bucket::target_sys::BucketTargetSys; use crate::admin::storage_api::bucket::utils::{deserialize, serialize}; @@ -160,6 +164,8 @@ const SITE_REPLICATION_PEER_EDIT_CAPABILITY_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit-capabilities?capability=endpoint-target-refresh"; const SITE_REPLICATION_PEER_TLS_CAPABILITY_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit-capabilities?capability=peer-tls-settings"; +const SITE_REPLICATION_PEER_DERIVED_RULE_CONTRACT_CAPABILITY_PATH: &str = + "/rustfs/admin/v3/site-replication/peer/edit-capabilities?capability=derived-rule-contract"; const SITE_REPLICATION_PEER_EDIT_REFRESH_PATH: &str = "/rustfs/admin/v3/site-replication/peer/edit?refresh-targets=true"; /// Peer-edit fencing token, carried as query parameters so a peer that predates /// the fence simply ignores them (unknown query keys are dropped) and keeps the @@ -1118,6 +1124,114 @@ async fn load_site_replication_state() -> S3Result { } } +/// Whether this deployment participates in site replication (two or more +/// peers in the persisted state). Read by the S3 interface layer to gate +/// replication-config edits (MinIO `ErrReplicationDenyEditError` semantics, +/// issue #1948); a state-read failure propagates so the gate fails closed. +pub(crate) async fn site_replication_enabled() -> S3Result { + Ok(load_site_replication_state().await?.enabled()) +} + +/// Deployment ids of the remote peers the reconciler derives a +/// `site-repl-` rule for on every bucket (the same peer filter as +/// `build_site_replication_config`); empty when site replication is not +/// enabled. Read by the bucket usecase so an S3 replication-config edit keeps +/// exactly the reconciler-owned rules (issue #1948); a state-read failure +/// propagates so the edit fails closed. +pub(crate) async fn site_replication_edit_context() -> S3Result<(HashSet, OperatorRuleContract)> { + let Some(runtime) = runtime_site_replication_targets().await? else { + // Enabled without a service account is a state this site cannot + // broadcast from either; the peers are still the reconciler's. + let state = load_site_replication_state().await?; + if !state.enabled() { + return Ok((HashSet::new(), OperatorRuleContract::Derived)); + } + let peers = remote_peer_deployment_ids(&state, ¤t_local_runtime_peer(&state)); + return Ok((peers, OperatorRuleContract::Legacy)); + }; + let peers = remote_peer_deployment_ids(&runtime.state, &runtime.local_peer); + let contract = site_replication_operator_rule_contract(&runtime).await; + Ok((peers, contract)) +} + +/// Whether every remote peer merges replication configs under the derived +/// contract, probed through the peer capability endpoint. A peer that does +/// not (or cannot be asked) pins the cluster to [`OperatorRuleContract::Legacy`] +/// for this edit: consistency across sites wins over keeping the operator's +/// priority values, and the legacy merge keeps their order anyway. +async fn site_replication_operator_rule_contract(runtime: &SiteReplicationRuntime) -> OperatorRuleContract { + let remote_peers: Vec<&PeerInfo> = runtime + .state + .peers + .values() + .filter(|peer| { + peer.deployment_id != runtime.local_peer.deployment_id + && !same_identity_endpoint(&peer.endpoint, &runtime.local_peer.endpoint) + }) + .collect(); + let probes = futures::future::join_all(remote_peers.iter().map(|peer| async move { + let transport = PeerTransport::for_runtime_peer(peer).await?; + let (status, body) = send_peer_admin_request_raw_with_client( + &transport.client, + &transport.connection, + SITE_REPLICATION_PEER_DERIVED_RULE_CONTRACT_CAPABILITY_PATH, + &runtime.state.service_account_access_key, + &runtime.service_account_secret_key, + &(), + ) + .await?; + peer_capability_response_supported(peer, status, &body) + })) + .await; + operator_rule_contract_from_probes(remote_peers.into_iter().zip(probes)) +} + +fn operator_rule_contract_from_probes<'a>( + probes: impl IntoIterator)>, +) -> OperatorRuleContract { + for (peer, probe) in probes { + match probe { + Ok(true) => {} + Ok(false) => return OperatorRuleContract::Legacy, + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "derived_rule_contract_probe_failed", + peer = %peer.endpoint, + error = %err, + "admin site replication state" + ); + return OperatorRuleContract::Legacy; + } + } + } + OperatorRuleContract::Derived +} + +fn remote_peer_deployment_ids(state: &SiteReplicationState, local_peer: &PeerInfo) -> HashSet { + state + .peers + .values() + .filter(|peer| { + peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) + }) + .map(|peer| peer.deployment_id.clone()) + .collect() +} + +/// Deployment ids of every site in the cluster, this one included: the set +/// a peer's derived rules can name (its rule towards this site carries this +/// site's id). Empty when site replication is not enabled. +async fn site_replication_deployment_ids() -> S3Result> { + let state = load_site_replication_state().await?; + if !state.enabled() { + return Ok(HashSet::new()); + } + Ok(state.peers.values().map(|peer| peer.deployment_id.clone()).collect()) +} + async fn load_site_replication_state_no_lock(store: Arc) -> S3Result { match read_config_no_lock(store, SITE_REPLICATION_STATE_PATH).await { Ok(data) => parse_site_replication_state(&data), @@ -1984,7 +2098,7 @@ fn peer_tls_settings_changed(existing: Option<&PeerInfo>, proposed: &PeerInfo) - } fn peer_edit_capability_supported(capability: &str) -> bool { - matches!(capability, "endpoint-target-refresh" | "peer-tls-settings") + matches!(capability, "endpoint-target-refresh" | "peer-tls-settings" | "derived-rule-contract") } fn validate_add_sites(sites: &[PeerSite], local_peer: &PeerInfo) -> S3Result<()> { @@ -2251,6 +2365,7 @@ fn bootstrap_bucket_meta_item(bucket: &SRBucketInfo, item_type: &str, updated_at r#type: item_type.to_string(), updated_at, api_version: Some(SITE_REPL_API_VERSION.to_string()), + derived_rule_contract: true, ..Default::default() } } @@ -6632,6 +6747,7 @@ fn bucket_metadata_snapshot_tombstone(item: &SRBucketMeta, observed_at: OffsetDa updated_at: Some(observed_at), expiry_updated_at: Some(observed_at), api_version: item.api_version.clone(), + derived_rule_contract: item.derived_rule_contract, ..Default::default() } } @@ -7762,20 +7878,6 @@ fn bucket_target_deployment_id(target: &BucketTarget) -> Option { replication_target_arn_deployment_id(&target.arn) } -fn replication_target_arn_deployment_id(arn: &str) -> Option { - let parts: Vec<_> = arn.split(':').collect(); - if parts.len() == 6 - && parts[0] == "arn" - && matches!(parts[1], "rustfs" | "minio") - && parts[2] == "replication" - && !parts[4].is_empty() - { - return Some(parts[4].to_string()); - } - - None -} - fn prune_removed_site_replication_bucket_targets( existing: BucketTargets, removed_deployment_ids: &HashSet, @@ -7800,10 +7902,6 @@ fn prune_removed_site_replication_bucket_targets( (BucketTargets { targets }, removed) } -fn is_site_replication_rule(rule: &ReplicationRule) -> bool { - rule.id.as_deref().is_some_and(|id| id.starts_with("site-repl-")) -} - /// Whether every `site-repl-*` rule on this bucket resolves to a live remote target. /// /// The rule set alone cannot answer this: a rule can be perfectly formed while the endpoint @@ -7816,7 +7914,7 @@ async fn site_replication_targets_online(bucket: &str, replication_config_xml: & return true; }; - for rule in config.rules.iter().filter(|rule| is_site_replication_rule(rule)) { + for rule in config.rules.iter().filter(|rule| is_derived_site_replication_rule(rule)) { if BucketTargetSys::get() .get_remote_target_client_by_arn(bucket, &rule.destination.bucket) .await @@ -7829,52 +7927,6 @@ async fn site_replication_targets_online(bucket: &str, replication_config_xml: & true } -/// Merge a peer's replication config into the local one. -/// -/// `site-repl-*` rules encode the *sender's* outbound direction — their destination ARN -/// names the receiver — so applying a peer's rule set verbatim replaces the receiver's -/// reverse rule with one pointing at itself. No bucket target can satisfy that ARN -/// (`reconcile_site_replication_bucket_targets` skips the local peer), so the receiver -/// silently stops replicating back: the one-directional symptom. Only operator-authored -/// rules travel between sites; each site owns its own `site-repl-*` rules. -fn merge_incoming_replication_config( - incoming: Option, - local: Option, -) -> Option { - let incoming_role = incoming.as_ref().map(|config| config.role.clone()).unwrap_or_default(); - // Operator rules first, then the local site rules — the same order - // `ensure_site_replication_bucket_replication_config_with_runtime` produces, so its - // no-op check matches and the bucket metadata is written once per broadcast, not twice. - let mut rules: Vec = incoming - .into_iter() - .flat_map(|config| config.rules) - .filter(|rule| !is_site_replication_rule(rule)) - .collect(); - rules.extend( - local - .into_iter() - .flat_map(|config| config.rules) - .filter(is_site_replication_rule), - ); - - if rules.is_empty() { - return None; - } - - for (index, rule) in rules.iter_mut().enumerate() { - rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX)); - } - - // A site-replication ARN in `role` is the sender's, and `site_replication_target_arns_by_peer` - // reads it — carrying it over would pin the receiver's targets to the sender's identity. - let role = match replication_target_arn_deployment_id(&incoming_role) { - Some(_) => String::new(), - None => incoming_role, - }; - - Some(ReplicationConfiguration { role, rules }) -} - /// Merge a peer's ILM expiry document into the local lifecycle config. /// /// Mirrors MinIO's `mergeWithCurrentLCConfig` with one hardening: incoming @@ -8169,6 +8221,17 @@ fn lifecycle_expiry_statement( } } +/// Whether `rule` is in the shape the reconciler derives (`site-repl-` +/// naming the deployment its ARN targets). The reconciler rebuilds every such +/// rule from the current peer set — current peer or not, so a leftover from a +/// removed peer or a self-pointing rule is rebuilt away — while the merges +/// keep only the current peers' rules and treat a leftover as operator state +/// the edit replaces. An operator-authored `site-repl-*` id on an operator +/// ARN is outside the shape and survives every pass. +fn is_derived_site_replication_rule(rule: &ReplicationRule) -> bool { + site_replication_rule_deployment_id(rule).is_some() +} + fn replication_rule_deployment_id(rule: &ReplicationRule) -> Option { if let Some(rule_id) = rule.id.as_deref() { if let Some(deployment_id) = rule_id.strip_prefix("site-repl-") @@ -8213,9 +8276,7 @@ fn prune_removed_site_replication_rules( return (None, removed); } - for (index, rule) in config.rules.iter_mut().enumerate() { - rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX)); - } + assign_site_replication_rule_priorities(&mut config.rules, is_derived_site_replication_rule); (Some(config), removed) } @@ -8375,30 +8436,32 @@ async fn ensure_site_replication_bucket_replication_config_with_runtime( return Ok(()); }; - // `site-repl-*` rules are derived state owned by this site: rebuild them from the - // current peer set on every pass instead of preserving whatever is on disk. A rule - // left over from a removed peer — or one whose destination ARN names this very - // deployment, which no bucket target can ever satisfy — must not survive, otherwise - // objects are queued against an ARN that resolves to nothing. + // Derived rules are state owned by this site: rebuild them from the current peer + // set on every pass instead of preserving whatever is on disk. A rule left over + // from a removed peer — or one whose destination ARN names this very deployment, + // which no bucket target can ever satisfy — must not survive, otherwise objects + // are queued against an ARN that resolves to nothing. let (existing_role, existing_rules) = existing .map(|config| (config.role, config.rules)) .unwrap_or_else(|| (String::new(), Vec::new())); let mut rules: Vec = existing_rules .iter() - .filter(|rule| !is_site_replication_rule(rule)) + .filter(|rule| !is_derived_site_replication_rule(rule)) .cloned() .collect(); rules.extend(desired.rules); - for (index, rule) in rules.iter_mut().enumerate() { - rule.priority = Some(i32::try_from(index + 1).unwrap_or(i32::MAX)); - } + // Operator priorities are the operator's policy; only the derived rules + // take free slots, by the same function as the config merges so a merged + // write and this pass agree byte for byte. + assign_site_replication_rule_priorities(&mut rules, is_derived_site_replication_rule); - // Only a site-replication ARN in `role` is ours to drop — an operator-authored role is + // Only a `role` naming a current peer is ours to drop — an operator-authored role is // part of the bucket's S3-visible configuration, and repairing a reverse rule must not // quietly rewrite it. Same rule as `merge_incoming_replication_config`. - let role = match replication_target_arn_deployment_id(&existing_role) { - Some(_) => String::new(), - None => existing_role.clone(), + let role = if is_site_replication_role(&existing_role, &remote_peer_deployment_ids(state, local_peer)) { + String::new() + } else { + existing_role.clone() }; if rules == existing_rules && role == existing_role { @@ -9284,7 +9347,13 @@ async fn apply_bucket_meta_item(item: SRBucketMeta) -> S3Result<()> { Err(err) => return Err(ApiError::from(err).into()), }; let local_absent = local.is_none(); - match merge_incoming_replication_config(incoming, local) { + let site_deployment_ids = site_replication_deployment_ids().await?; + let contract = if item.derived_rule_contract { + OperatorRuleContract::Derived + } else { + OperatorRuleContract::Legacy + }; + match merge_incoming_replication_config(incoming, local, &site_deployment_ids, contract) { Some(config) => Some(serialize(&config).map_err(|e| { S3Error::with_message(S3ErrorCode::InternalError, format!("serialize replication failed: {e}")) })?), @@ -13395,6 +13464,7 @@ mod tests { assert!(peer_edit_capability_supported("peer-tls-settings")); assert!(peer_edit_capability_supported("endpoint-target-refresh")); + assert!(peer_edit_capability_supported("derived-rule-contract")); assert!(!peer_edit_capability_supported("unknown")); assert!(peer_capability_response_supported(&remote, StatusCode::OK, br#"{"success":true}"#).expect("supported")); assert!(!peer_capability_response_supported(&remote, StatusCode::NOT_FOUND, b"").expect("legacy peer")); @@ -16186,6 +16256,10 @@ mod tests { assert_eq!(deployment_id.as_deref(), Some("remote-dep")); } + fn home_office() -> HashSet { + HashSet::from(["home".to_string(), "office".to_string()]) + } + fn site_repl_config(peer: &str) -> ReplicationConfiguration { ReplicationConfiguration { role: String::new(), @@ -16284,8 +16358,13 @@ mod tests { // itself. No bucket target backs that ARN, so every object was dropped without a log. #[test] fn test_merge_incoming_replication_config_keeps_local_reverse_rule() { - let merged = merge_incoming_replication_config(Some(site_repl_config("home")), Some(site_repl_config("office"))) - .expect("merge should keep the local rule"); + let merged = merge_incoming_replication_config( + Some(site_repl_config("home")), + Some(site_repl_config("office")), + &home_office(), + OperatorRuleContract::Derived, + ) + .expect("merge should keep the local rule"); assert_eq!(merged.rules.len(), 1); assert_eq!(merged.rules[0].id.as_deref(), Some("site-repl-office")); @@ -16296,8 +16375,13 @@ mod tests { // either — the delete travels as `replication-config` with no payload. #[test] fn test_merge_incoming_replication_config_survives_peer_delete() { - let merged = merge_incoming_replication_config(None, Some(site_repl_config("office"))) - .expect("local site rules must survive a peer delete"); + let merged = merge_incoming_replication_config( + None, + Some(site_repl_config("office")), + &home_office(), + OperatorRuleContract::Derived, + ) + .expect("local site rules must survive a peer delete"); assert_eq!(merged.rules.len(), 1); assert_eq!(merged.rules[0].id.as_deref(), Some("site-repl-office")); @@ -16309,8 +16393,13 @@ mod tests { incoming.rules.push(operator_rule("nightly-backup")); incoming.role = "arn:rustfs:replication::home:photos".to_string(); - let merged = merge_incoming_replication_config(Some(incoming), Some(site_repl_config("office"))) - .expect("merge should produce rules"); + let merged = merge_incoming_replication_config( + Some(incoming), + Some(site_repl_config("office")), + &home_office(), + OperatorRuleContract::Derived, + ) + .expect("merge should produce rules"); let ids: Vec<_> = merged.rules.iter().filter_map(|rule| rule.id.as_deref()).collect(); assert_eq!(ids, vec!["nightly-backup", "site-repl-office"]); @@ -16324,7 +16413,15 @@ mod tests { #[test] fn test_merge_incoming_replication_config_returns_none_when_nothing_remains() { - assert!(merge_incoming_replication_config(Some(site_repl_config("home")), None).is_none()); + assert!( + merge_incoming_replication_config( + Some(site_repl_config("home")), + None, + &home_office(), + OperatorRuleContract::Derived + ) + .is_none() + ); } fn lc_rule(id: &str, expiry_days: Option, transition_days: Option) -> s3s::dto::LifecycleRule { @@ -16727,26 +16824,31 @@ mod tests { } // `role` is part of the bucket's S3-visible configuration. Repairing a reverse rule must - // drop only a sender-owned site-replication ARN, never an operator's own role — the same - // rule the merge path applies, so both paths agree on what is ours to rewrite. + // drop only a role naming a current peer, never an operator's own role — an IAM role or + // a remote target whose ARN carries an empty region — the same rule the merge path + // applies, so both paths agree on what is ours to rewrite. #[test] - fn test_replication_role_is_only_cleared_when_it_is_a_site_replication_arn() { - let operator_role = "arn:aws:iam::123456789012:role/replication"; - assert!( - replication_target_arn_deployment_id(operator_role).is_none(), - "an operator IAM role is not a site-replication ARN and must be preserved" - ); - assert_eq!( - replication_target_arn_deployment_id("arn:rustfs:replication::home:photos").as_deref(), - Some("home"), - "a site-replication ARN is sender-owned and gets cleared" - ); + fn test_replication_role_is_only_cleared_when_it_names_a_peer() { + let sites = home_office(); + assert!(!is_site_replication_role("arn:aws:iam::123456789012:role/replication", &sites)); + assert!(!is_site_replication_role("arn:minio:replication::operator-dep:photos", &sites)); + assert!(is_site_replication_role("arn:rustfs:replication::home:photos", &sites)); - let mut incoming = site_repl_config("home"); - incoming.role = operator_role.to_string(); - let merged = merge_incoming_replication_config(Some(incoming), Some(site_repl_config("office"))) + for operator_role in [ + "arn:aws:iam::123456789012:role/replication", + "arn:minio:replication::operator-dep:photos", + ] { + let mut incoming = site_repl_config("home"); + incoming.role = operator_role.to_string(); + let merged = merge_incoming_replication_config( + Some(incoming), + Some(site_repl_config("office")), + &sites, + OperatorRuleContract::Derived, + ) .expect("merge should produce rules"); - assert_eq!(merged.role, operator_role, "operator role must survive the merge"); + assert_eq!(merged.role, operator_role, "operator role must survive the merge"); + } } // Rules and targets are keyed off the same ARN. Minting a fresh one while @@ -17089,7 +17191,7 @@ mod tests { } #[test] - fn test_prune_removed_site_replication_rules_removes_site_rule_and_reorders_priorities() { + fn test_prune_removed_site_replication_rules_removes_site_rule_and_keeps_operator_priority() { let removed_deployment_ids = HashSet::from(["removed-dep".to_string()]); let kept_rule = build_site_replication_rule("arn:rustfs:replication::kept-dep:photos", 3, "site-repl-kept-dep"); let removed_rule = build_site_replication_rule("arn:rustfs:replication::removed-dep:photos", 1, "site-repl-removed-dep"); @@ -17106,9 +17208,90 @@ mod tests { assert!(updated.role.is_empty()); assert_eq!(updated.rules.len(), 2); assert_eq!(updated.rules[0].id.as_deref(), Some("user-managed-rule")); - assert_eq!(updated.rules[0].priority, Some(1)); + assert_eq!(updated.rules[0].priority, Some(9), "the operator's priority is policy and stays"); assert_eq!(updated.rules[1].id.as_deref(), Some("site-repl-kept-dep")); - assert_eq!(updated.rules[1].priority, Some(2)); + assert_eq!(updated.rules[1].priority, Some(1), "the derived rule moves to the lowest free slot"); + } + + // Issue #1948 review: one pre-contract peer pins an S3 edit to the legacy + // merge; only a cluster where every remote peer answered the probe moves + // to the derived contract. A probe error counts as a pre-contract peer. + #[test] + fn test_operator_rule_contract_requires_every_remote_peer() { + let home = normalize_peer_info(PeerInfo { + endpoint: "https://home.example.com".to_string(), + ..Default::default() + }); + let office = normalize_peer_info(PeerInfo { + endpoint: "https://office.example.com".to_string(), + ..Default::default() + }); + + assert_eq!(operator_rule_contract_from_probes([]), OperatorRuleContract::Derived); + assert_eq!( + operator_rule_contract_from_probes([(&home, Ok(true)), (&office, Ok(true))]), + OperatorRuleContract::Derived + ); + assert_eq!( + operator_rule_contract_from_probes([(&home, Ok(true)), (&office, Ok(false))]), + OperatorRuleContract::Legacy + ); + assert_eq!( + operator_rule_contract_from_probes([(&home, Err(s3_error!(InternalError, "unreachable"))), (&office, Ok(true))]), + OperatorRuleContract::Legacy + ); + } + + // The contract travels with the payload: a pre-contract sender's item has + // no marker and is merged the legacy way; every item this site sends is + // marked, bootstrap snapshots included, so a preserved config is never + // renumbered by a peer on the derived contract. + #[test] + fn test_bucket_meta_items_carry_the_derived_rule_contract() { + let legacy: SRBucketMeta = serde_json::from_str(r#"{"type":"replication-config","bucket":"photos"}"#).expect("item"); + assert!(!legacy.derived_rule_contract); + + let bucket = SRBucketInfo { + bucket: "photos".to_string(), + ..Default::default() + }; + let item = bootstrap_bucket_meta_item(&bucket, "replication-config", None); + assert!(item.derived_rule_contract); + let wire = serde_json::to_value(&item).expect("json"); + assert_eq!(wire["derivedRuleContract"], serde_json::Value::Bool(true)); + assert!(bucket_metadata_snapshot_tombstone(&item, OffsetDateTime::now_utc()).derived_rule_contract); + } + + // Issue #1948 review: an owner's `site-repl-user` rule on an operator ARN + // is outside the derived shape, so neither the prune nor the reconciler + // treats it as theirs; a leftover in the derived shape still is. + #[test] + fn test_derived_shape_excludes_owner_site_repl_user_rule() { + let owner_rule = build_site_replication_rule("arn:minio:replication:us-east-1:2f1c-remote:photos", 9, "site-repl-user"); + assert!(!is_derived_site_replication_rule(&owner_rule)); + assert!(is_derived_site_replication_rule(&build_site_replication_rule( + "arn:rustfs:replication::gone-dep:photos", + 1, + "site-repl-gone-dep" + ))); + + let config = ReplicationConfiguration { + role: String::new(), + rules: vec![ + build_site_replication_rule("arn:rustfs:replication::removed-dep:photos", 1, "site-repl-removed-dep"), + owner_rule, + build_site_replication_rule("arn:rustfs:replication::kept-dep:photos", 2, "site-repl-kept-dep"), + ], + }; + let (updated, removed) = prune_removed_site_replication_rules(config, &HashSet::from(["removed-dep".to_string()])); + let updated = updated.expect("rules remain"); + assert_eq!(removed, 1); + let rules: Vec<_> = updated + .rules + .iter() + .map(|rule| (rule.id.as_deref().unwrap(), rule.priority)) + .collect(); + assert_eq!(rules, vec![("site-repl-user", Some(9)), ("site-repl-kept-dep", Some(1))]); } #[test] diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index bdf28f15e..f6cfbf033 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -441,8 +441,10 @@ pub(crate) mod quota { pub(crate) mod replication { pub(crate) use super::ecstore_bucket::replication::{ - REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, - REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, + OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, + REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, + REPLICATION_WRITABLE_FIELDS, assign_site_replication_rule_priorities, is_site_replication_role, + merge_incoming_replication_config, replication_target_arn_deployment_id, site_replication_rule_deployment_id, }; pub(crate) type BucketReplicationResyncStatus = super::ecstore_bucket::replication::BucketReplicationResyncStatus; pub(crate) type BucketStats = super::ecstore_bucket::replication::BucketStats; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index b51254b23..02bb10416 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -38,9 +38,9 @@ use super::storage_api::bucket_usecase::bucket::{ metadata_sys, policy_sys::PolicySys, replication::{ - ReplicationTargetValidationError, invalid_replication_config_status_field, replication_target_arns, - should_remove_replication_target, unsupported_replication_config_field, validate_replication_config_structure, - validate_replication_config_target_arns, + OperatorRuleContract, ReplicationTargetValidationError, invalid_replication_config_status_field, + merge_user_replication_config, replication_target_arns, should_remove_replication_target, + unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns, }, target::{BucketTargetType, BucketTargets}, utils::serialize, @@ -509,6 +509,7 @@ fn sr_bucket_meta_item(bucket: String, item_type: &str) -> SRBucketMeta { r#type: item_type.to_string(), updated_at: Some(time::OffsetDateTime::now_utc()), api_version: Some(SITE_REPL_API_VERSION.to_string()), + derived_rule_contract: true, ..Default::default() } } @@ -623,11 +624,56 @@ async fn validate_bucket_replication_update(bucket: &str, config: &ReplicationCo validate_replication_config_targets(&targets, config) } -async fn replication_targets_without_config_targets( +/// Defense in depth for site-replication-managed buckets (issue #1948): an S3 +/// PutBucketReplication replaces the operator-authored rules but must not wipe +/// the rules the reconciler derived for the current remote peers +/// (`site_peer_deployment_ids`) — until its next pass (600s period) every +/// peer link on this bucket would be silently dead. The same merge also drops +/// incoming impostors of those rules. An empty peer set (site replication +/// disabled) keeps the verbatim overwrite semantics: rule ids are not +/// reserved, so an operator's own `site-repl-*` rule is ordinary state there. +/// `contract` is what the peers were probed to support; the merged config is +/// what gets broadcast, so it is built the way every peer will merge it. +fn merge_user_replication_config_update( + incoming: ReplicationConfiguration, + existing: Option, + site_peer_deployment_ids: &HashSet, + contract: OperatorRuleContract, +) -> ReplicationConfiguration { + if site_peer_deployment_ids.is_empty() { + return incoming; + } + // `incoming` passed structure validation, so it holds at least one rule; + // `None` is only reachable when every incoming rule impersonates a + // reconciler rule, and then the stored reconciler rules are what remains. + merge_user_replication_config(Some(incoming.clone()), existing, site_peer_deployment_ids, contract).unwrap_or(incoming) +} + +/// Split of an S3 DeleteBucketReplication on the stored config (issue #1948): +/// the operator-authored rules are removed, the rules the reconciler derived +/// for the current remote peers survive (`None` means nothing survives and +/// the config is deleted), and the returned ARNs are the ones whose bucket +/// targets may be garbage-collected — never an ARN a surviving reconciler +/// rule still points at. +fn split_replication_config_for_user_delete( + config: ReplicationConfiguration, + site_peer_deployment_ids: &HashSet, + contract: OperatorRuleContract, +) -> (Option, HashSet) { + let mut removable_arns = replication_target_arns(&config); + let remaining = merge_user_replication_config(None, Some(config), site_peer_deployment_ids, contract); + if let Some(remaining) = remaining.as_ref() { + for rule in &remaining.rules { + removable_arns.remove(rule.destination.bucket.trim()); + } + } + (remaining, removable_arns) +} + +async fn replication_targets_without_arns( bucket: &str, - config: &ReplicationConfiguration, + target_arns: &HashSet, ) -> S3Result> { - let target_arns = replication_target_arns(config); if target_arns.is_empty() { return Ok(None); } @@ -638,7 +684,7 @@ async fn replication_targets_without_config_targets( Err(err) => return Err(ApiError::from(err).into()), }; - let removed = remove_replication_targets_from_config_targets(&mut targets, &target_arns); + let removed = remove_replication_targets_from_config_targets(&mut targets, target_arns); if removed == 0 { return Ok(None); } @@ -1582,9 +1628,16 @@ impl DefaultBucketUsecase { Ok(S3Response::new(DeleteBucketPolicyOutput {})) } + /// `site_peers` is the set of remote site-replication peer deployment ids + /// (empty when site replication is disabled). The interface layer reads it + /// from the persisted state and fails closed on a read error, so this + /// usecase stays a pure function of its inputs (layer rule: app never + /// imports interface). pub async fn execute_delete_bucket_replication( &self, req: S3Request, + site_peers: HashSet, + contract: OperatorRuleContract, ) -> S3Result> { let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); @@ -1604,15 +1657,29 @@ impl DefaultBucketUsecase { Err(StorageError::ConfigNotFound) => None, Err(err) => return Err(ApiError::from(err).into()), }; - let updated_targets = if let Some(config) = replication_config.as_ref() { - replication_targets_without_config_targets(&bucket, config).await? + let (remaining_config, updated_targets) = if let Some(config) = replication_config.as_ref() { + let (remaining, removable_arns) = split_replication_config_for_user_delete(config.clone(), &site_peers, contract); + let targets = replication_targets_without_arns(&bucket, &removable_arns).await?; + (remaining, targets) } else { - None + (None, None) }; - delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id) - .await - .map_err(ApiError::from)?; + match remaining_config { + // Site-replication rules and the targets backing them survive the + // S3 delete (issue #1948); only the operator-authored rules go. + Some(remaining) => { + let data = serialize_config(&remaining)?; + update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + } + None => { + delete_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, expected_incarnation_id) + .await + .map_err(ApiError::from)?; + } + } if let Some((targets, removed)) = updated_targets && let Err(err) = write_replication_targets_after_config_delete(&bucket, &targets, removed, expected_incarnation_id).await @@ -2459,9 +2526,13 @@ impl DefaultBucketUsecase { Ok(S3Response::new(PutBucketCorsOutput::default())) } + /// See [`Self::execute_delete_bucket_replication`] for `site_peers` and + /// `contract`. pub async fn execute_put_bucket_replication( &self, req: S3Request, + site_peers: HashSet, + contract: OperatorRuleContract, ) -> S3Result> { let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?; let request_context = req.extensions.get::().cloned(); @@ -2485,6 +2556,13 @@ impl DefaultBucketUsecase { let targets_guard = lock_bucket_targets_metadata(&bucket).await; validate_bucket_replication_update(&bucket, &replication_configuration).await?; + let existing_config = match metadata_sys::get_replication_config(&bucket).await { + Ok((config, _)) => Some(config), + Err(StorageError::ConfigNotFound) => None, + Err(err) => return Err(ApiError::from(err).into()), + }; + let replication_configuration = + merge_user_replication_config_update(replication_configuration, existing_config, &site_peers, contract); let data = serialize_config(&replication_configuration)?; update_bucket_config_for_incarnation(&bucket, BUCKET_REPLICATION_CONFIG, data, expected_incarnation_id) .await @@ -3114,6 +3192,200 @@ mod tests { assert!(arns.contains(destination)); } + fn replication_rule_with_id(arn: &str, id: &str, priority: i32) -> ReplicationRule { + let mut rule = replication_rule_for_target(arn); + rule.id = Some(id.to_string()); + rule.priority = Some(priority); + rule + } + + fn site_peers(deployment_ids: &[&str]) -> HashSet { + deployment_ids.iter().map(|id| id.to_string()).collect() + } + + #[test] + fn put_replication_merge_preserves_site_replication_rules() { + let existing = ReplicationConfiguration { + role: String::new(), + rules: vec![ + replication_rule_with_id("arn:rustfs:replication::peer-dep:bucket", "site-repl-peer-dep", 1), + replication_rule_with_id("arn:rustfs:replication:us-east-1:old:bucket", "old-user-rule", 2), + ], + }; + let incoming = ReplicationConfiguration { + role: String::new(), + rules: vec![ + replication_rule_with_id("arn:rustfs:replication:us-east-1:new:bucket", "new-user-rule", 1), + replication_rule_with_id("arn:rustfs:replication::forged-dep:bucket", "site-repl-peer-dep", 2), + replication_rule_with_id("arn:rustfs:replication::other-dep:bucket", "site-repl-other", 3), + ], + }; + + let merged = merge_user_replication_config_update( + incoming, + Some(existing), + &site_peers(&["peer-dep"]), + OperatorRuleContract::Derived, + ); + + let rules: Vec<_> = merged + .rules + .iter() + .map(|rule| (rule.id.as_deref().unwrap_or_default(), rule.destination.bucket.as_str())) + .collect(); + assert_eq!( + rules, + vec![ + ("new-user-rule", "arn:rustfs:replication:us-east-1:new:bucket"), + ("site-repl-other", "arn:rustfs:replication::other-dep:bucket"), + ("site-repl-peer-dep", "arn:rustfs:replication::peer-dep:bucket"), + ], + "user rules replaced, the reconciler rule for the current peer kept over the incoming impostor, \ + a site-repl-* id that names no current peer is ordinary operator state" + ); + } + + // Rule ids do not reserve `site-repl-*`: outside site replication an + // owner's `site-repl-user` rule is ordinary state, so PUT stores it + // verbatim and DELETE removes it and garbage-collects its target. + #[test] + fn put_then_delete_replication_without_site_replication_treats_site_repl_id_as_user_rule() { + let user_arn = "arn:minio:replication:us-east-1:2f1c-remote:bucket"; + let incoming = ReplicationConfiguration { + role: String::new(), + rules: vec![replication_rule_with_id(user_arn, "site-repl-user", 1)], + }; + + let stored = merge_user_replication_config_update(incoming.clone(), None, &HashSet::new(), OperatorRuleContract::Derived); + assert_eq!(stored, incoming, "PUT on a non-site-replication bucket is verbatim"); + + let (remaining, removable) = + split_replication_config_for_user_delete(stored, &HashSet::new(), OperatorRuleContract::Derived); + assert!(remaining.is_none(), "DELETE must remove the operator's site-repl-* rule"); + assert_eq!(removable, HashSet::from([user_arn.to_string()])); + } + + // Under site replication only a rule the reconciler would derive — id + // `site-repl-` for a current peer, destination ARN naming the same + // peer — is reconciler-owned. Everything else is operator state. + #[test] + fn delete_replication_split_keeps_only_reconciler_derived_rules() { + let peer_arn = "arn:rustfs:replication::peer-dep:bucket"; + let user_arn = "arn:minio:replication:us-east-1:2f1c-remote:bucket"; + let config = ReplicationConfiguration { + role: String::new(), + rules: vec![ + replication_rule_with_id(user_arn, "site-repl-user", 1), + replication_rule_with_id(user_arn, "site-repl-peer-dep", 2), + replication_rule_with_id("arn:rustfs:replication::gone-dep:bucket", "site-repl-gone-dep", 3), + replication_rule_with_id(peer_arn, "site-repl-peer-dep", 4), + ], + }; + + let (remaining, removable) = + split_replication_config_for_user_delete(config, &site_peers(&["peer-dep"]), OperatorRuleContract::Derived); + + let remaining = remaining.expect("the reconciler-derived rule must survive"); + assert_eq!(remaining.rules.len(), 1); + assert_eq!(remaining.rules[0].destination.bucket, peer_arn); + assert_eq!( + removable, + HashSet::from([user_arn.to_string(), "arn:rustfs:replication::gone-dep:bucket".to_string()]), + "targets of operator rules and of a removed peer are garbage-collected" + ); + } + + #[test] + fn put_replication_merge_returns_incoming_verbatim_without_site_rules() { + let existing = ReplicationConfiguration { + role: String::new(), + rules: vec![replication_rule_with_id( + "arn:rustfs:replication:us-east-1:old:bucket", + "old-user-rule", + 7, + )], + }; + let incoming = ReplicationConfiguration { + role: String::new(), + rules: vec![replication_rule_with_id( + "arn:rustfs:replication:us-east-1:new:bucket", + "new-user-rule", + 5, + )], + }; + + let merged = merge_user_replication_config_update( + incoming.clone(), + Some(existing), + &HashSet::new(), + OperatorRuleContract::Derived, + ); + + assert_eq!(merged.role, incoming.role); + assert_eq!(merged.rules, incoming.rules, "non-SR buckets keep the verbatim overwrite semantics"); + } + + #[test] + fn delete_replication_split_keeps_site_rules_and_their_targets() { + let sr_arn = "arn:rustfs:replication::peer-dep:bucket"; + let user_arn = "arn:rustfs:replication:us-east-1:user:bucket"; + let config = ReplicationConfiguration { + role: String::new(), + rules: vec![ + replication_rule_with_id(user_arn, "user-rule", 1), + replication_rule_with_id(sr_arn, "site-repl-peer-dep", 2), + ], + }; + + let (remaining, removable) = + split_replication_config_for_user_delete(config, &site_peers(&["peer-dep"]), OperatorRuleContract::Derived); + + let remaining = remaining.expect("site-replication rules must survive a user delete"); + let ids: Vec<_> = remaining + .rules + .iter() + .map(|rule| rule.id.as_deref().unwrap_or_default()) + .collect(); + assert_eq!(ids, vec!["site-repl-peer-dep"]); + assert_eq!(removable, HashSet::from([user_arn.to_string()])); + } + + #[test] + fn delete_replication_split_protects_targets_shared_with_site_rules() { + let sr_arn = "arn:rustfs:replication::peer-dep:bucket"; + let config = ReplicationConfiguration { + role: String::new(), + rules: vec![ + replication_rule_with_id(sr_arn, "user-rule-on-sr-target", 1), + replication_rule_with_id(sr_arn, "site-repl-peer-dep", 2), + ], + }; + + let (remaining, removable) = + split_replication_config_for_user_delete(config, &site_peers(&["peer-dep"]), OperatorRuleContract::Derived); + + assert!(remaining.is_some()); + assert!( + removable.is_empty(), + "a target still referenced by a surviving site-replication rule must not be removed" + ); + } + + #[test] + fn delete_replication_split_removes_everything_without_site_rules() { + let user_arn = "arn:rustfs:replication:us-east-1:user:bucket"; + let config = ReplicationConfiguration { + role: String::new(), + rules: vec![replication_rule_with_id(user_arn, "user-rule", 1)], + }; + + let (remaining, removable) = + split_replication_config_for_user_delete(config, &site_peers(&["peer-dep"]), OperatorRuleContract::Derived); + + assert!(remaining.is_none(), "without site-replication rules the whole config is deleted"); + assert_eq!(removable, HashSet::from([user_arn.to_string()])); + } + fn replication_targets_with_arn(arns: &[&str]) -> BucketTargets { BucketTargets { targets: arns @@ -3451,7 +3723,10 @@ mod tests { let req = build_request(input, Method::DELETE); let usecase = DefaultBucketUsecase::without_context(); - let err = usecase.execute_delete_bucket_replication(req).await.unwrap_err(); + let err = usecase + .execute_delete_bucket_replication(req, HashSet::new(), OperatorRuleContract::Derived) + .await + .unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InternalError); } @@ -4537,7 +4812,10 @@ mod tests { let req = build_request(input, Method::PUT); let usecase = DefaultBucketUsecase::without_context(); - let err = usecase.execute_put_bucket_replication(req).await.unwrap_err(); + let err = usecase + .execute_put_bucket_replication(req, HashSet::new(), OperatorRuleContract::Derived) + .await + .unwrap_err(); assert_eq!(err.code(), &S3ErrorCode::InternalError); } @@ -4555,7 +4833,7 @@ mod tests { .unwrap(); let err = DefaultBucketUsecase::without_context() - .execute_put_bucket_replication(build_request(input, Method::PUT)) + .execute_put_bucket_replication(build_request(input, Method::PUT), HashSet::new(), OperatorRuleContract::Derived) .await .expect_err("unsupported fields must be rejected before store access"); diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index a599cfde6..6bb5e9728 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -619,6 +619,8 @@ pub(crate) mod bucket { use crate::storage::storage_api::ecstore_bucket::replication as replication_contracts; + pub(crate) use replication_contracts::{OperatorRuleContract, merge_user_replication_config}; + type ReplicationObjectBridge = crate::storage::storage_api::ecstore_bucket::replication::ReplicationObjectBridge; pub(crate) type DeleteReplicationConfigSnapshot = crate::storage::storage_api::ecstore_bucket::replication::DeleteReplicationConfigSnapshot; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index da0c9b775..aedd3e4d5 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -59,10 +59,74 @@ const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock"; const LOG_SUBSYSTEM_TAGGING: &str = "tagging"; use crate::app::storage_api::object_usecase::bucket::replication::{ - ReplicateDecision, get_read_proxy_targets, must_replicate_metadata, schedule_metadata_replication, + OperatorRuleContract, ReplicateDecision, get_read_proxy_targets, must_replicate_metadata, schedule_metadata_replication, }; use crate::storage::storage_api::ecfs_consumer::StorageObjectOptions as ObjectOptions; +#[cfg(test)] +static SITE_REPLICATION_GATE_TEST_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); +#[cfg(test)] +const SITE_REPLICATION_GATE_FORCE_DISABLED: u8 = 1; +#[cfg(test)] +const SITE_REPLICATION_GATE_FORCE_ENABLED: u8 = 2; + +async fn site_replication_gate_enabled() -> S3Result { + #[cfg(test)] + match SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) { + SITE_REPLICATION_GATE_FORCE_DISABLED => return Ok(false), + SITE_REPLICATION_GATE_FORCE_ENABLED => return Ok(true), + _ => {} + } + crate::admin::handlers::site_replication::site_replication_enabled().await +} + +/// Remote site-replication peer deployment ids and the operator-rule contract +/// the peers support, handed to the bucket usecase so an S3 replication-config +/// edit keeps exactly the reconciler-owned rules and merges the way every +/// peer will (issue #1948). Read here, in the interface layer, because the +/// usecase must not import the admin handlers (layer guard); a state-read +/// failure propagates so the edit fails closed. +async fn site_replication_edit_context() -> S3Result<(std::collections::HashSet, OperatorRuleContract)> { + // While the gate override is in effect the test exercises the deny/allow + // branch, not the peer set; there is no persisted state to read. + #[cfg(test)] + if SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) != 0 { + return Ok((std::collections::HashSet::new(), OperatorRuleContract::Derived)); + } + crate::admin::handlers::site_replication::site_replication_edit_context().await +} + +/// MinIO `ErrReplicationDenyEditError`. +fn replication_deny_edit_error() -> S3Error { + let mut err = S3Error::with_message( + S3ErrorCode::Custom("XMinioReplicationDenyEdit".into()), + "Sub-User is not allowed to edit Replication configuration", + ); + err.set_status_code(StatusCode::BAD_REQUEST); + err +} + +/// Site-replication gate for S3 replication-config edits (issue #1948). +/// +/// On a site-replication deployment the bucket's replication config carries +/// the operator-managed `site-repl-*` rules that keep every peer in sync, and +/// a successful edit is broadcast to all peers — so a user holding only +/// bucket-scoped `s3:PutReplicationConfiguration` could rewrite or erase +/// replication net-wide. MinIO parity (`ErrReplicationDenyEditError`): only +/// owner credentials (root or root-parented) may edit. Runs after the policy +/// authorization in the access layer and only on the external S3 path — the +/// reconciler and peer bucket-meta ingestion never route through these +/// handlers. +async fn deny_replication_config_edit_for_non_owner(req: &S3Request) -> S3Result<()> { + if crate::storage::access::req_info_ref(req)?.is_owner { + return Ok(()); + } + if site_replication_gate_enabled().await? { + return Err(replication_deny_edit_error()); + } + Ok(()) +} + #[derive(Debug, Clone)] pub struct FS { /// This server's late-bound application-context slot (backlog#1052 S2). @@ -500,8 +564,10 @@ impl S3 for FS { &self, req: S3Request, ) -> S3Result> { + deny_replication_config_edit_for_non_owner(&req).await?; + let (site_peers, contract) = site_replication_edit_context().await?; let usecase = s3_api::bucket_usecase_for(self); - usecase.execute_delete_bucket_replication(req).await + usecase.execute_delete_bucket_replication(req, site_peers, contract).await } #[instrument(level = "debug", skip(self))] @@ -1353,8 +1419,10 @@ impl S3 for FS { &self, req: S3Request, ) -> S3Result> { + deny_replication_config_edit_for_non_owner(&req).await?; + let (site_peers, contract) = site_replication_edit_context().await?; let usecase = s3_api::bucket_usecase_for(self); - usecase.execute_put_bucket_replication(req).await + usecase.execute_put_bucket_replication(req, site_peers, contract).await } async fn put_bucket_request_payment( @@ -1919,3 +1987,103 @@ impl S3 for FS { Box::pin(usecase.execute_upload_part_copy(req)).await } } + +#[cfg(test)] +mod tests { + use super::{ + FS, SITE_REPLICATION_GATE_FORCE_DISABLED, SITE_REPLICATION_GATE_FORCE_ENABLED, SITE_REPLICATION_GATE_TEST_OVERRIDE, + }; + use crate::storage::access::ReqInfo; + use http::Method; + use http::StatusCode; + use s3s::dto::{DeleteBucketReplicationInput, PutBucketReplicationInput, ReplicationConfiguration}; + use s3s::{S3, S3Error, S3ErrorCode, S3Request}; + use std::sync::atomic::Ordering; + + fn replication_config_edit_request(input: T, is_owner: bool) -> S3Request { + let mut req = S3Request { + input, + method: Method::PUT, + uri: http::Uri::from_static("/"), + headers: http::HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + req.extensions.insert(ReqInfo { + is_owner, + ..Default::default() + }); + req + } + + fn put_bucket_replication_input() -> PutBucketReplicationInput { + PutBucketReplicationInput { + bucket: "test-bucket".to_string(), + checksum_algorithm: None, + content_md5: None, + expected_bucket_owner: None, + replication_configuration: ReplicationConfiguration { + role: String::new(), + rules: Vec::new(), + }, + token: None, + } + } + + fn delete_bucket_replication_input() -> DeleteBucketReplicationInput { + DeleteBucketReplicationInput { + bucket: "test-bucket".to_string(), + expected_bucket_owner: None, + } + } + + fn assert_replication_deny_edit(err: &S3Error) { + match err.code() { + S3ErrorCode::Custom(code) => assert_eq!(code, "XMinioReplicationDenyEdit"), + other => panic!("expected XMinioReplicationDenyEdit, got {other:?}"), + } + assert_eq!(err.status_code(), Some(StatusCode::BAD_REQUEST)); + } + + /// Single test on purpose: the branches share the process-wide gate + /// override, and parallel tests would race it. + #[tokio::test] + async fn replication_config_edit_gate_denies_only_non_owner_under_site_replication() { + let fs = FS::new(); + SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_ENABLED, Ordering::SeqCst); + + // Non-owner PUT/DELETE through the real S3 handlers: denied by the + // gate before the usecase (and thus the store) is ever touched. + let err = fs + .put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false)) + .await + .expect_err("non-owner PutBucketReplication must be denied while site replication is enabled"); + assert_replication_deny_edit(&err); + let err = fs + .delete_bucket_replication(replication_config_edit_request(delete_bucket_replication_input(), false)) + .await + .expect_err("non-owner DeleteBucketReplication must be denied while site replication is enabled"); + assert_replication_deny_edit(&err); + + // Owner passes the gate (the usecase's empty-rules structure error + // proves the request reached the usecase instead of the deny path). + let err = fs + .put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), true)) + .await + .expect_err("owner request should pass the gate and fail later on config validation"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + + // Without site replication the policy check alone still governs the edit. + SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_DISABLED, Ordering::SeqCst); + let err = fs + .put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false)) + .await + .expect_err("non-owner request should pass the gate and fail later on config validation"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + + SITE_REPLICATION_GATE_TEST_OVERRIDE.store(0, Ordering::SeqCst); + } +} From 450ec7f66a658e4b5e8ed35a256913281c7e0c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 23 Aug 2026 16:42:45 +0800 Subject: [PATCH 09/41] fix(admin): bound site replication lifecycle lock and parallelize add preflight (#6378) The site replication add preflight probed peer sites serially while holding the process-wide lifecycle lock, so k unreachable sites held the lock for k peer-request timeouts, and every concurrent add/remove/refresh waited on an unbounded lock acquire for the whole time. Probe all sites concurrently (matching the file's other peer fan-outs) so k unreachable sites cost roughly one timeout, and bound the lifecycle lock acquire at 30s, returning a retryable 503 to waiters instead of hanging indefinitely. Regression tests pin the preflight fan-out concurrency, the bounded acquire's 503, and the 10s/3s peer client timeout constants. Refs rustfs/backlog#1952, rustfs/backlog#1946, rustfs/backlog#1889 Co-authored-by: houseme --- rustfs/src/admin/handlers/site_replication.rs | 178 ++++++++++++++++-- 1 file changed, 158 insertions(+), 20 deletions(-) diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index d28033085..f41f9d182 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -142,6 +142,11 @@ const SITE_REPL_RESYNC_DEFAULT_PAGE_SIZE: usize = 100; const SITE_REPL_RESYNC_MAX_PAGE_SIZE: usize = 1000; const SITE_REPLICATION_PEER_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); const SITE_REPLICATION_PEER_CONNECT_TIMEOUT: Duration = Duration::from_secs(3); +/// Bound on waiting for the lifecycle lock (below). 3x the peer request +/// timeout: outlives one full peer round of a healthy concurrent lifecycle +/// operation, while converting a holder wedged on unreachable peers into a +/// retryable 503 for the waiter instead of an unbounded hang. +const SITE_REPLICATION_LIFECYCLE_LOCK_TIMEOUT: Duration = Duration::from_secs(30); const SITE_REPLICATION_PEER_ERROR_DETAIL_LIMIT: usize = 256; const SITE_REPLICATION_INITIAL_SYNC_ERROR_LIMIT: usize = 32; const MAX_PEER_CA_CERT_PEM_SIZE: usize = 256 * 1024; @@ -394,9 +399,17 @@ struct SiteReplicationLifecycleGuard { } impl SiteReplicationLifecycleGuard { - async fn acquire() -> Self { - Self { - _guard: SITE_REPLICATION_LIFECYCLE_LOCK.lock().await, + /// Bounded acquire: a holder wedged on unreachable peers (each probe + /// costs up to [`SITE_REPLICATION_PEER_REQUEST_TIMEOUT`]) must not hang + /// every other lifecycle operation indefinitely, so waiters get a + /// retryable 503 after [`SITE_REPLICATION_LIFECYCLE_LOCK_TIMEOUT`]. + async fn acquire() -> S3Result { + match tokio::time::timeout(SITE_REPLICATION_LIFECYCLE_LOCK_TIMEOUT, SITE_REPLICATION_LIFECYCLE_LOCK.lock()).await { + Ok(guard) => Ok(Self { _guard: guard }), + Err(_) => Err(S3Error::with_message( + S3ErrorCode::ServiceUnavailable, + "another site replication lifecycle operation is in progress; retry later".to_string(), + )), } } @@ -2242,6 +2255,27 @@ async fn remote_add_preflight_info(site: &PeerSite) -> S3Result S3Result> { + futures::future::join_all(sites.iter().map(|site| async move { + if same_identity_endpoint(&site.endpoint, &local_peer.endpoint) { + local_add_preflight_info(current_state, local_peer, site).await + } else { + remote_add_preflight_info(site).await + } + })) + .await + .into_iter() + .collect() +} + fn validate_add_preflight_topology(infos: &[SiteReplicationAddPreflightInfo], local_peer: &PeerInfo) -> S3Result<()> { let mut deployment_ids = HashSet::new(); let mut local_seen = false; @@ -9941,7 +9975,7 @@ impl Operation for SiteReplicationAddHandler { let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?; reject_site_replicator_on_public_admin(&cred)?; let replicate_ilm_expiry = sr_add_replicate_ilm_expiry(&req.uri); - let lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; + let lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?; // Everything up to the commit below is preflight: peer probes, IAM // work and the join fan-out all talk to the network, so none of it may // run inside the state transaction. The snapshot read here is what the @@ -9956,14 +9990,7 @@ impl Operation for SiteReplicationAddHandler { // inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`. ensure_local_site_present(&mut sites, &local_peer); validate_add_sites(&sites, &local_peer)?; - let mut preflight_infos = Vec::with_capacity(sites.len()); - for site in &sites { - if same_identity_endpoint(&site.endpoint, &local_peer.endpoint) { - preflight_infos.push(local_add_preflight_info(¤t_state, &local_peer, site).await?); - } else { - preflight_infos.push(remote_add_preflight_info(site).await?); - } - } + let preflight_infos = add_preflight_infos(&sites, ¤t_state, &local_peer).await?; validate_add_preflight_topology(&preflight_infos, &local_peer)?; let expected_updated_at = current_state.updated_at; require_add_peer_tls_capability(&sites, &local_peer).await?; @@ -10171,7 +10198,7 @@ impl Operation for SiteReplicationRemoveHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationRemoveAction).await?; reject_site_replicator_on_public_admin(&cred)?; - let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; + let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?; // The request body is read before the bucket-op guard and the state // transaction: a client that stalls mid-body must hold neither the // state-object lock nor the write half of the bucket-op RwLock (which @@ -10381,7 +10408,7 @@ where F: FnOnce(SRPeerJoinReq) -> Fut + Send + 'static, Fut: std::future::Future> + Send + 'static, { - let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; + let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?; admit_peer_join_across_nodes(local_endpoint, join_req, defer_sync_state_enable, apply_iam).await } @@ -11211,7 +11238,7 @@ impl Operation for SRPeerRemoveHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { validate_site_replication_admin_request(&req, AdminAction::SiteReplicationRemoveAction).await?; let remove_req: SRRemoveReq = read_site_replication_json(req, "", false).await?; - let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; + let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?; let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.write().await; let removed_deployment_ids = update_site_replication_state(move |state| { if pending_endpoint_refresh(state).is_some() { @@ -11255,7 +11282,7 @@ impl Operation for SiteReplicationResyncOpHandler { let operation = query.get("operation").cloned().unwrap_or_default(); let resolved_store = object_store_from_req(&req); let requested_peer: PeerInfo = read_site_replication_json(req, "", false).await?; - let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; + let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?; let (peer, existing_status) = { let state = load_site_replication_state().await?; let local_peer = current_local_runtime_peer(&state); @@ -11547,7 +11574,7 @@ impl Operation for SRRotateServiceAccountHandler { // mid-repair and race its own IAM write against the reconciler's // stale one. (The removed process mutex used to provide this // exclusion as a side effect.) - let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await; + let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?; let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers); let rotation_parent = cred.access_key.clone(); let (pending_rotation, local_peer, previous_access_key) = update_site_replication_state_when_changed(move |state| { @@ -14089,7 +14116,9 @@ mod tests { async fn test_add_bootstrap_scope_only_allows_expected_bucket_setup_until_guard_drops() { let token; { - let lifecycle = SiteReplicationLifecycleGuard::acquire().await; + let lifecycle = SiteReplicationLifecycleGuard::acquire() + .await + .expect("acquire lifecycle guard"); let guard = SiteReplicationAddInProgressGuard::start(lifecycle, HashSet::from(["legacy-bucket".to_string()])) .expect("start site replication add guard"); token = guard.token.to_string(); @@ -14145,14 +14174,18 @@ mod tests { #[tokio::test] #[serial] async fn test_add_lifecycle_allows_callback_before_remove_writer() { - let lifecycle = SiteReplicationLifecycleGuard::acquire().await; + let lifecycle = SiteReplicationLifecycleGuard::acquire() + .await + .expect("acquire lifecycle guard"); let add_guard = SiteReplicationAddInProgressGuard::start(lifecycle, HashSet::new()).expect("start site replication add guard"); let (started_tx, started_rx) = tokio::sync::oneshot::channel(); let (entered_tx, mut entered_rx) = tokio::sync::oneshot::channel(); let remove = tokio::spawn(async move { let _ = started_tx.send(()); - let _lifecycle = SiteReplicationLifecycleGuard::acquire().await; + let _lifecycle = SiteReplicationLifecycleGuard::acquire() + .await + .expect("acquire lifecycle guard"); let _bucket_op = SITE_REPLICATION_BUCKET_OP_LOCK.write().await; let _ = entered_tx.send(()); }); @@ -14172,6 +14205,111 @@ mod tests { entered_rx.await.expect("remove entered lifecycle"); } + /// Deleting either constant (or "simplifying" the client builders to + /// inline values) removes the only bound on how long a lifecycle + /// operation can be wedged per unreachable peer (#1889 C1 / #1952 C2). + #[test] + fn test_peer_timeout_constants_bound_unreachable_peer_probes() { + assert_eq!(SITE_REPLICATION_PEER_REQUEST_TIMEOUT, Duration::from_secs(10)); + assert_eq!(SITE_REPLICATION_PEER_CONNECT_TIMEOUT, Duration::from_secs(3)); + assert!( + SITE_REPLICATION_LIFECYCLE_LOCK_TIMEOUT >= SITE_REPLICATION_PEER_REQUEST_TIMEOUT, + "a waiter must not give up before the holder's single wedged peer probe can finish" + ); + } + + #[tokio::test(start_paused = true)] + #[serial] + async fn test_lifecycle_guard_acquire_times_out_with_retryable_503() { + let holder = SiteReplicationLifecycleGuard::acquire().await.expect("first acquire"); + let err = + match tokio::time::timeout(SITE_REPLICATION_LIFECYCLE_LOCK_TIMEOUT * 2, SiteReplicationLifecycleGuard::acquire()) + .await + .expect("bounded acquire must not hang while the lock is held") + { + Ok(_) => panic!("acquire while the lock is held should time out"), + Err(err) => err, + }; + assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable); + + drop(holder); + tokio::time::timeout(Duration::from_secs(1), SiteReplicationLifecycleGuard::acquire()) + .await + .expect("acquire after release must not wait") + .expect("acquire after release"); + } + + #[derive(Clone)] + struct PreflightFanoutTestState { + metainfo_barrier: Arc, + } + + async fn preflight_fanout_test_handler(State(state): State, uri: Uri) -> (StatusCode, String) { + if uri.path().ends_with("/site-replication/metainfo") { + state.metainfo_barrier.wait().await; + } + (StatusCode::OK, "{}".to_string()) + } + + #[tokio::test] + #[serial] + async fn test_add_preflight_probes_sites_concurrently() { + temp_env::async_with_vars( + [(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV, Some("true"))], + add_preflight_probes_sites_concurrently_inner(), + ) + .await; + } + + async fn add_preflight_probes_sites_concurrently_inner() { + const REMOTE_SITES: usize = 3; + let listener = match TcpListener::bind("127.0.0.1:0").await { + Ok(listener) => listener, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return, + Err(err) => panic!("bind preflight test server: {err}"), + }; + let endpoint = format!("http://{}", listener.local_addr().expect("preflight test address")); + let state = PreflightFanoutTestState { + metainfo_barrier: Arc::new(tokio::sync::Barrier::new(REMOTE_SITES)), + }; + let server = tokio::spawn(async move { + axum::serve(listener, Router::new().fallback(any(preflight_fanout_test_handler)).with_state(state)) + .await + .expect("serve preflight test requests"); + }); + + let sites: Vec = (0..REMOTE_SITES) + .map(|index| PeerSite { + name: format!("site-{index}"), + endpoint: endpoint.clone(), + access_key: "test-access".to_string(), + secret_key: "test-secret".to_string(), + ..Default::default() + }) + .collect(); + let local_peer = PeerInfo { + deployment_id: "local".to_string(), + endpoint: "http://192.0.2.1:9000".to_string(), + ..Default::default() + }; + let current_state = SiteReplicationState::default(); + + // Each site's metainfo request parks on a barrier that only releases + // once every site's request has arrived: serial probing never sends + // the second request and dies on the peer request timeout, so + // finishing well inside that timeout proves the probes overlap — + // which is what caps k unreachable sites at one timeout, not k. + let infos = tokio::time::timeout( + SITE_REPLICATION_PEER_REQUEST_TIMEOUT / 2, + add_preflight_infos(&sites, ¤t_state, &local_peer), + ) + .await + .expect("preflight probes must fan out concurrently, not serially") + .expect("preflight infos"); + assert_eq!(infos.len(), REMOTE_SITES); + server.abort(); + } + #[test] fn test_merge_add_sites_propagates_replicate_ilm_expiry() { let state = merge_add_sites( From 34bbc1adb34d84e1820824f1e75e1c5ae413db7c Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 16:43:11 +0800 Subject: [PATCH 10/41] fix(ecstore): preserve ILM state during pool decommission (#6369) * fix(ecstore): migrate ILM metadata during decommission * fix(ecstore): verify ILM metadata before decommission * fix(ecstore): track ILM recovery across decommission * fix(ecstore): close ILM receipt recovery gaps * fix(ecstore): anchor decommission ILM receipts * fix(ecstore): re-export durable ILM checkpoint * fix(ecstore): avoid terminal receipt shadowing * fix(ecstore): harden durable ILM cursor receipts * fix(ecstore): repair durable ILM receipt recovery * test(ecstore): cover durable ILM recovery boundaries * test(ecstore): serialize multi-source ILM recovery * test(ecstore): compile multi-source ILM recovery * test(ecstore): isolate durable ILM scenario stack * fix(ecstore): preserve active ILM source journals * fix(ecstore): distinguish active ILM target cleanup * fix(ecstore): restore decommission test imports * fix(ecstore): drop removed decommission test import * fix(ecstore): remove duplicate decommission error helper * fix(ecstore): fence final decommission sweep * test(ecstore): cover final sweep cancel fence * fix(ecstore): fence decommission cancellation * fix(ecstore): remove redundant clone in test * fix(ecstore): keep manual transition progress compatible * fix(ecstore): restore decommission worker wrapper * fix(ecstore): restore decommission compile contracts * test(ecstore): adapt reload worker canceler --- .config/nextest.toml | 10 + .../bucket/lifecycle/bucket_lifecycle_ops.rs | 73 +- .../src/bucket/lifecycle/durable_namespace.rs | 1161 +++++++++++ .../bucket/lifecycle/manual_transition_job.rs | 242 ++- crates/ecstore/src/bucket/lifecycle/mod.rs | 6 + .../bucket/lifecycle/tier_delete_journal.rs | 18 +- .../lifecycle/transition_transaction.rs | 36 +- crates/ecstore/src/config/com.rs | 19 + crates/ecstore/src/core/pools.rs | 1721 ++++++++++++++++- crates/ecstore/src/set_disk/ops/object.rs | 9 +- crates/ecstore/src/store/init.rs | 1093 ++++++++++- crates/ecstore/src/store/rebalance.rs | 4 +- 12 files changed, 4272 insertions(+), 120 deletions(-) create mode 100644 crates/ecstore/src/bucket/lifecycle/durable_namespace.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 097c79e3e..edf5b5df7 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -78,6 +78,12 @@ test-group = 'embedded-test-ports' filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)' test-group = 'ecstore-serial-flaky' +# The durable ILM decommission regressions build isolated multi-pool stores and +# deliberately take source or target disks offline while checking fencing. +[[profile.default.overrides]] +filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))' +test-group = 'ecstore-serial-flaky' + # Serialize the bucket-incarnation / lifecycle-fence tests. They drive # init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global # OnceLock state that serial_test's #[serial] cannot protect across nextest's @@ -190,6 +196,10 @@ test-group = 'embedded-test-ports' filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)' test-group = 'ecstore-serial-flaky' +[[profile.ci.overrides]] +filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))' +test-group = 'ecstore-serial-flaky' + # Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile # too (see the matching default-profile override near the top). No retries. [[profile.ci.overrides]] diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 1162787f7..48039a7c1 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -3739,17 +3739,55 @@ impl ManualTransitionRunReport { } pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) { + let previous = self.clone(); + let resumed_after_checkpoint = previous.continuation_token.is_some() && scan_report.scanned < previous.scanned; let mut tier_failure_by_reason = self.tier_failure_by_reason.clone(); for (reason, count) in &scan_report.tier_failure_by_reason { let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default(); - tier_failure_by_reason.insert(*reason, current.max(*count)); + let merged = if resumed_after_checkpoint { + current.saturating_add(*count) + } else { + current.max(*count) + }; + tier_failure_by_reason.insert(*reason, merged); } let transition_completed = self.transition_completed; let transition_failed = self.transition_failed; *self = scan_report.clone(); + if resumed_after_checkpoint { + self.scanned = previous.scanned.saturating_add(scan_report.scanned); + self.eligible = previous.eligible.saturating_add(scan_report.eligible); + self.enqueued = previous.enqueued.saturating_add(scan_report.enqueued); + self.dry_run_eligible = previous.dry_run_eligible.saturating_add(scan_report.dry_run_eligible); + self.skipped_not_transition = previous + .skipped_not_transition + .saturating_add(scan_report.skipped_not_transition); + self.skipped_tier = previous.skipped_tier.saturating_add(scan_report.skipped_tier); + self.skipped_delete_marker = previous + .skipped_delete_marker + .saturating_add(scan_report.skipped_delete_marker); + self.skipped_directory = previous.skipped_directory.saturating_add(scan_report.skipped_directory); + self.skipped_replication = previous.skipped_replication.saturating_add(scan_report.skipped_replication); + self.skipped_already_transitioned = previous + .skipped_already_transitioned + .saturating_add(scan_report.skipped_already_transitioned); + self.skipped_already_in_flight = previous + .skipped_already_in_flight + .saturating_add(scan_report.skipped_already_in_flight); + self.skipped_queue_full = previous.skipped_queue_full.saturating_add(scan_report.skipped_queue_full); + self.skipped_queue_closed = previous.skipped_queue_closed.saturating_add(scan_report.skipped_queue_closed); + self.skipped_queue_timeout = previous + .skipped_queue_timeout + .saturating_add(scan_report.skipped_queue_timeout); + self.tier_failure = previous.tier_failure.saturating_add(scan_report.tier_failure); + } + self.lifecycle_config_found = previous.lifecycle_config_found || scan_report.lifecycle_config_found; + self.truncated_by_limit = previous.truncated_by_limit || scan_report.truncated_by_limit; + self.truncated_by_duration = previous.truncated_by_duration || scan_report.truncated_by_duration; + self.cancelled = previous.cancelled || scan_report.cancelled; self.transition_completed = transition_completed; self.transition_failed = transition_failed; - self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed); + self.tier_failure = self.tier_failure.saturating_add(transition_failed); self.tier_failure_by_reason = tier_failure_by_reason; } @@ -3765,7 +3803,10 @@ struct ManualTransitionContinuationToken { version_marker: Option, } -fn encode_manual_transition_continuation_token(marker: Option, version_marker: Option) -> Option { +pub(super) fn encode_manual_transition_continuation_token( + marker: Option, + version_marker: Option, +) -> Option { if marker.is_none() && version_marker.is_none() { return None; } @@ -9136,6 +9177,7 @@ mod tests { assert_eq!(loaded.report.scanned, 37); assert_eq!(loaded.report.eligible, 11); assert_eq!(loaded.report.enqueued, 5); + assert_eq!(loaded.cursor_revision, Some(37)); assert!(loaded.lease_expires_at_unix_nanos > 0); let token = loaded .report @@ -9154,6 +9196,30 @@ mod tests { assert_eq!(admission.lease_id, loaded.lease_id); assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos); + let mut same_marker_report = report.clone(); + same_marker_report.scanned += 1; + persist_manual_transition_page_checkpoint( + &checkpoint_options, + &same_marker_report, + Some("logs/page-end".to_string()), + Some("opaque-next-version".to_string()), + ) + .await + .expect("same-marker version checkpoint should persist through the durable progress sink"); + let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id) + .await + .expect("same-marker version checkpoint should reload"); + assert_eq!(same_marker_checkpointed.cursor_revision, Some(38)); + let (_, version_marker) = decode_manual_transition_continuation_token( + same_marker_checkpointed + .report + .continuation_token + .as_deref() + .expect("same-marker version checkpoint should persist a cursor"), + ) + .expect("same-marker version cursor should decode"); + assert_eq!(version_marker.as_deref(), Some("opaque-next-version")); + create_test_bucket(&ecstore, &bucket).await; let lifecycle_xml = format!( r#" @@ -9210,6 +9276,7 @@ mod tests { assert_eq!(checkpointed.report.scanned, 1000); assert_eq!(checkpointed.report.eligible, 1000); assert_eq!(checkpointed.report.dry_run_eligible, 1000); + assert_eq!(checkpointed.cursor_revision, Some(1000)); let token = checkpointed .report .continuation_token diff --git a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs new file mode 100644 index 000000000..606b5f536 --- /dev/null +++ b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs @@ -0,0 +1,1161 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeMap; + +use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::{ + bucket_lifecycle_ops::{ + ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token, + }, + manual_transition_job, tier_delete_journal, transition_transaction, +}; +use crate::error::{Error, Result}; + +pub(crate) const ILM_META_PREFIX: &str = "ilm"; +const ILM_META_OBJECT_PREFIX: &str = "ilm/"; +const MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE: usize = 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DurableIlmRecordKind { + TierDeleteJournal, + TransitionTransaction, + ManualTransitionJob, + ManualTransitionScope, + ManualTransitionTask, + ManualTransitionWorkerResult, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DurableIlmNamespace { + pub(crate) name: &'static str, + pub(crate) prefix: &'static str, + pub(crate) max_record_size: usize, + kind: DurableIlmRecordKind, +} + +pub(crate) const TIER_DELETE_JOURNAL_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "tier-delete-journal", + prefix: "ilm/tier-delete-journal/", + max_record_size: 64 * 1024, + kind: DurableIlmRecordKind::TierDeleteJournal, +}; +pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "transition-transaction", + prefix: "ilm/transition-transactions/records", + max_record_size: transition_transaction::MAX_TRANSITION_TRANSACTION_SIZE, + kind: DurableIlmRecordKind::TransitionTransaction, +}; +pub(crate) const MANUAL_TRANSITION_JOB_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "manual-transition-job", + prefix: "ilm/manual-transition/jobs", + max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE, + kind: DurableIlmRecordKind::ManualTransitionJob, +}; +pub(crate) const MANUAL_TRANSITION_SCOPE_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "manual-transition-scope", + prefix: "ilm/manual-transition/scopes", + max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE, + kind: DurableIlmRecordKind::ManualTransitionScope, +}; +pub(crate) const MANUAL_TRANSITION_TASK_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "manual-transition-task", + prefix: "ilm/manual-transition/tasks", + max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE, + kind: DurableIlmRecordKind::ManualTransitionTask, +}; +pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "manual-transition-worker-result", + prefix: "ilm/manual-transition/results", + max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE, + kind: DurableIlmRecordKind::ManualTransitionWorkerResult, +}; + +pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 6] = [ + TIER_DELETE_JOURNAL_NAMESPACE, + TRANSITION_TRANSACTION_NAMESPACE, + MANUAL_TRANSITION_JOB_NAMESPACE, + MANUAL_TRANSITION_SCOPE_NAMESPACE, + MANUAL_TRANSITION_TASK_NAMESPACE, + MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE, +]; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ValidatedDurableIlmRecord { + pub(crate) namespace: &'static str, + pub(crate) id_kind: &'static str, + pub(crate) id: String, + pub(crate) checkpoint: DurableIlmRecordCheckpoint, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManualTransitionJobProgressCheckpoint { + report: ManualTransitionRunReport, + queue_snapshot: ManualTransitionQueueSnapshot, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManualTransitionJobProgressProof { + scope_sha256: String, + scanned: u64, + eligible: u64, + enqueued: u64, + dry_run_eligible: u64, + skipped_not_transition: u64, + skipped_tier: u64, + skipped_delete_marker: u64, + skipped_directory: u64, + skipped_replication: u64, + skipped_already_transitioned: u64, + skipped_already_in_flight: u64, + skipped_queue_full: u64, + skipped_queue_closed: u64, + skipped_queue_timeout: u64, + transition_completed: u64, + transition_failed: u64, + tier_failure: u64, + tier_failure_by_reason: BTreeMap, + lifecycle_config_found: bool, + truncated_by_limit: bool, + truncated_by_duration: bool, + cancelled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + continuation_token_sha256: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cursor_marker: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cursor_revision: Option, + queue_snapshot: ManualTransitionQueueSnapshot, +} + +impl ValidatedDurableIlmRecord { + pub(crate) fn context(&self) -> String { + format!("namespace `{}` {} `{}`", self.namespace, self.id_kind, self.id) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub(crate) enum DurableIlmRecordCheckpoint { + TierDeleteJournal { + content_sha256: String, + identity_sha256: String, + committed: bool, + }, + TransitionTransaction { + content_sha256: String, + identity_sha256: String, + remote_version_sha256: String, + remote_version_known: bool, + revision: u64, + state: transition_transaction::TransitionTransactionState, + }, + ManualTransitionJob { + content_sha256: String, + identity_sha256: String, + updated_at_unix_nanos: i64, + state: manual_transition_job::ManualTransitionJobState, + scan_completed: bool, + cancel_requested: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + progress: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + progress_proof: Option>, + }, + ManualTransitionScope { + content_sha256: String, + identity_sha256: String, + updated_at_unix_nanos: i64, + }, + ManualTransitionTask { + content_sha256: String, + }, + ManualTransitionWorkerResult { + content_sha256: String, + }, +} + +impl DurableIlmRecordCheckpoint { + pub(crate) fn content_sha256(&self) -> &str { + match self { + Self::TierDeleteJournal { content_sha256, .. } + | Self::TransitionTransaction { content_sha256, .. } + | Self::ManualTransitionJob { content_sha256, .. } + | Self::ManualTransitionScope { content_sha256, .. } + | Self::ManualTransitionTask { content_sha256 } + | Self::ManualTransitionWorkerResult { content_sha256 } => content_sha256, + } + } + + pub(crate) fn compacted(&self) -> Result { + let mut checkpoint = self.clone(); + if let Self::ManualTransitionJob { + progress, + progress_proof, + .. + } = &mut checkpoint + { + match (progress.take(), progress_proof.take()) { + (Some(progress), None) => { + *progress_proof = Some(Box::new(ManualTransitionJobProgressProof::new( + &progress.report, + &progress.queue_snapshot, + None, + )?)); + } + (None, Some(proof)) if proof.is_valid() => *progress_proof = Some(proof), + (None, None) => {} + _ => return Err(Error::other("durable ILM manual transition checkpoint is invalid")), + } + } + Ok(checkpoint) + } + + pub(crate) fn validate_successor(&self, next: &Self) -> Result<()> { + if self == next { + if let Self::ManualTransitionJob { + progress, + progress_proof, + .. + } = self + && !manual_job_progress_checkpoint_is_valid(progress.as_deref(), progress_proof.as_deref()) + { + return Err(Error::other("durable ILM manual transition checkpoint is invalid")); + } + return Ok(()); + } + + let valid = match (self, next) { + ( + Self::TierDeleteJournal { + identity_sha256: previous_identity, + committed: previous_committed, + .. + }, + Self::TierDeleteJournal { + identity_sha256: next_identity, + committed: next_committed, + .. + }, + ) => { + previous_identity == next_identity + && (previous_committed == next_committed || (!previous_committed && *next_committed)) + } + ( + Self::TransitionTransaction { + identity_sha256: previous_identity, + remote_version_sha256: previous_remote_version, + remote_version_known: previous_remote_version_known, + revision: previous_revision, + state: previous_state, + .. + }, + Self::TransitionTransaction { + identity_sha256: next_identity, + remote_version_sha256: next_remote_version, + revision: next_revision, + state: next_state, + .. + }, + ) => { + previous_identity == next_identity + && transition_state_distance(*previous_state, *next_state) + .and_then(|distance| previous_revision.checked_add(distance)) + .is_some_and(|expected_revision| *next_revision == expected_revision) + && (!previous_remote_version_known || previous_remote_version == next_remote_version) + } + ( + Self::ManualTransitionJob { + content_sha256: previous_content, + identity_sha256: previous_identity, + updated_at_unix_nanos: previous_updated_at, + state: previous_state, + scan_completed: previous_scan_completed, + cancel_requested: previous_cancel_requested, + progress: previous_progress, + progress_proof: previous_progress_proof, + .. + }, + Self::ManualTransitionJob { + content_sha256: next_content, + identity_sha256: next_identity, + updated_at_unix_nanos: next_updated_at, + state: next_state, + scan_completed: next_scan_completed, + cancel_requested: next_cancel_requested, + progress: next_progress, + progress_proof: next_progress_proof, + .. + }, + ) => { + let same_generation = previous_content == next_content + && previous_identity == next_identity + && previous_updated_at == next_updated_at + && previous_state == next_state + && previous_scan_completed == next_scan_completed + && previous_cancel_requested == next_cancel_requested + && manual_job_progress_equivalent( + previous_progress.as_deref(), + previous_progress_proof.as_deref(), + next_progress.as_deref(), + next_progress_proof.as_deref(), + ); + same_generation + || (previous_identity == next_identity + && next_updated_at > previous_updated_at + && manual_job_state_reaches(*previous_state, *next_state) + && (!previous_scan_completed || *next_scan_completed) + && (!previous_cancel_requested || *next_cancel_requested) + && manual_job_progress_reaches( + previous_progress.as_deref(), + previous_progress_proof.as_deref(), + next_progress.as_deref(), + next_progress_proof.as_deref(), + *next_scan_completed, + )) + } + ( + Self::ManualTransitionScope { + identity_sha256: previous_identity, + updated_at_unix_nanos: previous_updated_at, + .. + }, + Self::ManualTransitionScope { + identity_sha256: next_identity, + updated_at_unix_nanos: next_updated_at, + .. + }, + ) => previous_identity == next_identity && next_updated_at > previous_updated_at, + _ => false, + }; + + if valid { + Ok(()) + } else { + Err(Error::other("durable ILM record generation is not a monotonic successor")) + } + } +} + +fn transition_state_distance( + from: transition_transaction::TransitionTransactionState, + to: transition_transaction::TransitionTransactionState, +) -> Option { + use transition_transaction::TransitionTransactionState::{ + AbortedNoRemote, CleanupPending, Committed, LocalCommitStarted, UploadOutcomeUnknown, UploadStarted, Uploaded, + }; + + match (from, to) { + (UploadStarted, UploadOutcomeUnknown | AbortedNoRemote | Uploaded) => Some(1), + (UploadStarted, LocalCommitStarted | CleanupPending) => Some(2), + (UploadStarted, Committed) => Some(3), + (UploadOutcomeUnknown, Uploaded | CleanupPending) => Some(1), + (UploadOutcomeUnknown, LocalCommitStarted) => Some(2), + (UploadOutcomeUnknown, Committed) => Some(3), + (Uploaded, LocalCommitStarted | CleanupPending) => Some(1), + (Uploaded, Committed) => Some(2), + (LocalCommitStarted, Committed | CleanupPending) => Some(1), + _ => None, + } +} + +fn manual_job_state_reaches( + from: manual_transition_job::ManualTransitionJobState, + to: manual_transition_job::ManualTransitionJobState, +) -> bool { + from == to || from == manual_transition_job::ManualTransitionJobState::Running +} + +impl ManualTransitionJobProgressProof { + fn new( + report: &ManualTransitionRunReport, + queue_snapshot: &ManualTransitionQueueSnapshot, + cursor_revision: Option, + ) -> Result { + let cursor_marker = match report.continuation_token.as_deref() { + Some(token) => { + let marker = decode_manual_transition_continuation_token(token)? + .0 + .ok_or_else(|| Error::other("durable ILM manual transition cursor marker is missing"))?; + (marker.len() <= MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE).then_some(marker) + } + None => None, + }; + if !manual_job_worker_results_are_valid(report) + || !manual_job_queue_snapshot_is_valid(queue_snapshot) + || (report.continuation_token.is_some() && cursor_revision == Some(0)) + { + return Err(Error::other("durable ILM manual transition progress is invalid")); + } + Ok(Self { + scope_sha256: checkpoint_hash(&(report.bucket.as_str(), report.prefix.as_str(), &report.tier, report.dry_run))?, + scanned: report.scanned, + eligible: report.eligible, + enqueued: report.enqueued, + dry_run_eligible: report.dry_run_eligible, + skipped_not_transition: report.skipped_not_transition, + skipped_tier: report.skipped_tier, + skipped_delete_marker: report.skipped_delete_marker, + skipped_directory: report.skipped_directory, + skipped_replication: report.skipped_replication, + skipped_already_transitioned: report.skipped_already_transitioned, + skipped_already_in_flight: report.skipped_already_in_flight, + skipped_queue_full: report.skipped_queue_full, + skipped_queue_closed: report.skipped_queue_closed, + skipped_queue_timeout: report.skipped_queue_timeout, + transition_completed: report.transition_completed, + transition_failed: report.transition_failed, + tier_failure: report.tier_failure, + tier_failure_by_reason: report.tier_failure_by_reason.clone(), + lifecycle_config_found: report.lifecycle_config_found, + truncated_by_limit: report.truncated_by_limit, + truncated_by_duration: report.truncated_by_duration, + cancelled: report.cancelled, + continuation_token_sha256: report + .continuation_token + .as_deref() + .map(|token| hex_sha256(token.as_bytes(), ToOwned::to_owned)), + cursor_marker, + cursor_revision, + queue_snapshot: *queue_snapshot, + }) + } + + fn is_valid(&self) -> bool { + let reason_total = self + .tier_failure_by_reason + .values() + .try_fold(0u64, |total, count| total.checked_add(*count)); + is_sha256_checksum(&self.scope_sha256) + && self.continuation_token_sha256.as_deref().is_none_or(is_sha256_checksum) + && match (&self.continuation_token_sha256, &self.cursor_marker) { + (None, None) | (Some(_), None) => true, + (Some(_), Some(marker)) => !marker.is_empty() && marker.len() <= MANUAL_TRANSITION_CURSOR_MARKER_PROOF_MAX_SIZE, + (None, Some(_)) => false, + } + && !(self.continuation_token_sha256.is_some() && self.cursor_revision == Some(0)) + && self + .transition_completed + .checked_add(self.transition_failed) + .is_some_and(|total| total <= self.enqueued) + && self.transition_failed <= self.tier_failure + && reason_total.is_some_and(|total| total <= self.tier_failure) + && manual_job_queue_snapshot_is_valid(&self.queue_snapshot) + } +} + +fn manual_job_progress_checkpoint_is_valid( + progress: Option<&ManualTransitionJobProgressCheckpoint>, + proof: Option<&ManualTransitionJobProgressProof>, +) -> bool { + match (progress, proof) { + (Some(progress), None) => manual_job_progress_is_valid(progress), + (None, Some(proof)) => proof.is_valid(), + (None, None) => true, + (Some(_), Some(_)) => false, + } +} + +fn manual_job_progress_proof( + progress: Option<&ManualTransitionJobProgressCheckpoint>, + proof: Option<&ManualTransitionJobProgressProof>, +) -> Option { + match (progress, proof) { + (Some(progress), None) => ManualTransitionJobProgressProof::new(&progress.report, &progress.queue_snapshot, None).ok(), + (None, Some(proof)) if proof.is_valid() => Some(proof.clone()), + _ => None, + } +} + +fn manual_job_progress_equivalent( + previous: Option<&ManualTransitionJobProgressCheckpoint>, + previous_proof: Option<&ManualTransitionJobProgressProof>, + next: Option<&ManualTransitionJobProgressCheckpoint>, + next_proof: Option<&ManualTransitionJobProgressProof>, +) -> bool { + if !manual_job_progress_checkpoint_is_valid(previous, previous_proof) + || !manual_job_progress_checkpoint_is_valid(next, next_proof) + { + return false; + } + match ( + manual_job_progress_proof(previous, previous_proof), + manual_job_progress_proof(next, next_proof), + ) { + (Some(previous), Some(next)) => previous == next, + (None, None) => true, + _ => false, + } +} + +fn manual_job_progress_reaches( + previous: Option<&ManualTransitionJobProgressCheckpoint>, + previous_proof: Option<&ManualTransitionJobProgressProof>, + next: Option<&ManualTransitionJobProgressCheckpoint>, + next_proof: Option<&ManualTransitionJobProgressProof>, + next_scan_completed: bool, +) -> bool { + if previous.is_none() && previous_proof.is_none() { + return (next.is_some() || next_proof.is_some()) && manual_job_progress_checkpoint_is_valid(next, next_proof); + } + let (Some(previous_compact), Some(next_compact)) = ( + manual_job_progress_proof(previous, previous_proof), + manual_job_progress_proof(next, next_proof), + ) else { + return false; + }; + + macro_rules! counters_do_not_regress { + ($($field:ident),+ $(,)?) => { + $(previous_compact.$field <= next_compact.$field)&&+ + }; + } + + let counters_monotonic = counters_do_not_regress!( + scanned, + eligible, + enqueued, + dry_run_eligible, + skipped_not_transition, + skipped_tier, + skipped_delete_marker, + skipped_directory, + skipped_replication, + skipped_already_transitioned, + skipped_already_in_flight, + skipped_queue_full, + skipped_queue_closed, + skipped_queue_timeout, + transition_completed, + transition_failed, + tier_failure, + ); + let failure_reasons_monotonic = previous_compact + .tier_failure_by_reason + .iter() + .all(|(reason, previous_count)| { + next_compact + .tier_failure_by_reason + .get(reason) + .is_some_and(|next_count| next_count >= previous_count) + }); + let flags_monotonic = (!previous_compact.lifecycle_config_found || next_compact.lifecycle_config_found) + && (!previous_compact.truncated_by_limit || next_compact.truncated_by_limit) + && (!previous_compact.truncated_by_duration || next_compact.truncated_by_duration) + && (!previous_compact.cancelled || next_compact.cancelled); + let cursor_monotonic = manual_job_cursor_reaches( + &previous_compact, + &next_compact, + previous.map(|progress| &progress.report), + next.map(|progress| &progress.report), + next_scan_completed, + ); + + previous_compact.scope_sha256 == next_compact.scope_sha256 + && counters_monotonic + && failure_reasons_monotonic + && flags_monotonic + && cursor_monotonic + && previous_compact.is_valid() + && next_compact.is_valid() +} + +fn manual_job_progress_is_valid(progress: &ManualTransitionJobProgressCheckpoint) -> bool { + manual_job_worker_results_are_valid(&progress.report) + && manual_job_queue_snapshot_is_valid(&progress.queue_snapshot) + && manual_job_cursor_is_valid(progress.report.continuation_token.as_deref()) +} + +fn manual_job_worker_results_are_valid(report: &ManualTransitionRunReport) -> bool { + let reason_total = report + .tier_failure_by_reason + .values() + .try_fold(0u64, |total, count| total.checked_add(*count)); + report + .transition_completed + .checked_add(report.transition_failed) + .is_some_and(|total| total <= report.enqueued) + && report.transition_failed <= report.tier_failure + && reason_total.is_some_and(|total| total <= report.tier_failure) +} + +fn manual_job_cursor_reaches( + previous: &ManualTransitionJobProgressProof, + next: &ManualTransitionJobProgressProof, + previous_legacy: Option<&ManualTransitionRunReport>, + next_legacy: Option<&ManualTransitionRunReport>, + next_scan_completed: bool, +) -> bool { + if previous.continuation_token_sha256 == next.continuation_token_sha256 { + return previous.cursor_marker == next.cursor_marker && previous.cursor_revision == next.cursor_revision; + } + match (&previous.continuation_token_sha256, &next.continuation_token_sha256) { + (None, Some(_)) => { + next.scanned > previous.scanned + && (manual_job_cursor_revision_advances(previous.cursor_revision, next.cursor_revision) + || (previous.cursor_revision.is_none() && next.cursor_revision.is_none())) + } + (Some(_), None) => next_scan_completed, + (Some(_), Some(_)) if next.scanned > previous.scanned => { + manual_job_cursor_revision_advances(previous.cursor_revision, next.cursor_revision) + || manual_job_legacy_cursor_reaches(previous, next, previous_legacy, next_legacy) + } + _ => false, + } +} + +fn manual_job_cursor_revision_advances(previous: Option, next: Option) -> bool { + match (previous, next) { + (Some(previous), Some(next)) => next > previous, + (None, Some(next)) => next > 0, + _ => false, + } +} + +fn manual_job_legacy_cursor_reaches( + previous_proof: &ManualTransitionJobProgressProof, + next_proof: &ManualTransitionJobProgressProof, + previous_legacy: Option<&ManualTransitionRunReport>, + next_legacy: Option<&ManualTransitionRunReport>, +) -> bool { + if let (Some(previous_marker), Some(next_marker)) = (&previous_proof.cursor_marker, &next_proof.cursor_marker) { + return next_marker > previous_marker; + } + let (Some(previous_token), Some(next_token)) = ( + previous_legacy.and_then(|report| report.continuation_token.as_deref()), + next_legacy.and_then(|report| report.continuation_token.as_deref()), + ) else { + return false; + }; + let (Ok((Some(previous_marker), _)), Ok((Some(next_marker), _))) = ( + decode_manual_transition_continuation_token(previous_token), + decode_manual_transition_continuation_token(next_token), + ) else { + return false; + }; + next_marker > previous_marker +} + +fn manual_job_cursor_is_valid(token: Option<&str>) -> bool { + let Some(token) = token else { + return true; + }; + matches!(decode_manual_transition_continuation_token(token), Ok((Some(_), _))) +} + +fn manual_job_queue_snapshot_is_valid(snapshot: &ManualTransitionQueueSnapshot) -> bool { + (snapshot.queue_capacity > 0 || snapshot.queued == 0) + && (snapshot.queue_capacity == 0 || snapshot.queued <= snapshot.queue_capacity) + && (snapshot.workers > 0 || snapshot.active == 0) + && (snapshot.workers == 0 || snapshot.active <= snapshot.workers) +} + +fn checkpoint_hash(value: &T) -> Result { + let encoded = serde_json::to_vec(value).map_err(Error::other)?; + Ok(hex_sha256(&encoded, ToOwned::to_owned)) +} + +fn path_is_in_namespace(path: &str, namespace: &DurableIlmNamespace) -> bool { + let Some(suffix) = path.strip_prefix(namespace.prefix) else { + return false; + }; + if namespace.prefix.ends_with('/') { + !suffix.is_empty() + } else { + suffix.starts_with('/') && suffix.len() > 1 + } +} + +pub(crate) fn classify_durable_ilm_record(path: &str) -> Result> { + if path != ILM_META_PREFIX && !path.starts_with(ILM_META_OBJECT_PREFIX) { + return Ok(None); + } + + DURABLE_ILM_NAMESPACES + .iter() + .find(|namespace| path_is_in_namespace(path, namespace)) + .map(Some) + .ok_or_else(|| Error::other(format!("unregistered durable ILM namespace for path `{path}`"))) +} + +fn parse_manual_sharded_record(path: &str, prefix: &str) -> Result<(Uuid, String)> { + let suffix = path + .strip_prefix(prefix) + .and_then(|suffix| suffix.strip_prefix('/')) + .ok_or_else(|| Error::other("manual transition record path has wrong prefix"))?; + let mut parts = suffix.split('/'); + let first = parts + .next() + .ok_or_else(|| Error::other("manual transition record first shard is missing"))?; + let second = parts + .next() + .ok_or_else(|| Error::other("manual transition record second shard is missing"))?; + let job_key = parts + .next() + .ok_or_else(|| Error::other("manual transition record job id is missing"))?; + let task_key = parts + .next() + .and_then(|file| file.strip_suffix(".json")) + .ok_or_else(|| Error::other("manual transition record task key is missing"))?; + if parts.next().is_some() + || job_key.len() != 32 + || first != &job_key[..2] + || second != &job_key[2..4] + || !job_key + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err(Error::other("manual transition record job id or shards are invalid")); + } + let job_id = Uuid::parse_str(job_key).map_err(|_| Error::other("manual transition record job id is invalid"))?; + Ok((job_id, task_key.to_string())) +} + +pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result { + let namespace = + classify_durable_ilm_record(path)?.ok_or_else(|| Error::other(format!("path `{path}` is not a durable ILM record")))?; + if data.len() > namespace.max_record_size { + return Err(Error::other(format!( + "durable ILM record exceeds {} byte limit", + namespace.max_record_size + ))); + } + + let content_sha256 = hex_sha256(data, ToOwned::to_owned); + let (id_kind, id, checkpoint) = match namespace.kind { + DurableIlmRecordKind::TierDeleteJournal => { + let entry = tier_delete_journal::decode_tier_delete_journal_entry(data)?; + if tier_delete_journal::tier_delete_journal_object_name(&entry) != path { + return Err(Error::other("tier delete journal content does not match its path")); + } + let operation_id = path + .strip_prefix(namespace.prefix) + .and_then(|suffix| suffix.strip_suffix(".json")) + .ok_or_else(|| Error::other("tier delete journal path is invalid"))?; + let identity_sha256 = checkpoint_hash(&( + &entry.obj_name, + &entry.version_id, + &entry.tier_name, + entry.backend_identity, + entry.version_id_exact, + entry.version_state, + &entry.source, + ))?; + ( + "operation_id", + operation_id.to_string(), + DurableIlmRecordCheckpoint::TierDeleteJournal { + content_sha256, + identity_sha256, + committed: entry.state == super::tier_sweeper::TierDeleteJournalState::Committed, + }, + ) + } + DurableIlmRecordKind::TransitionTransaction => { + let transaction = transition_transaction::decode_transition_transaction_record(path, data) + .map_err(|err| Error::other(err.to_string()))?; + let identity_sha256 = checkpoint_hash(&( + transaction.deployment_id, + transaction.transaction_id, + transaction.owner_epoch, + transaction.write_id, + &transaction.source, + &transaction.tier_name, + transaction.backend_fingerprint, + &transaction.remote_object, + transaction.not_after_unix_nanos, + ))?; + let remote_version_sha256 = checkpoint_hash(&transaction.remote_version)?; + ( + "transaction_id", + transaction.transaction_id.to_string(), + DurableIlmRecordCheckpoint::TransitionTransaction { + content_sha256, + identity_sha256, + remote_version_sha256, + remote_version_known: !transaction.remote_version.is_unknown(), + revision: transaction.revision, + state: transaction.state, + }, + ) + } + DurableIlmRecordKind::ManualTransitionJob => { + let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path) + .map_err(|err| Error::other(err.to_string()))?; + let canonical = manual_transition_job::manual_transition_job_record_object_name(job_id) + .map_err(|err| Error::other(err.to_string()))?; + if canonical != path { + return Err(Error::other("manual transition job path is not canonical")); + } + let job = manual_transition_job::ManualTransitionJobRecord::decode(job_id, data) + .map_err(|err| Error::other(err.to_string()))?; + let identity_sha256 = checkpoint_hash(&( + job.job_id, + &job.scope_key, + &job.bucket, + &job.prefix, + &job.tier, + job.dry_run, + job.max_objects, + job.max_duration, + job.created_at_unix_nanos, + ))?; + let progress_proof = ManualTransitionJobProgressProof::new(&job.report, &job.queue_snapshot, job.cursor_revision)?; + let updated_at_unix_nanos = i64::try_from(job.updated_at_unix_nanos) + .map_err(|_| Error::other("manual transition job updated_at exceeds durable ILM checkpoint range"))?; + ( + "job_id", + job_id.to_string(), + DurableIlmRecordCheckpoint::ManualTransitionJob { + content_sha256, + identity_sha256, + updated_at_unix_nanos, + state: job.state, + scan_completed: job.scan_completed, + cancel_requested: job.cancel_requested, + progress: None, + progress_proof: Some(Box::new(progress_proof)), + }, + ) + } + DurableIlmRecordKind::ManualTransitionScope => { + let admission: manual_transition_job::ManualTransitionScopeAdmission = + serde_json::from_slice(data).map_err(Error::other)?; + admission.validate().map_err(|err| Error::other(err.to_string()))?; + let canonical = manual_transition_job::manual_transition_scope_record_object_name(&admission.scope_key) + .map_err(|err| Error::other(err.to_string()))?; + if canonical != path { + return Err(Error::other("manual transition scope content does not match its path")); + } + let identity_sha256 = checkpoint_hash(&( + &admission.schema, + &admission.scope_key, + admission.job_id, + &admission.bucket, + &admission.prefix, + &admission.tier, + admission.dry_run, + ))?; + let updated_at_unix_nanos = i64::try_from(admission.updated_at_unix_nanos) + .map_err(|_| Error::other("manual transition scope updated_at exceeds durable ILM checkpoint range"))?; + ( + "job_id", + admission.job_id.to_string(), + DurableIlmRecordCheckpoint::ManualTransitionScope { + content_sha256, + identity_sha256, + updated_at_unix_nanos, + }, + ) + } + DurableIlmRecordKind::ManualTransitionTask => { + let (job_id, task_key) = parse_manual_sharded_record(path, namespace.prefix)?; + let canonical = manual_transition_job::manual_transition_task_object_name(job_id, &task_key) + .map_err(|err| Error::other(err.to_string()))?; + if canonical != path { + return Err(Error::other("manual transition task path is not canonical")); + } + manual_transition_job::ManualTransitionTaskRecord::decode(job_id, &task_key, data) + .map_err(|err| Error::other(err.to_string()))?; + ( + "job_id", + job_id.to_string(), + DurableIlmRecordCheckpoint::ManualTransitionTask { content_sha256 }, + ) + } + DurableIlmRecordKind::ManualTransitionWorkerResult => { + let (job_id, task_key) = parse_manual_sharded_record(path, namespace.prefix)?; + let canonical = manual_transition_job::manual_transition_worker_result_object_name(job_id, &task_key) + .map_err(|err| Error::other(err.to_string()))?; + if canonical != path { + return Err(Error::other("manual transition worker result path is not canonical")); + } + manual_transition_job::ManualTransitionWorkerResultRecord::decode(job_id, &task_key, data) + .map_err(|err| Error::other(err.to_string()))?; + ( + "job_id", + job_id.to_string(), + DurableIlmRecordCheckpoint::ManualTransitionWorkerResult { content_sha256 }, + ) + } + }; + + Ok(ValidatedDurableIlmRecord { + namespace: namespace.name, + id_kind, + id, + checkpoint, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn try_manual_job_checkpoint(job: &manual_transition_job::ManualTransitionJobRecord) -> Result { + let path = + manual_transition_job::manual_transition_job_record_object_name(job.job_id).expect("manual job path should build"); + let encoded = job.encode().expect("manual job should encode"); + Ok(validate_durable_ilm_record(&path, &encoded)?.checkpoint) + } + + fn manual_job_checkpoint(job: &manual_transition_job::ManualTransitionJobRecord) -> DurableIlmRecordCheckpoint { + try_manual_job_checkpoint(job).expect("manual job checkpoint should validate") + } + + fn continuation_token_with_version(marker: &str, version_marker: Option<&str>) -> String { + let encoded = serde_json::to_vec(&serde_json::json!({ "marker": marker, "version_marker": version_marker })) + .expect("continuation token should encode"); + base64_simd::URL_SAFE_NO_PAD.encode_to_string(&encoded) + } + + fn continuation_token(marker: &str) -> String { + continuation_token_with_version(marker, None) + } + + #[test] + fn unknown_ilm_record_requires_namespace_registration() { + let err = classify_durable_ilm_record("ilm/future-durable/jobs/one.json") + .expect_err("unknown durable ILM path must fail closed"); + + assert!(err.to_string().contains("ilm/future-durable/jobs/one.json")); + } + + #[test] + fn durable_ilm_registry_has_unique_non_overlapping_prefixes() { + for (index, namespace) in DURABLE_ILM_NAMESPACES.iter().enumerate() { + assert!(namespace.prefix.starts_with(ILM_META_OBJECT_PREFIX)); + assert!(namespace.max_record_size > 0); + for other in DURABLE_ILM_NAMESPACES.iter().skip(index + 1) { + assert_ne!(namespace.prefix, other.prefix); + assert!(!path_is_in_namespace(namespace.prefix, other)); + assert!(!path_is_in_namespace(other.prefix, namespace)); + } + } + } + + #[test] + fn manual_transition_job_checkpoint_compacts_legacy_progress_compatibly() { + let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default(); + let mut job = + manual_transition_job::ManualTransitionJobRecord::new(Uuid::new_v4(), "legacy-checkpoint-bucket", &options, "owner"); + job.cursor_revision = None; + job.updated_at_unix_nanos += 1; + job.report.scanned = 1; + job.report.continuation_token = Some(continuation_token("logs/a")); + let compact = manual_job_checkpoint(&job); + let mut legacy = compact.clone(); + let DurableIlmRecordCheckpoint::ManualTransitionJob { + progress, + progress_proof, + .. + } = &mut legacy + else { + panic!("manual job should produce a manual checkpoint"); + }; + *progress = Some(Box::new(ManualTransitionJobProgressCheckpoint { + report: job.report.clone(), + queue_snapshot: job.queue_snapshot, + })); + *progress_proof = None; + + compact + .validate_successor(&legacy) + .expect("bounded checkpoints should accept the same legacy generation"); + legacy + .validate_successor(&compact) + .expect("legacy checkpoints should accept the same bounded generation"); + assert_eq!(legacy.compacted().expect("legacy checkpoint should compact"), compact); + } + + #[test] + fn manual_transition_job_checkpoint_rejects_timestamp_outside_wire_range() { + let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default(); + let mut job = manual_transition_job::ManualTransitionJobRecord::new( + Uuid::new_v4(), + "checkpoint-timestamp-bucket", + &options, + "owner", + ); + job.updated_at_unix_nanos = i128::from(i64::MAX) + 1; + + let err = try_manual_job_checkpoint(&job).expect_err("out-of-range checkpoint timestamp must fail closed"); + + assert!(err.to_string().contains("updated_at exceeds durable ILM checkpoint range")); + } + + #[test] + fn manual_transition_scope_checkpoint_rejects_timestamp_outside_wire_range() { + let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default(); + let job = manual_transition_job::ManualTransitionJobRecord::new( + Uuid::new_v4(), + "scope-checkpoint-timestamp-bucket", + &options, + "owner", + ); + let mut admission = manual_transition_job::ManualTransitionScopeAdmission::from_job(&job); + admission.updated_at_unix_nanos = i128::from(i64::MAX) + 1; + let path = manual_transition_job::manual_transition_scope_record_object_name(&admission.scope_key) + .expect("manual transition scope path should build"); + let encoded = serde_json::to_vec(&admission).expect("manual transition scope should encode"); + + let err = + validate_durable_ilm_record(&path, &encoded).expect_err("out-of-range scope checkpoint timestamp must fail closed"); + + assert!(err.to_string().contains("updated_at exceeds durable ILM checkpoint range")); + } + + #[test] + fn manual_transition_job_checkpoint_rejects_progress_poison() { + let options = super::super::bucket_lifecycle_ops::ManualTransitionRunOptions::default(); + let mut initial = + manual_transition_job::ManualTransitionJobRecord::new(Uuid::new_v4(), "manual-checkpoint-bucket", &options, "owner"); + let initial_checkpoint = manual_job_checkpoint(&initial); + let mut first_page = initial.report.clone(); + first_page.scanned = 1; + first_page.continuation_token = Some(continuation_token("logs/a")); + initial.update_running_progress(first_page, ManualTransitionQueueSnapshot::default()); + let first_page_checkpoint = manual_job_checkpoint(&initial); + initial_checkpoint + .validate_successor(&first_page_checkpoint) + .expect("the first durable cursor should advance from no cursor"); + + let mut legacy_checkpoint = initial_checkpoint; + let DurableIlmRecordCheckpoint::ManualTransitionJob { + progress, + progress_proof, + .. + } = &mut legacy_checkpoint + else { + panic!("manual job should produce a manual checkpoint"); + }; + *progress = None; + *progress_proof = None; + legacy_checkpoint + .validate_successor(&first_page_checkpoint) + .expect("legacy checkpoints should upgrade to validated progress"); + + let mut previous = initial; + let mut previous_report = previous.report.clone(); + previous_report.scanned = 10; + previous_report.eligible = 8; + previous_report.enqueued = 2; + previous_report.continuation_token = Some(continuation_token("logs/b")); + let previous_queue = ManualTransitionQueueSnapshot { + queue_capacity: 10, + queued: 1, + active: 1, + workers: 2, + queue_full: 2, + queue_send_timeout: 1, + ..Default::default() + }; + previous.update_running_progress(previous_report, previous_queue); + previous.report.transition_completed = 1; + let previous_checkpoint = manual_job_checkpoint(&previous); + + let mut next = previous.clone(); + let mut next_report = next.report.clone(); + next_report.scanned = 11; + next_report.eligible = 9; + next_report.continuation_token = Some(continuation_token("logs/c")); + let mut next_queue = next.queue_snapshot; + next_queue.queued = 0; + next_queue.active = 0; + next_queue.queue_full = 3; + next.update_running_progress(next_report, next_queue); + next.report.transition_completed = 2; + let next_checkpoint = manual_job_checkpoint(&next); + previous_checkpoint + .validate_successor(&next_checkpoint) + .expect("forward job progress should validate"); + + let mut counter_rollback = next.clone(); + counter_rollback.updated_at_unix_nanos += 1; + counter_rollback.report.scanned = 9; + assert!( + previous_checkpoint + .validate_successor(&manual_job_checkpoint(&counter_rollback)) + .is_err() + ); + + let mut cursor_rollback = previous.clone(); + cursor_rollback.updated_at_unix_nanos += 1; + cursor_rollback.report.scanned += 1; + cursor_rollback.report.continuation_token = Some(continuation_token("logs/a")); + assert!( + previous_checkpoint + .validate_successor(&manual_job_checkpoint(&cursor_rollback)) + .is_err() + ); + + let mut same_marker_version_previous = previous.clone(); + let mut same_marker_report = same_marker_version_previous.report.clone(); + same_marker_report.continuation_token = Some(continuation_token_with_version("logs/b", Some("opaque-z-version"))); + same_marker_version_previous.update_running_progress(same_marker_report, same_marker_version_previous.queue_snapshot); + let same_marker_version_previous_checkpoint = manual_job_checkpoint(&same_marker_version_previous); + let mut same_marker_version_next = same_marker_version_previous.clone(); + let mut same_marker_next_report = same_marker_version_next.report.clone(); + same_marker_next_report.scanned += 1; + same_marker_next_report.continuation_token = Some(continuation_token_with_version("logs/b", Some("opaque-a-version"))); + same_marker_version_next.update_running_progress(same_marker_next_report, same_marker_version_next.queue_snapshot); + same_marker_version_previous_checkpoint + .validate_successor(&manual_job_checkpoint(&same_marker_version_next)) + .expect("producer cursor revision should prove same-marker version progress"); + + let mut same_marker_version_rollback = same_marker_version_previous.clone(); + same_marker_version_rollback.updated_at_unix_nanos += 1; + same_marker_version_rollback.report.scanned += 1; + same_marker_version_rollback.report.continuation_token = + Some(continuation_token_with_version("logs/b", Some("opaque-arbitrary-version"))); + assert!( + same_marker_version_previous_checkpoint + .validate_successor(&manual_job_checkpoint(&same_marker_version_rollback)) + .is_err(), + "a different opaque version marker without producer evidence must fail closed" + ); + + let mut worker_result_rollback = next.clone(); + worker_result_rollback.updated_at_unix_nanos += 1; + worker_result_rollback.report.transition_completed = 0; + assert!( + previous_checkpoint + .validate_successor(&manual_job_checkpoint(&worker_result_rollback)) + .is_err() + ); + + let mut worker_result_overflow = next.clone(); + worker_result_overflow.updated_at_unix_nanos += 1; + worker_result_overflow.report.enqueued = u64::MAX; + worker_result_overflow.report.transition_completed = u64::MAX; + worker_result_overflow.report.transition_failed = 1; + worker_result_overflow.report.tier_failure = 1; + assert!(try_manual_job_checkpoint(&worker_result_overflow).is_err()); + + let mut invalid_cursor = next.clone(); + invalid_cursor.updated_at_unix_nanos += 1; + invalid_cursor.report.continuation_token = Some("not-base64".to_string()); + assert!(try_manual_job_checkpoint(&invalid_cursor).is_err()); + + let mut queue_state_poison = next; + queue_state_poison.updated_at_unix_nanos += 1; + queue_state_poison.queue_snapshot.queued = queue_state_poison.queue_snapshot.queue_capacity + 1; + assert!(try_manual_job_checkpoint(&queue_state_poison).is_err()); + } +} diff --git a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs index b1fe45cad..36a0120d5 100644 --- a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs +++ b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs @@ -20,10 +20,16 @@ use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use uuid::Uuid; +#[cfg(test)] +use crate::bucket::lifecycle::bucket_lifecycle_ops::encode_manual_transition_continuation_token; use crate::bucket::lifecycle::bucket_lifecycle_ops::{ ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, }; use crate::bucket::lifecycle::config_boundary; +use crate::bucket::lifecycle::durable_namespace::{ + MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE, + MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE, +}; use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result as EcstoreResult}; use crate::object_api::ObjectOptions; @@ -34,10 +40,10 @@ use crate::store::ECStore; pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1"; pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1"; pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1"; -pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs"; -pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes"; -pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks"; -pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results"; +pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix; +pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix; +pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix; +pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix; pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024; pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024; pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024; @@ -195,6 +201,8 @@ pub struct ManualTransitionJobRecord { pub updated_at_unix_nanos: i128, #[serde(default, skip_serializing_if = "Option::is_none")] pub completed_at_unix_nanos: Option, + #[serde(default, skip_serializing)] + pub cursor_revision: Option, pub report: ManualTransitionRunReport, pub queue_snapshot: ManualTransitionQueueSnapshot, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -224,6 +232,7 @@ impl ManualTransitionJobRecord { created_at_unix_nanos: now, updated_at_unix_nanos: now, completed_at_unix_nanos: None, + cursor_revision: None, report: ManualTransitionRunReport { bucket: bucket.to_string(), prefix: options.prefix.clone(), @@ -238,7 +247,7 @@ impl ManualTransitionJobRecord { pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) { self.scan_completed = true; - self.report.merge_scan_report_preserving_worker(&report); + self.merge_scan_report(&report); self.queue_snapshot = queue_snapshot; self.error = None; self.mark_terminal_if_worker_drained(); @@ -316,7 +325,7 @@ impl ManualTransitionJobRecord { } } self.queue_snapshot = queue_snapshot; - self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos(); + self.advance_updated_at(); self.mark_terminal_if_worker_drained(); } @@ -359,14 +368,14 @@ impl ManualTransitionJobRecord { self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed); self.report.tier_failure_by_reason = scan_tier_failure_by_reason; self.queue_snapshot = queue_snapshot; - self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos(); + self.advance_updated_at(); self.mark_terminal_if_worker_drained(); true } pub fn mark_cancel_requested(&mut self) { self.cancel_requested = true; - self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos(); + self.advance_updated_at(); } pub fn claim_recovery_lease(&mut self, owner_id: impl Into, queue_snapshot: ManualTransitionQueueSnapshot) { @@ -380,7 +389,7 @@ impl ManualTransitionJobRecord { pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) { if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id { self.lease_expires_at_unix_nanos = 0; - self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos(); + self.advance_updated_at(); } } @@ -398,7 +407,7 @@ impl ManualTransitionJobRecord { pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) { let now = OffsetDateTime::now_utc().unix_timestamp_nanos(); - self.updated_at_unix_nanos = now; + self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now); self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now); self.queue_snapshot = queue_snapshot; } @@ -438,11 +447,16 @@ impl ManualTransitionJobRecord { pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) { if self.state == ManualTransitionJobState::Running { - self.report.merge_scan_report_preserving_worker(&report); + self.merge_scan_report(&report); self.renew_lease(queue_snapshot); } } + fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) { + self.report.merge_scan_report_preserving_worker(report); + self.cursor_revision = manual_transition_cursor_revision(&self.report); + } + pub fn mark_unknown_if_unowned(&mut self) { if self.state == ManualTransitionJobState::Running { self.state = ManualTransitionJobState::Unknown; @@ -463,9 +477,13 @@ impl ManualTransitionJobRecord { } fn mark_updated_terminal(&mut self) { + self.advance_updated_at(); + self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos); + } + + fn advance_updated_at(&mut self) { let now = OffsetDateTime::now_utc().unix_timestamp_nanos(); - self.updated_at_unix_nanos = now; - self.completed_at_unix_nanos = Some(now); + self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now); } fn mark_terminal_if_worker_drained(&mut self) { @@ -543,6 +561,7 @@ impl ManualTransitionJobRecord { if job.state == ManualTransitionJobState::Cancelled && job.cancel_requested { job.report.cancelled = true; } + job.cursor_revision = manual_transition_cursor_revision(&job.report); job.validate()?; Ok(job) } @@ -1109,7 +1128,8 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result, job: &ManualTransitionJobRecord) -> EcstoreResult<()> { let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?; let data = job.encode().map_err(manual_transition_job_store_error)?; - config_boundary::save_config(api, &object, data).await + config_boundary::save_config(api.clone(), &object, data.clone()).await?; + api.record_durable_ilm_decommission_progress(&object, &data).await } pub async fn load_manual_transition_job_record(api: Arc, job_id: Uuid) -> EcstoreResult { @@ -1142,9 +1162,9 @@ pub async fn save_manual_transition_job_record_if_current( let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?; let data = job.encode().map_err(manual_transition_job_store_error)?; config_boundary::save_config_with_opts_quiet( - api, + api.clone(), &object, - data, + data.clone(), &ObjectOptions { max_parity: true, http_preconditions: Some(HTTPPreconditions { @@ -1154,7 +1174,8 @@ pub async fn save_manual_transition_job_record_if_current( ..Default::default() }, ) - .await + .await?; + api.record_durable_ilm_decommission_progress(&object, &data).await } /// Applies a job-record mutation with optimistic concurrency control. @@ -1592,9 +1613,9 @@ pub async fn save_manual_transition_scope_admission_if_absent( let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?; let data = serde_json::to_vec(admission).map_err(Error::other)?; config_boundary::save_config_with_opts( - api, + api.clone(), &object, - data, + data.clone(), &ObjectOptions { max_parity: true, http_preconditions: Some(HTTPPreconditions { @@ -1604,7 +1625,8 @@ pub async fn save_manual_transition_scope_admission_if_absent( ..Default::default() }, ) - .await + .await?; + api.record_durable_ilm_decommission_progress(&object, &data).await } pub async fn load_manual_transition_scope_admission( @@ -1642,9 +1664,9 @@ pub async fn save_manual_transition_scope_admission_if_current( let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?; let data = serde_json::to_vec(admission).map_err(Error::other)?; match config_boundary::save_config_with_opts( - api, + api.clone(), &object, - data, + data.clone(), &ObjectOptions { max_parity: true, http_preconditions: Some(HTTPPreconditions { @@ -1660,7 +1682,8 @@ pub async fn save_manual_transition_scope_admission_if_current( Err(Error::PreconditionFailed) } result => result, - } + }?; + api.record_durable_ilm_decommission_progress(&object, &data).await } pub async fn claim_manual_transition_scope_admission( @@ -1953,13 +1976,15 @@ pub async fn delete_manual_transition_scope_admission_if_current( job_id: Uuid, lease_id: Uuid, ) -> EcstoreResult { - let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await { - Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag, + let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await { + Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag), Ok(_) => return Ok(false), Err(Error::ConfigNotFound) => return Ok(true), Err(err) => return Err(err), }; let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?; + let data = serde_json::to_vec(&admission).map_err(Error::other)?; + api.record_durable_ilm_decommission_terminal(&object, &data).await?; match config_boundary::delete_config_if_match(api, &object, &etag).await { Ok(()) | Err(Error::ConfigNotFound) => Ok(true), Err(Error::PreconditionFailed) => Ok(false), @@ -1971,6 +1996,11 @@ fn manual_transition_job_store_error(err: ManualTransitionJobError) -> Error { Error::other(err) } +fn manual_transition_cursor_revision(report: &ManualTransitionRunReport) -> Option { + report.continuation_token.as_ref()?; + (report.scanned > 0).then_some(report.scanned) +} + pub fn manual_transition_scope_admission_lease_expired(admission: &ManualTransitionScopeAdmission) -> bool { OffsetDateTime::now_utc().unix_timestamp_nanos() > admission.lease_expires_at_unix_nanos } @@ -2210,6 +2240,76 @@ mod tests { ); } + #[test] + fn manual_transition_job_scan_progress_accumulates_resumed_checkpoint_counters() { + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + let first_token = + encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string())); + record.update_running_progress( + ManualTransitionRunReport { + bucket: "bucket".to_string(), + lifecycle_config_found: true, + scanned: 1000, + eligible: 800, + enqueued: 50, + dry_run_eligible: 10, + skipped_not_transition: 2, + skipped_tier: 3, + skipped_delete_marker: 4, + skipped_directory: 5, + skipped_replication: 6, + skipped_already_transitioned: 7, + skipped_already_in_flight: 8, + skipped_queue_full: 9, + skipped_queue_closed: 10, + skipped_queue_timeout: 11, + tier_failure: 12, + truncated_by_duration: true, + continuation_token: first_token, + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + + let next_token = + encode_manual_transition_continuation_token(Some("logs/page-b".to_string()), Some("version-b".to_string())); + record.update_running_progress( + ManualTransitionRunReport { + bucket: "bucket".to_string(), + scanned: 3, + eligible: 2, + enqueued: 1, + dry_run_eligible: 1, + skipped_not_transition: 1, + tier_failure: 1, + continuation_token: next_token.clone(), + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + + assert_eq!(record.report.scanned, 1003); + assert_eq!(record.report.eligible, 802); + assert_eq!(record.report.enqueued, 51); + assert_eq!(record.report.dry_run_eligible, 11); + assert_eq!(record.report.skipped_not_transition, 3); + assert_eq!(record.report.skipped_tier, 3); + assert_eq!(record.report.skipped_delete_marker, 4); + assert_eq!(record.report.skipped_directory, 5); + assert_eq!(record.report.skipped_replication, 6); + assert_eq!(record.report.skipped_already_transitioned, 7); + assert_eq!(record.report.skipped_already_in_flight, 8); + assert_eq!(record.report.skipped_queue_full, 9); + assert_eq!(record.report.skipped_queue_closed, 10); + assert_eq!(record.report.skipped_queue_timeout, 11); + assert_eq!(record.report.tier_failure, 13); + assert!(record.report.lifecycle_config_found); + assert!(record.report.truncated_by_duration); + assert_eq!(record.report.continuation_token, next_token); + assert_eq!(record.cursor_revision, Some(1003)); + } + #[test] fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() { let options = ManualTransitionRunOptions::default(); @@ -2577,6 +2677,98 @@ mod tests { assert!(decoded.report.tier_failure_by_reason.is_empty()); } + #[test] + fn manual_transition_job_record_derives_revision_from_legacy_cursor() { + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + let continuation_token = + encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string())); + record.update_running_progress( + ManualTransitionRunReport { + bucket: "bucket".to_string(), + scanned: 9, + continuation_token, + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + let encoded = record.encode().expect("job record should encode"); + let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json"); + value["job"] + .as_object_mut() + .expect("job should be object") + .remove("cursor_revision"); + let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode"); + value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned)); + let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode"); + + let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode"); + + assert_eq!(decoded.cursor_revision, Some(9)); + } + + #[test] + fn manual_transition_job_record_omits_cursor_revision_for_old_readers() { + #[allow(dead_code)] + #[derive(serde::Deserialize)] + #[serde(deny_unknown_fields)] + struct LegacyPersistedManualTransitionJobRecord { + schema: String, + content_sha256: String, + job: LegacyManualTransitionJobRecord, + } + + #[allow(dead_code)] + #[derive(serde::Deserialize)] + #[serde(deny_unknown_fields)] + struct LegacyManualTransitionJobRecord { + job_id: Uuid, + scope_key: String, + bucket: String, + prefix: String, + tier: Option, + dry_run: bool, + max_objects: Option, + max_duration: Option, + owner_id: String, + lease_id: Uuid, + lease_expires_at_unix_nanos: i128, + state: ManualTransitionJobState, + scan_completed: bool, + cancel_requested: bool, + created_at_unix_nanos: i128, + updated_at_unix_nanos: i128, + completed_at_unix_nanos: Option, + report: ManualTransitionRunReport, + queue_snapshot: ManualTransitionQueueSnapshot, + error: Option, + } + + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + let continuation_token = + encode_manual_transition_continuation_token(Some("logs/page-a".to_string()), Some("version-a".to_string())); + record.update_running_progress( + ManualTransitionRunReport { + bucket: "bucket".to_string(), + scanned: 7, + continuation_token: continuation_token.clone(), + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + assert_eq!(record.cursor_revision, Some(7)); + + let encoded = record.encode().expect("job record should encode"); + let value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json"); + assert!(value["job"].get("cursor_revision").is_none()); + let legacy: LegacyPersistedManualTransitionJobRecord = + serde_json::from_slice(&encoded).expect("old reader should accept new job record"); + + assert_eq!(legacy.job.job_id, record.job_id); + assert_eq!(legacy.job.report.continuation_token, continuation_token); + } + #[test] fn manual_transition_job_record_rejects_unknown_report_fields() { let options = ManualTransitionRunOptions::default(); diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index 6d8e64f1b..20956ca70 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -16,6 +16,7 @@ pub mod bucket_lifecycle_audit; pub mod bucket_lifecycle_ops; mod config_boundary; pub mod core; +mod durable_namespace; pub mod evaluator; pub mod manual_transition_job; mod metadata_boundary; @@ -31,3 +32,8 @@ pub mod tier_free_version_recovery; pub mod tier_last_day_stats; pub mod tier_sweeper; pub mod transition_transaction; + +pub(crate) use durable_namespace::{ + DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record, + validate_durable_ilm_record, +}; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index 6308b3767..22bc2ca70 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, warn}; use crate::bucket::lifecycle::config_boundary; +use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE; use crate::bucket::lifecycle::runtime_boundary; use crate::bucket::lifecycle::tier_sweeper::{ Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, @@ -49,7 +50,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2; const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3; const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4; const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5; -pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/"; +pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] @@ -432,6 +433,21 @@ async fn process_committed_tier_delete_journal_entry(api: Arc, je: &Jen ) .await?; } + let path = tier_delete_journal_object_name(je); + let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?; + let target_pool_indices = api + .record_durable_ilm_decommission_terminal_target_pools(&path, &data) + .await + .map_err(std::io::Error::other)?; + if let Some(target_pool_indices) = target_pool_indices { + for target_pool_idx in target_pool_indices { + match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await { + Ok(()) | Err(Error::ConfigNotFound) => {} + Err(err) => return Err(std::io::Error::other(err)), + } + } + return Ok(()); + } remove_tier_delete_journal_entry(api, je).await } diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index f9831d394..70bf4ed53 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -21,6 +21,7 @@ use tracing::{debug, warn}; use uuid::Uuid; use crate::bucket::lifecycle::config_boundary; +use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE; use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; use crate::bucket::lifecycle::tier_sweeper::{ delete_confirmed_transition_candidate_exact_with_lease_idempotent, @@ -42,7 +43,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6 const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300); pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1"; pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions"; -pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records"; +pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix; pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024; pub type Result = std::result::Result; @@ -584,7 +585,8 @@ pub(crate) async fn save_transition_transaction_record( let object = transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?; let data = transaction.encode().map_err(transition_transaction_store_error)?; - config_boundary::save_config(api, &object, data).await + config_boundary::save_config(api.clone(), &object, data.clone()).await?; + api.record_durable_ilm_decommission_progress(&object, &data).await } pub(crate) async fn load_transition_transaction_record( @@ -596,8 +598,14 @@ pub(crate) async fn load_transition_transaction_record( TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error) } -pub(crate) async fn delete_transition_transaction_record(api: Arc, transaction_id: Uuid) -> EcstoreResult<()> { - let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?; +pub(crate) async fn delete_transition_transaction_record( + api: Arc, + transaction: &TransitionTransaction, +) -> EcstoreResult<()> { + let object = + transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?; + let data = transaction.encode().map_err(transition_transaction_store_error)?; + api.record_durable_ilm_decommission_terminal(&object, &data).await?; match config_boundary::delete_config(api, &object).await { Ok(()) | Err(Error::ConfigNotFound) => Ok(()), Err(err) => Err(err), @@ -813,7 +821,7 @@ pub async fn finalize_missing_transition_transaction_for_operator( if probe != TransitionOperatorProbe::Missing { return Err(TransitionOperatorError::CandidateNotMissing(probe)); } - delete_transition_transaction_record(api, transaction_id) + delete_transition_transaction_record(api, &transaction) .await .map_err(TransitionOperatorError::Store) } @@ -849,22 +857,22 @@ pub async fn process_transition_transaction_record( match transaction.state { TransitionTransactionState::Uploaded => { delete_transition_remote_candidate(api.clone(), transaction).await?; - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await { Ok(true) => { - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) } Ok(false) => { delete_transition_remote_candidate(api.clone(), transaction).await?; - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } Err(err) if transition_source_is_missing(&err) => { delete_transition_remote_candidate(api.clone(), transaction).await?; - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } Err(err) => Err(err), @@ -872,7 +880,7 @@ pub async fn process_transition_transaction_record( TransitionTransactionState::LocalCommitStarted => { match local_commit_matches_transaction(api.clone(), transaction).await { Ok(true) => { - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) } Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained), @@ -881,7 +889,7 @@ pub async fn process_transition_transaction_record( } } TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => { - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) } TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await, @@ -907,7 +915,7 @@ async fn recover_unknown_upload_outcome( .map_err(Error::other)? { TransitionCandidateProbe::Missing => { - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) } TransitionCandidateProbe::UnversionedPresent => { @@ -925,7 +933,7 @@ async fn recover_unknown_upload_outcome( ) .await .map_err(Error::other)?; - delete_transition_transaction_record(api, transaction.transaction_id).await?; + delete_transition_transaction_record(api, transaction).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } TransitionCandidateProbe::VersionedPresent(version_id) => { @@ -958,7 +966,7 @@ async fn cleanup_recovered_unknown_upload_candidate( .map_err(transition_transaction_store_error)?; save_transition_transaction_record(api.clone(), &cleanup).await?; delete_transition_remote_candidate(api.clone(), &cleanup).await?; - delete_transition_transaction_record(api, cleanup.transaction_id).await?; + delete_transition_transaction_record(api, &cleanup).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index 3b2729b8b..472c87e63 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -406,6 +406,25 @@ where Ok(data) } +pub(crate) async fn read_config_limited_preserve_empty(api: Arc, file: &str, max_bytes: usize) -> Result> +where + S: EcstoreObjectIO, +{ + let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?; + Ok(data) +} + +pub(crate) async fn read_config_limited_preserve_empty_with_metadata( + api: Arc, + file: &str, + max_bytes: usize, +) -> Result<(Vec, ObjectInfo)> +where + S: EcstoreObjectIO, +{ + read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await +} + /// Read an existing config object without treating an empty payload as absent. /// Callers that validate their own payload format need to distinguish corruption /// from `ConfigNotFound`. diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 0c39d1f57..16e4394a5 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -16,19 +16,23 @@ use crate::bucket::replication::replication_state_from_filemeta; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::bucket::{ lifecycle::{ - LifecycleExpiryConfigs, + DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, ValidatedDurableIlmRecord, bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::{ LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_in, eval_action_from_lifecycle, lifecycle_delete_all_versions_blocked_by_replication, }, - get_expiry_configs, + classify_durable_ilm_record, get_expiry_configs, lifecycle::IlmAction, + validate_durable_ilm_record, }, metadata_sys, }; use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw}; -use crate::config::com::{CONFIG_PREFIX, read_config, read_config_no_lock, save_config, save_config_with_opts}; +use crate::config::com::{ + CONFIG_PREFIX, delete_config, read_config, read_config_limited_preserve_empty, + read_config_limited_preserve_empty_with_metadata, read_config_no_lock, save_config, save_config_with_opts, +}; use crate::data_movement; use crate::data_movement::backpressure::{self, DataMovementOperation}; use crate::data_usage::DATA_USAGE_CACHE_NAME; @@ -48,8 +52,9 @@ use crate::storage_api_contracts::{ admin::StorageAdminApi, bucket::{BucketOperations, BucketOptions, MakeBucketOptions}, heal::HealOperations as _, + list::ListOperations as _, namespace::NamespaceLocking as _, - object::{EcstoreObjectIO, ObjectIO as _, ObjectOperations as _}, + object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _}, }; use crate::{core::sets::Sets, store::ECStore}; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; @@ -60,6 +65,7 @@ use rmp_serde::Deserializer; use rmp_serde::Serializer; use rustfs_common::heal_channel::HealOpts; use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; +use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration}; use serde::{Deserialize, Serialize}; @@ -97,10 +103,18 @@ const DECOMMISSION_ENTRY_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_ENTRY_CONC const DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP: usize = 8; const DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP: usize = 64; const DECOMMISSION_ENTRY_WORKERS_PER_SET: usize = 2; +const DECOMMISSION_META_PREFIXES: [&str; 3] = [CONFIG_PREFIX, BUCKET_META_PREFIX, ILM_META_PREFIX]; const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30; const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3; const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5); const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1); +const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts"; +const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests"; +const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2"; +const DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA: &str = "v1"; +const DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE: usize = 16 * 1024; +const DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE: usize = 4 * 1024; +const DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS: usize = 3; /// Background decommission walks must tolerate slow object migrations; the /// stall timeout is the drive-health bound, not the total listing duration. const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); @@ -130,6 +144,11 @@ impl DecommissionCanceler { } } + #[cfg(test)] + pub(crate) fn new_for_test(token: CancellationToken) -> Self { + Self::new(token) + } + fn token(&self) -> &CancellationToken { &self.operation.token } @@ -377,6 +396,19 @@ fn is_decommission_meta_bucket(bucket: &DecomBucketInfo) -> bool { bucket.name == RUSTFS_META_BUCKET } +fn decommission_meta_buckets() -> [DecomBucketInfo; DECOMMISSION_META_PREFIXES.len()] { + DECOMMISSION_META_PREFIXES.map(|prefix| DecomBucketInfo { + name: RUSTFS_META_BUCKET.to_owned(), + prefix: prefix.to_owned(), + }) +} + +fn reconcile_decommission_meta_buckets(meta: &mut PoolMeta, idx: usize) -> bool { + let before = meta.pending_buckets(idx).len(); + meta.queue_buckets(idx, decommission_meta_buckets().into()); + meta.pending_buckets(idx).len() != before +} + fn split_decommission_buckets(buckets: Vec) -> (Vec, Vec) { let mut regular = Vec::with_capacity(buckets.len()); let mut meta = Vec::new(); @@ -1022,6 +1054,362 @@ fn resolve_decommission_partial_listing_entry( )) } +fn validate_decommission_durable_ilm_copy( + path: &str, + source_record: &ValidatedDurableIlmRecord, + target: &[u8], +) -> Result { + let target_record = validate_durable_ilm_record(path, target).map_err(|err| { + Error::other(format!( + "target durable ILM record is invalid at path `{path}` {}: {err}", + source_record.context() + )) + })?; + source_record + .checkpoint + .validate_successor(&target_record.checkpoint) + .map_err(|err| { + Error::other(format!( + "target durable ILM record generation mismatch at path `{path}` {}: {err}", + source_record.context() + )) + })?; + Ok(target_record) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct DecommissionDurableIlmReceipt { + source_path: String, + namespace: String, + id_kind: String, + id: String, + checkpoint: DurableIlmRecordCheckpoint, + terminal_checkpoint: Option, +} + +impl DecommissionDurableIlmReceipt { + fn new(path: &str, record: &ValidatedDurableIlmRecord) -> Self { + Self { + source_path: path.to_string(), + namespace: record.namespace.to_string(), + id_kind: record.id_kind.to_string(), + id: record.id.clone(), + checkpoint: record.checkpoint.clone(), + terminal_checkpoint: None, + } + } + + fn context(&self) -> String { + format!("namespace `{}` {} `{}`", self.namespace, self.id_kind, self.id) + } + + fn validate(&self) -> Result<()> { + let namespace = classify_durable_ilm_record(&self.source_path)? + .ok_or_else(|| Error::other(format!("receipt source path `{}` is not a durable ILM record", self.source_path)))?; + if namespace.name != self.namespace { + return Err(Error::other(format!( + "receipt namespace `{}` does not match source path `{}`", + self.namespace, self.source_path + ))); + } + if self.id_kind.is_empty() || self.id.is_empty() { + return Err(Error::other(format!( + "receipt identity is missing for source path `{}`", + self.source_path + ))); + } + if !is_sha256_checksum(self.checkpoint.content_sha256()) { + return Err(Error::other(format!( + "receipt target checksum is invalid for source path `{}` {}", + self.source_path, + self.context() + ))); + } + if let Some(terminal_checkpoint) = &self.terminal_checkpoint { + self.checkpoint.validate_successor(terminal_checkpoint).map_err(|err| { + Error::other(format!( + "receipt terminal checkpoint is invalid for source path `{}` {}: {err}", + self.source_path, + self.context() + )) + })?; + } + Ok(()) + } + + fn encode(&self) -> Result> { + let mut receipt = self.clone(); + receipt.checkpoint = receipt.checkpoint.compacted()?; + receipt.terminal_checkpoint = receipt + .terminal_checkpoint + .as_ref() + .map(DurableIlmRecordCheckpoint::compacted) + .transpose()?; + receipt.validate()?; + let receipt_bytes = serde_json::to_vec(&receipt)?; + let persisted = PersistedDecommissionDurableIlmReceipt { + schema: DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA.to_string(), + content_sha256: hex_sha256(&receipt_bytes, ToOwned::to_owned), + receipt, + }; + let encoded = serde_json::to_vec(&persisted)?; + if encoded.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE { + return Err(Error::other(format!( + "durable ILM receipt exceeds maximum size for source path `{}` {}", + self.source_path, + self.context() + ))); + } + Ok(encoded) + } + + fn decode(data: &[u8]) -> Result { + if data.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE { + return Err(Error::other("durable ILM receipt exceeds maximum size")); + } + let persisted: PersistedDecommissionDurableIlmReceipt = serde_json::from_slice(data)?; + if persisted.schema != DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA { + return Err(Error::other(format!("unsupported durable ILM receipt schema `{}`", persisted.schema))); + } + if !is_sha256_checksum(&persisted.content_sha256) { + return Err(Error::other("durable ILM receipt checksum is invalid")); + } + let receipt_bytes = serde_json::to_vec(&persisted.receipt)?; + let actual_checksum = hex_sha256(&receipt_bytes, ToOwned::to_owned); + if persisted.content_sha256 != actual_checksum { + return Err(Error::other("durable ILM receipt checksum mismatch")); + } + persisted.receipt.validate()?; + Ok(persisted.receipt) + } +} + +fn merge_decommission_durable_ilm_receipts( + existing: &DecommissionDurableIlmReceipt, + incoming: &DecommissionDurableIlmReceipt, +) -> Result { + if existing.source_path != incoming.source_path + || existing.namespace != incoming.namespace + || existing.id_kind != incoming.id_kind + || existing.id != incoming.id + { + return Err(Error::other(format!( + "durable ILM receipt identity conflict for source path `{}` {}; incoming {}", + existing.source_path, + existing.context(), + incoming.context() + ))); + } + + let checkpoint = + if existing.checkpoint == incoming.checkpoint || incoming.checkpoint.validate_successor(&existing.checkpoint).is_ok() { + existing.checkpoint.clone() + } else { + existing.checkpoint.validate_successor(&incoming.checkpoint).map_err(|err| { + Error::other(format!( + "durable ILM receipt checkpoint conflict for source path `{}` {}: {err}", + existing.source_path, + existing.context() + )) + })?; + incoming.checkpoint.clone() + }; + let terminal_checkpoint = match (&existing.terminal_checkpoint, &incoming.terminal_checkpoint) { + (Some(existing_terminal), Some(incoming_terminal)) if existing_terminal == incoming_terminal => { + Some(existing_terminal.clone()) + } + (Some(existing_terminal), Some(incoming_terminal)) if incoming_terminal.validate_successor(existing_terminal).is_ok() => { + Some(existing_terminal.clone()) + } + (Some(existing_terminal), Some(incoming_terminal)) => { + existing_terminal.validate_successor(incoming_terminal).map_err(|err| { + Error::other(format!( + "durable ILM receipt terminal checkpoint conflict for source path `{}` {}: {err}", + existing.source_path, + existing.context() + )) + })?; + Some(incoming_terminal.clone()) + } + (Some(existing_terminal), None) => Some(existing_terminal.clone()), + (None, Some(incoming_terminal)) => Some(incoming_terminal.clone()), + (None, None) => None, + }; + let merged = DecommissionDurableIlmReceipt { + source_path: existing.source_path.clone(), + namespace: existing.namespace.clone(), + id_kind: existing.id_kind.clone(), + id: existing.id.clone(), + checkpoint, + terminal_checkpoint, + }; + merged.validate()?; + Ok(merged) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedDecommissionDurableIlmReceipt { + schema: String, + content_sha256: String, + receipt: DecommissionDurableIlmReceipt, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct DecommissionDurableIlmManifest { + schema: String, + run_token: String, + receipt_count: u64, + receipt_paths_sha256: String, +} + +impl DecommissionDurableIlmManifest { + fn new(run_token: &str, receipt_paths: &[String]) -> Result { + let manifest = Self { + schema: DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA.to_string(), + run_token: run_token.to_string(), + receipt_count: u64::try_from(receipt_paths.len()) + .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?, + receipt_paths_sha256: decommission_durable_ilm_manifest_paths_sha256(receipt_paths)?, + }; + manifest.validate(run_token, receipt_paths)?; + Ok(manifest) + } + + fn validate(&self, run_token: &str, receipt_paths: &[String]) -> Result<()> { + if self.schema != DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA { + return Err(Error::other(format!( + "unsupported durable ILM expected manifest schema `{}`", + self.schema + ))); + } + if self.run_token != run_token || !is_sha256_checksum(&self.run_token) { + return Err(Error::other("durable ILM expected manifest run token is invalid")); + } + let receipt_count = u64::try_from(receipt_paths.len()) + .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?; + if self.receipt_count != receipt_count { + return Err(Error::other(format!( + "durable ILM expected manifest receipt count mismatch: expected {}, found {receipt_count}", + self.receipt_count + ))); + } + if !is_sha256_checksum(&self.receipt_paths_sha256) + || self.receipt_paths_sha256 != decommission_durable_ilm_manifest_paths_sha256(receipt_paths)? + { + return Err(Error::other("durable ILM expected manifest receipt paths checksum mismatch")); + } + Ok(()) + } + + fn encode(&self) -> Result> { + let encoded = serde_json::to_vec(self)?; + if encoded.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE { + return Err(Error::other("durable ILM expected manifest exceeds maximum size")); + } + Ok(encoded) + } + + fn decode(data: &[u8], run_token: &str, receipt_paths: &[String]) -> Result { + if data.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE { + return Err(Error::other("durable ILM expected manifest exceeds maximum size")); + } + let manifest: Self = serde_json::from_slice(data)?; + manifest.validate(run_token, receipt_paths)?; + Ok(manifest) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct DecommissionDurableIlmReceiptLocator { + run_token: String, + source_path: String, + id_kind: String, + id: String, +} + +impl DecommissionDurableIlmReceiptLocator { + fn context(&self) -> String { + format!("source path `{}` {} `{}`", self.source_path, self.id_kind, self.id) + } +} + +fn decommission_durable_ilm_receipt_run_token(cmd_line: &str, start_time: OffsetDateTime) -> String { + let identity = format!("{cmd_line}\0{}", start_time.unix_timestamp_nanos()); + hex_sha256(identity.as_bytes(), ToOwned::to_owned) +} + +fn decommission_durable_ilm_receipt_run_prefix(run_token: &str) -> String { + format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/{run_token}/") +} + +fn decommission_durable_ilm_receipt_path(run_token: &str, source_path: &str, id_kind: &str, id: &str) -> String { + format!( + "{}{}/{}/{}.json", + decommission_durable_ilm_receipt_run_prefix(run_token), + source_path, + id_kind, + id + ) +} + +fn decommission_durable_ilm_manifest_path(run_token: &str) -> String { + format!("{DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT}/{run_token}.json") +} + +fn decommission_durable_ilm_manifest_paths_sha256(receipt_paths: &[String]) -> Result { + let mut sorted_paths = receipt_paths.iter().map(String::as_str).collect::>(); + sorted_paths.sort_unstable(); + let encoded = serde_json::to_vec(&sorted_paths)?; + Ok(hex_sha256(&encoded, ToOwned::to_owned)) +} + +fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result { + let prefix = format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/"); + let suffix = path + .strip_prefix(&prefix) + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has the wrong root")))?; + let (run_token, record_path) = suffix + .split_once('/') + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record path")))?; + let mut parts = record_path.rsplitn(3, '/'); + let id = parts + .next() + .and_then(|file| file.strip_suffix(".json")) + .filter(|id| !id.is_empty()) + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?; + let id_kind = parts + .next() + .filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id")) + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?; + let source_path = parts + .next() + .filter(|source_path| !source_path.is_empty()) + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its source path")))?; + if !is_sha256_checksum(run_token) { + return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token"))); + } + match id_kind { + "operation_id" if !is_sha256_checksum(id) => { + return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid operation id"))); + } + "transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => { + return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID"))); + } + _ => {} + } + classify_durable_ilm_record(source_path)? + .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` does not identify a durable ILM source path")))?; + Ok(DecommissionDurableIlmReceiptLocator { + run_token: run_token.to_string(), + source_path: source_path.to_string(), + id_kind: id_kind.to_string(), + id: id.to_string(), + }) +} + fn resolve_decommission_pool_meta_reload_result(result: Result<()>, stage: &str) -> Result<()> { result.map_err(|err| Error::other(format!("decommission pool meta reload failed during {stage}: {err}"))) } @@ -3101,6 +3489,8 @@ impl ECStore { async fn decommission_cancel_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> { ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?; let _start_guard = self.start_gate.lock().await; + let operation_gate = self.ctx.decommission_operation_gate(); + let operation_guard = operation_gate.write().await; // Lock order: decommission_cancelers before pool_meta. Holding both makes // owner validation and the terminal transition one atomic operation. @@ -3159,8 +3549,6 @@ impl ECStore { ); } - self.wait_for_decommission_side_effects().await; - if should_save_pool_meta && let Err(err) = self.save_current_pool_meta().await { if let Some(previous_pool_meta) = previous_pool_meta { let mut pool_meta = self.pool_meta.write().await; @@ -3168,6 +3556,7 @@ impl ECStore { } return Err(err); } + drop(operation_guard); if let Some(canceler) = terminal_canceler.as_ref() { self.release_decommission_canceler_slot(idx, canceler).await; @@ -3226,24 +3615,24 @@ impl ECStore { } async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result { - // Serialize promotion and generation capture with clear/restart transitions. - let (promoted, generation, save_error) = { + let (changed, generation, save_error) = { let _start_guard = self.start_gate.lock().await; let mut pool_meta = self.pool_meta.write().await; if pool_meta.pools.get(idx).is_none() { return Err(Error::other("failed to start decommission: target pool was not found")); } + let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx); let promoted = pool_meta.promote_queued_decommission(idx); + let changed = reconciled || promoted; drop(pool_meta); - let save_error = if promoted { + let save_error = if changed { self.save_current_pool_meta().await.err() } else { None }; - let generation = self.active_decommission_generation(idx).await?; - (promoted, generation, save_error) + (changed, generation, save_error) }; if let Some(err) = save_error { @@ -3255,7 +3644,7 @@ impl ECStore { return Err(err); } - if promoted && let Some(notification_sys) = runtime_sources::notification_sys() { + if changed && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("promote_queued_decommission for pool {idx}"); if let Err(err) = resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) @@ -3272,6 +3661,12 @@ impl ECStore { Ok(generation) } + #[cfg(test)] + pub(crate) async fn promote_queued_decommission_for_test(&self, idx: usize) -> Result<()> { + let owner = DecommissionCanceler::new(CancellationToken::new()); + self.promote_queued_decommission(idx, &owner).await.map(|_| ()) + } + async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> { let changed = { let mut pool_meta = self.pool_meta.write().await; @@ -3812,6 +4207,12 @@ impl ECStore { ); return Ok(()); } + let durable_ilm_record = if bucket == RUSTFS_META_BUCKET { + classify_durable_ilm_record(&entry.name) + .map_err(|err| with_decommission_entry_context("durable_ilm_namespace", &bucket, &entry.name, err))? + } else { + None + }; if self.decommission_cancel_requested(idx, &rx).await { rx.cancel(); } @@ -4138,7 +4539,8 @@ impl ECStore { } } - if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) { + if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) && durable_ilm_record.is_none() + { if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) { return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup")); } @@ -4193,6 +4595,17 @@ impl ECStore { }) .await; resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())? + } else if durable_ilm_record.is_some() { + debug!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + bucket = %bucket, + object = %entry.name, + state = "retained_for_final_verification", + "Decommission durable ILM source retained for final verification" + ); } else if decommissioned != fivs.versions.len() || expired > 0 { warn!( event = EVENT_DECOMMISSION_ENTRY, @@ -4442,6 +4855,18 @@ impl ECStore { Ok(()) } + #[cfg(test)] + pub(crate) async fn decommission_pool_for_test( + self: &Arc, + rx: CancellationToken, + idx: usize, + pool: Arc, + bucket: DecomBucketInfo, + ) -> Result<()> { + self.decommission_pool(rx, idx, pool, bucket, Arc::new(Semaphore::new(decommission_entry_concurrency_limit()))) + .await + } + #[tracing::instrument(skip(self, canceler))] pub async fn do_decommission_in_routine( self: &Arc, @@ -4602,7 +5027,10 @@ impl ECStore { state = "verifying_completion", "Decommission completion verification started" ); - if let Err(err) = self.check_after_decommission(idx).await { + if let Err(err) = self.check_after_decommission(idx, &rx, generation).await { + if is_err_operation_canceled(&err) { + return Err(err); + } resolve_decommission_terminal_mark_result( self.decommission_failed_for_operation(idx, canceler).await, "failed", @@ -4612,13 +5040,6 @@ impl ECStore { "failed to finalize decommission for pool {cmd_line}: post-check failed: {err}" ))); } - - if self.decommission_cancel_requested(idx, &rx).await { - rx.cancel(); - } - decommission_cancel_signal_result(rx.is_cancelled())?; - self.ensure_decommission_generation_current(idx, generation).await?; - info!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, @@ -4628,11 +5049,14 @@ impl ECStore { state = "marking_completed", "Decommission marking completed state" ); - resolve_decommission_terminal_mark_result( - self.complete_decommission_for_operation(idx, canceler).await, - "completed", - &cmd_line, - )?; + if let Err(err) = self.complete_decommission_for_operation(idx, canceler).await { + resolve_decommission_terminal_mark_result( + self.decommission_failed_for_operation(idx, canceler).await, + "failed", + &cmd_line, + )?; + return Err(Error::other(format!("failed to finalize decommission for pool {cmd_line}: {err}"))); + } } DecommissionFinalState::Failed => { warn!( @@ -4774,11 +5198,18 @@ impl ECStore { async fn complete_decommission_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> { ensure_decommission_terminal_operation_supported(self.single_pool(), "complete decommission")?; + ensure_valid_decommission_pool_index(self.pools.len(), idx)?; + if let Some(owner) = owner { + let cancelers = self.decommission_cancelers.read().await; + if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) { + owner.release(); + return Ok(()); + } + } + self.verify_decommission_durable_ilm_receipts(idx).await?; let _start_guard = self.start_gate.lock().await; - // Lock order: decommission_cancelers before pool_meta. Holding both makes - // owner validation and the terminal transition one atomic operation. - let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = { + let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; let previous_pool_meta = pool_meta.clone(); @@ -4789,12 +5220,17 @@ impl ECStore { else { return Ok(()); }; + let completed = pool_meta + .pools + .get(idx) + .and_then(|pool| pool.decommission.as_ref()) + .is_some_and(|decommission| decommission.complete); let terminal_canceler = if let Some(owner) = owner { Some(owner.clone()) } else { cancelers.get(idx).and_then(Option::as_ref).cloned() }; - (changed, changed.then_some(previous_pool_meta), terminal_canceler) + (changed, completed, changed.then_some(previous_pool_meta), terminal_canceler) }; if should_reload_pool_meta && let Err(err) = self.save_current_pool_meta().await { @@ -4846,6 +5282,18 @@ impl ECStore { } } + if completed && let Err(err) = self.cleanup_decommission_durable_ilm_receipts(idx).await { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "receipt_cleanup_failed", + error = %err, + "Decommission durable ILM receipt cleanup failed" + ); + } + Ok(()) } @@ -5007,17 +5455,16 @@ impl ECStore { let decom_buckets = self.get_buckets_to_decommission().await?; + let mut healed_buckets = HashSet::with_capacity(decom_buckets.len()); for bk in decom_buckets.iter() { - resolve_decommission_preflight_heal_result(&bk.name, self.heal_bucket(&bk.name, &HealOpts::default()).await)?; + if healed_buckets.insert(bk.name.as_str()) { + resolve_decommission_preflight_heal_result(&bk.name, self.heal_bucket(&bk.name, &HealOpts::default()).await)?; + } } - let meta_buckets = [ - path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(CONFIG_PREFIX)]), - path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(BUCKET_META_PREFIX)]), - ]; - let meta_bucket_opts = decommission_meta_bucket_options(); - for bk in meta_buckets.iter() { + for prefix in DECOMMISSION_META_PREFIXES { + let bk = path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(prefix)]); if let Err(err) = self .make_bucket(bk.to_string_lossy().to_string().as_str(), &meta_bucket_opts) .await @@ -5136,19 +5583,955 @@ impl ECStore { }) .collect(); - ret.push(DecomBucketInfo { - name: RUSTFS_META_BUCKET.to_owned(), - prefix: CONFIG_PREFIX.to_owned(), - }); - ret.push(DecomBucketInfo { - name: RUSTFS_META_BUCKET.to_owned(), - prefix: BUCKET_META_PREFIX.to_owned(), - }); + ret.extend(decommission_meta_buckets()); Ok(ret) } - async fn check_after_decommission(self: &Arc, idx: usize) -> Result<()> { + async fn durable_ilm_receipt_run_token(&self, source_pool_idx: usize) -> Result { + let pool_meta = self.pool_meta.read().await; + let pool = pool_meta + .pools + .get(source_pool_idx) + .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), source_pool_idx))?; + let start_time = pool + .decommission + .as_ref() + .and_then(|info| info.start_time) + .ok_or_else(|| Error::other(format!("decommission run identity is missing for pool {source_pool_idx}")))?; + Ok(decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time)) + } + + async fn load_decommissioned_durable_ilm_target( + &self, + source_pool_idx: usize, + path: &str, + max_record_size: usize, + record_context: &str, + ) -> Result)>> { + let mut target = None::<(usize, Vec)>; + let mut first_read_error = None; + for (target_pool_idx, pool) in self.pools.iter().enumerate() { + if target_pool_idx == source_pool_idx { + continue; + } + match read_config_limited_preserve_empty(pool.clone(), path, max_record_size).await { + Ok(data) => { + if let Some((existing_pool_idx, existing)) = target.as_ref() + && existing != &data + { + return Err(Error::other(format!( + "divergent target durable ILM records at path `{path}` {record_context} in pools {existing_pool_idx} and {target_pool_idx}" + ))); + } + target = Some((target_pool_idx, data)); + } + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => {} + Err(err) => { + first_read_error.get_or_insert_with(|| { + Error::other(format!( + "failed to read target durable ILM record at path `{path}` {record_context} from pool {target_pool_idx}: {err}" + )) + }); + } + } + } + + if let Some(err) = first_read_error { + return Err(err); + } + Ok(target) + } + + async fn list_decommission_durable_ilm_receipt_paths_in_pool(&self, pool_idx: usize, prefix: &str) -> Result> { + let pool = self + .pools + .get(pool_idx) + .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), pool_idx))?; + let mut receipts = Vec::new(); + let mut continuation = None; + loop { + let page = pool + .clone() + .list_objects_v2(RUSTFS_META_BUCKET, prefix, continuation, None, 1000, false, None, false) + .await + .map_err(|err| { + Error::other(format!( + "failed to list durable ILM decommission receipts under `{prefix}` in pool {pool_idx}: {err}" + )) + })?; + receipts.extend(page.objects.into_iter().map(|object| object.name)); + if !page.is_truncated { + break; + } + continuation = Some(page.next_continuation_token.ok_or_else(|| { + Error::other(format!( + "durable ILM decommission receipt listing under `{prefix}` in pool {pool_idx} was truncated without a continuation token" + )) + })?); + } + Ok(receipts) + } + + async fn list_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let prefix = decommission_durable_ilm_receipt_run_prefix(&run_token); + let mut receipts = Vec::new(); + for pool_idx in 0..self.pools.len() { + if pool_idx == source_pool_idx { + continue; + } + for receipt_path in self + .list_decommission_durable_ilm_receipt_paths_in_pool(pool_idx, &prefix) + .await? + { + let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?; + if locator.run_token != run_token { + return Err(Error::other(format!( + "durable ILM receipt path `{receipt_path}` has an unexpected run token" + ))); + } + receipts.push((pool_idx, receipt_path)); + } + } + Ok(receipts) + } + + async fn list_decommission_durable_ilm_manifest_receipts(&self, source_pool_idx: usize) -> Result> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let prefix = decommission_durable_ilm_receipt_run_prefix(&run_token); + let receipt_paths = self + .list_decommission_durable_ilm_receipt_paths_in_pool(source_pool_idx, &prefix) + .await?; + for receipt_path in &receipt_paths { + let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?; + if locator.run_token != run_token { + return Err(Error::other(format!( + "durable ILM expected manifest receipt path `{receipt_path}` has an unexpected run token" + ))); + } + } + Ok(receipt_paths) + } + + async fn persist_decommission_durable_ilm_manifest(&self, source_pool_idx: usize) -> Result<()> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let receipt_paths = self.list_decommission_durable_ilm_manifest_receipts(source_pool_idx).await?; + for receipt_path in &receipt_paths { + self.read_decommission_durable_ilm_receipt(source_pool_idx, receipt_path) + .await?; + } + let manifest = DecommissionDurableIlmManifest::new(&run_token, &receipt_paths)?; + let manifest_path = decommission_durable_ilm_manifest_path(&run_token); + let encoded = manifest.encode()?; + let mut attempt = 1; + loop { + match read_config_limited_preserve_empty( + self.pools[source_pool_idx].clone(), + &manifest_path, + DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE, + ) + .await + { + Ok(existing) => { + DecommissionDurableIlmManifest::decode(&existing, &run_token, &receipt_paths).map_err(|err| { + Error::other(format!( + "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}" + )) + })?; + return Ok(()); + } + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => {} + Err(err) => { + return Err(Error::other(format!( + "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}" + ))); + } + } + match save_config_with_opts( + self.pools[source_pool_idx].clone(), + &manifest_path, + encoded.clone(), + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + { + Ok(()) => return Ok(()), + Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => { + attempt += 1; + } + Err(Error::PreconditionFailed) => { + return Err(Error::other(format!( + "failed to persist durable ILM expected manifest `{manifest_path}` after concurrent updates" + ))); + } + Err(err) => { + return Err(Error::other(format!( + "failed to persist durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx}: {err}" + ))); + } + } + } + } + + async fn load_decommission_durable_ilm_manifest( + &self, + source_pool_idx: usize, + ) -> Result> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let receipt_paths = self.list_decommission_durable_ilm_manifest_receipts(source_pool_idx).await?; + let manifest_path = decommission_durable_ilm_manifest_path(&run_token); + let data = read_config_limited_preserve_empty( + self.pools[source_pool_idx].clone(), + &manifest_path, + DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE, + ) + .await + .map_err(|err| { + Error::other(format!( + "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}" + )) + })?; + DecommissionDurableIlmManifest::decode(&data, &run_token, &receipt_paths).map_err(|err| { + Error::other(format!( + "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}" + )) + })?; + + let mut receipts = HashMap::with_capacity(receipt_paths.len()); + for receipt_path in receipt_paths { + let receipt = self + .read_decommission_durable_ilm_receipt(source_pool_idx, &receipt_path) + .await?; + if receipts.insert(receipt_path.clone(), receipt).is_some() { + return Err(Error::other(format!( + "durable ILM expected manifest contains duplicate receipt path `{receipt_path}`" + ))); + } + } + Ok(receipts) + } + + async fn persist_decommission_durable_ilm_receipt( + &self, + source_pool_idx: usize, + target_pool_idx: usize, + receipt: &DecommissionDurableIlmReceipt, + ) -> Result<()> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let receipt_path = decommission_durable_ilm_receipt_path(&run_token, &receipt.source_path, &receipt.id_kind, &receipt.id); + let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?; + let mut attempt = 1; + loop { + let (merged, http_preconditions) = match read_config_limited_preserve_empty_with_metadata( + self.pools[target_pool_idx].clone(), + &receipt_path, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + ) + .await + { + Ok((existing_data, metadata)) => { + let existing = DecommissionDurableIlmReceipt::decode(&existing_data).map_err(|err| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} for {} is invalid: {err}", + locator.context() + )) + })?; + Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &existing)?; + let merged = merge_decommission_durable_ilm_receipts(&existing, receipt)?; + if merged == existing { + return Ok(()); + } + let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} is missing an ETag" + )) + })?; + ( + merged, + HTTPPreconditions { + if_match: Some(etag), + ..Default::default() + }, + ) + } + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => + { + ( + receipt.clone(), + HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }, + ) + } + Err(err) => { + return Err(Error::other(format!( + "failed to read durable ILM decommission receipt `{receipt_path}` from pool {target_pool_idx} for {}: {err}", + locator.context() + ))); + } + }; + let encoded = merged.encode().map_err(|err| { + Error::other(format!( + "failed to encode durable ILM decommission receipt `{receipt_path}` for source path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + match save_config_with_opts( + self.pools[target_pool_idx].clone(), + &receipt_path, + encoded, + &ObjectOptions { + max_parity: true, + http_preconditions: Some(http_preconditions), + ..Default::default() + }, + ) + .await + { + Ok(()) => return Ok(()), + Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => { + attempt += 1; + } + Err(Error::PreconditionFailed) => { + return Err(Error::other(format!( + "failed to persist durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates", + locator.context() + ))); + } + Err(err) => { + return Err(Error::other(format!( + "failed to persist durable ILM decommission receipt `{receipt_path}` for {}: {err}", + locator.context() + ))); + } + } + } + } + + fn validate_decommission_durable_ilm_receipt_locator( + receipt_path: &str, + locator: &DecommissionDurableIlmReceiptLocator, + receipt: &DecommissionDurableIlmReceipt, + ) -> Result<()> { + if locator.source_path != receipt.source_path || locator.id_kind != receipt.id_kind || locator.id != receipt.id { + return Err(Error::other(format!( + "durable ILM decommission receipt path `{receipt_path}` identity {} does not match receipt {}", + locator.context(), + receipt.context() + ))); + } + Ok(()) + } + + async fn read_decommission_durable_ilm_receipt( + &self, + receipt_pool_idx: usize, + receipt_path: &str, + ) -> Result { + let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?; + let data = read_config_limited_preserve_empty( + self.pools[receipt_pool_idx].clone(), + receipt_path, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + ) + .await + .map_err(|err| { + Error::other(format!( + "failed to read durable ILM decommission receipt `{receipt_path}` from pool {receipt_pool_idx} for {}: {err}", + locator.context() + )) + })?; + let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} for {} is invalid: {err}", + locator.context() + )) + })?; + Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?; + Ok(receipt) + } + + async fn load_decommission_durable_ilm_terminal_receipt( + &self, + source_pool_idx: usize, + path: &str, + source_record: &ValidatedDurableIlmRecord, + ) -> Result> { + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, source_record.id_kind, &source_record.id); + let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?; + let mut proof = None::; + for pool_idx in 0..self.pools.len() { + if pool_idx == source_pool_idx { + continue; + } + let data = match read_config_limited_preserve_empty( + self.pools[pool_idx].clone(), + &receipt_path, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + ) + .await + { + Ok(data) => data, + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => + { + continue; + } + Err(err) => { + return Err(Error::other(format!( + "failed to read terminal durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}", + source_record.context() + ))); + } + }; + let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| { + Error::other(format!( + "terminal durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}", + source_record.context() + )) + })?; + Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?; + if receipt.namespace != source_record.namespace + || receipt.id_kind != source_record.id_kind + || receipt.id != source_record.id + { + return Err(Error::other(format!( + "terminal durable ILM decommission receipt identity mismatch at path `{path}` {}; receipt {}", + source_record.context(), + receipt.context() + ))); + } + source_record + .checkpoint + .validate_successor(&receipt.checkpoint) + .map_err(|err| { + Error::other(format!( + "terminal durable ILM decommission receipt does not cover source at path `{path}` {}: {err}", + source_record.context() + )) + })?; + if receipt.terminal_checkpoint.is_some() { + proof = Some(match proof { + Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?, + None => receipt, + }); + } + } + Ok(proof) + } + + async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> { + let expected_receipts = self.load_decommission_durable_ilm_manifest(source_pool_idx).await?; + let receipt_paths = self.list_decommission_durable_ilm_receipts(source_pool_idx).await?; + let present_receipt_paths = receipt_paths + .iter() + .map(|(_, receipt_path)| receipt_path.as_str()) + .collect::>(); + for (expected_path, expected) in &expected_receipts { + if !present_receipt_paths.contains(expected_path.as_str()) { + return Err(Error::other(format!( + "durable ILM decommission receipt is missing at `{expected_path}` for source path `{}` {}", + expected.source_path, + expected.context() + ))); + } + } + + for (receipt_pool_idx, receipt_path) in receipt_paths { + let expected = expected_receipts.get(&receipt_path).ok_or_else(|| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} is absent from the expected manifest" + )) + })?; + let receipt = self + .read_decommission_durable_ilm_receipt(receipt_pool_idx, &receipt_path) + .await?; + if receipt.source_path != expected.source_path + || receipt.namespace != expected.namespace + || receipt.id_kind != expected.id_kind + || receipt.id != expected.id + { + return Err(Error::other(format!( + "durable ILM decommission receipt identity mismatch at `{receipt_path}` for source path `{}` {}; decoded {}", + expected.source_path, + expected.context(), + receipt.context() + ))); + } + expected.checkpoint.validate_successor(&receipt.checkpoint).map_err(|err| { + Error::other(format!( + "durable ILM decommission receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}", + expected.source_path, + expected.context() + )) + })?; + match (&expected.terminal_checkpoint, &receipt.terminal_checkpoint) { + (Some(expected_terminal), Some(receipt_terminal)) => { + expected_terminal.validate_successor(receipt_terminal).map_err(|err| { + Error::other(format!( + "durable ILM decommission terminal receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}", + expected.source_path, + expected.context() + )) + })?; + } + (Some(_), None) => { + return Err(Error::other(format!( + "durable ILM decommission terminal receipt is missing at `{receipt_path}` for source path `{}` {}", + expected.source_path, + expected.context() + ))); + } + (None, _) => {} + } + let namespace = classify_durable_ilm_record(&receipt.source_path)? + .ok_or_else(|| Error::other(format!("path `{}` is not a durable ILM record", receipt.source_path)))?; + let target = self + .load_decommissioned_durable_ilm_target( + source_pool_idx, + &receipt.source_path, + namespace.max_record_size, + &receipt.context(), + ) + .await?; + if let Some((_, target)) = target { + let target_record = validate_durable_ilm_record(&receipt.source_path, &target).map_err(|err| { + Error::other(format!( + "target durable ILM record is invalid at path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + let identity_matches = target_record.namespace == receipt.namespace + && target_record.id_kind == receipt.id_kind + && target_record.id == receipt.id; + let reused_manual_scope = receipt.terminal_checkpoint.is_some() + && matches!( + (&receipt.checkpoint, &target_record.checkpoint), + ( + DurableIlmRecordCheckpoint::ManualTransitionScope { .. }, + DurableIlmRecordCheckpoint::ManualTransitionScope { .. } + ) + ); + if !identity_matches && !reused_manual_scope { + return Err(Error::other(format!( + "target durable ILM record identity mismatch at path `{}` {}; decoded {}", + receipt.source_path, + receipt.context(), + target_record.context() + ))); + } + if identity_matches { + receipt + .terminal_checkpoint + .as_ref() + .unwrap_or(&receipt.checkpoint) + .validate_successor(&target_record.checkpoint) + .map_err(|err| { + Error::other(format!( + "target durable ILM record generation mismatch at path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + } + } else if receipt.terminal_checkpoint.is_none() { + return Err(Error::other(format!( + "target durable ILM record is missing at path `{}` {} without a recovery terminal checkpoint", + receipt.source_path, + receipt.context() + ))); + } + } + Ok(()) + } + + async fn advance_durable_ilm_decommission_receipt( + &self, + pool_idx: usize, + receipt_path: &str, + record: &ValidatedDurableIlmRecord, + terminal: bool, + ) -> Result { + let stage = if terminal { "terminal" } else { "progress" }; + let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?; + let mut attempt = 1; + loop { + let (receipt_data, metadata) = match read_config_limited_preserve_empty_with_metadata( + self.pools[pool_idx].clone(), + receipt_path, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, + ) + .await + { + Ok(receipt) => receipt, + Err(err) + if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) + || is_err_object_not_found(&err) + || is_err_version_not_found(&err) => + { + return Ok(false); + } + Err(err) => { + return Err(Error::other(format!( + "failed to read durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}", + locator.context() + ))); + } + }; + let mut receipt = DecommissionDurableIlmReceipt::decode(&receipt_data).map_err(|err| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}", + locator.context() + )) + })?; + Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?; + receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| { + Error::other(format!( + "{stage} durable ILM record generation mismatch at path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + + if terminal { + if let Some(existing) = &receipt.terminal_checkpoint { + if existing == &record.checkpoint || record.checkpoint.validate_successor(existing).is_ok() { + return Ok(true); + } + existing.validate_successor(&record.checkpoint).map_err(|err| { + Error::other(format!( + "terminal durable ILM record checkpoint conflicts at path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + } + receipt.terminal_checkpoint = Some(record.checkpoint.clone()); + } else { + if let Some(existing) = &receipt.terminal_checkpoint { + if existing == &record.checkpoint { + return Ok(true); + } + existing.validate_successor(&record.checkpoint).map_err(|err| { + Error::other(format!( + "progress durable ILM record conflicts with terminal checkpoint at path `{}` {}: {err}", + receipt.source_path, + receipt.context() + )) + })?; + receipt.terminal_checkpoint = None; + } + if receipt.checkpoint == record.checkpoint { + return Ok(true); + } + receipt.checkpoint = record.checkpoint.clone(); + } + + let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| { + Error::other(format!( + "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} is missing an ETag" + )) + })?; + let encoded = receipt.encode()?; + match save_config_with_opts( + self.pools[pool_idx].clone(), + receipt_path, + encoded, + &ObjectOptions { + max_parity: true, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(etag), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + { + Ok(()) => return Ok(true), + Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => { + attempt += 1; + continue; + } + Err(Error::PreconditionFailed) => { + return Err(Error::other(format!( + "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates", + locator.context() + ))); + } + Err(err) => { + return Err(Error::other(format!( + "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {}: {err}", + locator.context() + ))); + } + } + } + } + + async fn advance_durable_ilm_decommission_receipts( + &self, + path: &str, + data: &[u8], + terminal: bool, + ) -> Result>> { + let active_runs = { + let pool_meta = self.pool_meta.read().await; + pool_meta + .pools + .iter() + .enumerate() + .filter_map(|(pool_idx, pool)| { + pool.decommission + .as_ref() + .filter(|info| info.has_decommission_state() && !info.complete) + .and_then(|info| info.start_time) + .map(|start_time| (pool_idx, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))) + }) + .collect::>() + }; + if active_runs.is_empty() { + return Ok(None); + } + + let stage = if terminal { "terminal" } else { "progress" }; + let record = validate_durable_ilm_record(path, data) + .map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?; + let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::>(); + let mut terminal_target_pool_indices = Vec::new(); + for (source_pool_idx, run_token) in active_runs { + let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, record.id_kind, &record.id); + let mut receipt_found = false; + for pool_idx in 0..self.pools.len() { + if pool_idx != source_pool_idx { + let found = self + .advance_durable_ilm_decommission_receipt(pool_idx, &receipt_path, &record, terminal) + .await?; + receipt_found |= found; + if terminal + && found + && !active_source_pool_indices.contains(&pool_idx) + && !terminal_target_pool_indices.contains(&pool_idx) + { + terminal_target_pool_indices.push(pool_idx); + } + } + } + if terminal && !receipt_found { + return Err(Error::other(format!( + "terminal durable ILM record at path `{path}` {} is retained until its decommission receipt is committed", + record.context() + ))); + } + } + Ok(Some(terminal_target_pool_indices)) + } + + pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> { + self.advance_durable_ilm_decommission_receipts(path, data, false) + .await + .map(|_| ()) + } + + pub(crate) async fn record_durable_ilm_decommission_terminal(&self, path: &str, data: &[u8]) -> Result<()> { + self.record_durable_ilm_decommission_terminal_target_pools(path, data) + .await + .map(|_| ()) + } + + /// Record terminal proof and return its non-source receipt pools for targeted cleanup. + pub(crate) async fn record_durable_ilm_decommission_terminal_target_pools( + &self, + path: &str, + data: &[u8], + ) -> Result>> { + self.advance_durable_ilm_decommission_receipts(path, data, true).await + } + + async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> { + for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? { + match delete_config(self.pools[pool_idx].clone(), &receipt_path).await { + Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {} + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} + Err(err) => { + return Err(Error::other(format!( + "failed to clean durable ILM decommission receipt `{receipt_path}` from pool {pool_idx}: {err}" + ))); + } + } + } + for receipt_path in self.list_decommission_durable_ilm_manifest_receipts(source_pool_idx).await? { + match delete_config(self.pools[source_pool_idx].clone(), &receipt_path).await { + Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {} + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} + Err(err) => { + return Err(Error::other(format!( + "failed to clean durable ILM expected manifest receipt `{receipt_path}` from source pool {source_pool_idx}: {err}" + ))); + } + } + } + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + let manifest_path = decommission_durable_ilm_manifest_path(&run_token); + match delete_config(self.pools[source_pool_idx].clone(), &manifest_path).await { + Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {} + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} + Err(err) => { + return Err(Error::other(format!( + "failed to clean durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}" + ))); + } + } + Ok(()) + } + + async fn verify_and_cleanup_decommissioned_durable_ilm_record( + &self, + source_pool_idx: usize, + source_set: Arc, + path: &str, + ) -> Result<()> { + let namespace = classify_durable_ilm_record(path)? + .ok_or_else(|| Error::other(format!("path `{path}` is not a durable ILM record")))?; + let source_versions = source_set + .load_file_info_versions_exact(RUSTFS_META_BUCKET, path) + .await + .map_err(|err| Error::other(format!("failed to load source durable ILM versions at path `{path}`: {err}")))? + .ok_or_else(|| Error::other(format!("source durable ILM record is missing at path `{path}`")))?; + let source = read_config_limited_preserve_empty(source_set.clone(), path, namespace.max_record_size) + .await + .map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?; + let source_record = validate_durable_ilm_record(path, &source) + .map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?; + let target = self + .load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context()) + .await?; + let manifest_receipt = if let Some((target_pool_idx, target)) = target { + let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?; + let receipt = DecommissionDurableIlmReceipt::new(path, &target_record); + self.persist_decommission_durable_ilm_receipt(source_pool_idx, target_pool_idx, &receipt) + .await?; + receipt + } else { + self.load_decommission_durable_ilm_terminal_receipt(source_pool_idx, path, &source_record) + .await? + .ok_or_else(|| { + Error::other(format!( + "target durable ILM record is missing at path `{path}` {} without a matching terminal receipt", + source_record.context() + )) + })? + }; + self.persist_decommission_durable_ilm_receipt(source_pool_idx, source_pool_idx, &manifest_receipt) + .await?; + + let cleanup_result = data_movement::cleanup_source_entry_if_unchanged( + source_set, + RUSTFS_META_BUCKET, + path, + &source_versions, + &[], + data_movement::SourceCleanupBucketFence::default(), + "decommission durable ILM final sweep", + ) + .await + .map_err(|err| { + Error::other(format!( + "source durable ILM cleanup failed at path `{path}` {}: {err}", + source_record.context() + )) + }); + resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path) + } + + #[cfg(test)] + pub(crate) async fn verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + &self, + source_pool_idx: usize, + source_set: Arc, + path: &str, + ) -> Result<()> { + self.verify_and_cleanup_decommissioned_durable_ilm_record(source_pool_idx, source_set, path) + .await + } + + #[cfg(test)] + pub(crate) async fn decommission_durable_ilm_receipt_count_for_test(&self, source_pool_idx: usize) -> Result { + Ok(self.list_decommission_durable_ilm_receipts(source_pool_idx).await?.len()) + } + + #[cfg(test)] + pub(crate) async fn decommission_durable_ilm_receipt_paths_for_test( + &self, + source_pool_idx: usize, + ) -> Result> { + self.list_decommission_durable_ilm_receipts(source_pool_idx).await + } + + #[cfg(test)] + pub(crate) async fn persist_decommission_durable_ilm_receipt_for_test( + &self, + source_pool_idx: usize, + target_pool_idx: usize, + source_path: &str, + record: &ValidatedDurableIlmRecord, + terminal: bool, + ) -> Result { + let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record); + if terminal { + receipt.terminal_checkpoint = Some(record.checkpoint.clone()); + } + self.persist_decommission_durable_ilm_receipt(source_pool_idx, target_pool_idx, &receipt) + .await?; + let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?; + Ok(decommission_durable_ilm_receipt_path(&run_token, source_path, record.id_kind, &record.id)) + } + + #[cfg(test)] + pub(crate) async fn persist_decommission_durable_ilm_manifest_for_test(&self, source_pool_idx: usize) -> Result<()> { + self.persist_decommission_durable_ilm_manifest(source_pool_idx).await + } + + #[cfg(test)] + pub(crate) async fn cleanup_decommission_durable_ilm_receipts_for_test(&self, source_pool_idx: usize) -> Result<()> { + self.cleanup_decommission_durable_ilm_receipts(source_pool_idx).await + } + + async fn check_after_decommission( + self: &Arc, + idx: usize, + rx: &CancellationToken, + generation: OffsetDateTime, + ) -> Result<()> { + self.ensure_decommission_generation_current(idx, generation).await?; + let operation_gate = self.ctx.decommission_operation_gate(); + run_decommission_side_effect(rx, &operation_gate, || self.check_after_decommission_unfenced(idx)).await + } + + async fn check_after_decommission_unfenced(self: &Arc, idx: usize) -> Result<()> { let buckets = self.get_buckets_to_decommission().await?; let pool = self.pools[idx].clone(); @@ -5164,22 +6547,27 @@ impl ECStore { let versions_found = Arc::new(AtomicUsize::new(0)); let entry_error = Arc::new(tokio::sync::Mutex::new(None::)); + let first_remaining_path = Arc::new(tokio::sync::Mutex::new(None::)); let callback_rx = CancellationToken::new(); let versions_found_cb = versions_found.clone(); let entry_error_cb = entry_error.clone(); + let first_remaining_path_cb = first_remaining_path.clone(); let bucket_name = bucket_info.name.clone(); let lifecycle_config_cb = lifecycle_config.clone(); let object_lock_config_cb = object_lock_config.clone(); let store = Arc::clone(self); + let source_set = set.clone(); let callback_rx_cb = callback_rx.clone(); let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| { let versions_found = versions_found_cb.clone(); let entry_error = entry_error_cb.clone(); + let first_remaining_path = first_remaining_path_cb.clone(); let bucket_name = bucket_name.clone(); let lifecycle_config = lifecycle_config_cb.clone(); let object_lock_config = object_lock_config_cb.clone(); let store = Arc::clone(&store); + let source_set = source_set.clone(); let callback_rx = callback_rx_cb.clone(); Box::pin(async move { if callback_rx.is_cancelled() { @@ -5194,6 +6582,41 @@ impl ECStore { return; } + let durable_ilm_record = if bucket_name == RUSTFS_META_BUCKET { + match classify_durable_ilm_record(&entry.name) { + Ok(record) => record, + Err(err) => { + let mut first_err = entry_error.lock().await; + if first_err.is_none() { + *first_err = Some(with_decommission_entry_context( + "check_after_decommission.durable_ilm_namespace", + &bucket_name, + &entry.name, + err, + )); + callback_rx.cancel(); + } + return; + } + } + } else { + None + }; + + if durable_ilm_record.is_some() { + if let Err(err) = store + .verify_and_cleanup_decommissioned_durable_ilm_record(idx, source_set, &entry.name) + .await + { + let mut first_err = entry_error.lock().await; + if first_err.is_none() { + *first_err = Some(err); + callback_rx.cancel(); + } + } + return; + } + let fivs = match load_decommission_entry_versions( &entry, &bucket_name, @@ -5242,6 +6665,13 @@ impl ECStore { remaining += 1; } + if remaining > 0 { + let mut first_path = first_remaining_path.lock().await; + if first_path.is_none() { + *first_path = Some(format!("{bucket_name}/{}", entry.name)); + } + } + versions_found.fetch_add(remaining, Ordering::Relaxed); }) }); @@ -5254,17 +6684,32 @@ impl ECStore { let versions_found = versions_found.load(Ordering::Relaxed); if versions_found > 0 { + let first_remaining_path = first_remaining_path + .lock() + .await + .clone() + .unwrap_or_else(|| format!("{}/", bucket_info.name)); return Err(Error::other(format!( - "at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning", - bucket_info.name + "at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning; first remaining path `{first_remaining_path}`", + bucket_info.name, ))); } } } + self.persist_decommission_durable_ilm_manifest(idx).await?; + self.verify_decommission_durable_ilm_receipts(idx).await?; + Ok(()) } + #[cfg(test)] + pub(crate) async fn check_after_decommission_for_test(self: &Arc, idx: usize) -> Result<()> { + let generation = self.active_decommission_generation(idx).await?; + self.check_after_decommission(idx, &CancellationToken::new(), generation) + .await + } + #[tracing::instrument(skip(self, rd))] async fn decommission_object( self: Arc, @@ -6357,14 +7802,18 @@ mod pools_tests { use super::DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF; use super::record_decommission_entry_error; use super::resolve_decommission_listing_error; + use super::resolve_decommission_partial_listing_entry; use super::{ - DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, + DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP, DECOMMISSION_META_PREFIXES, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo, DecommissionCanceler, - DecommissionEntryEnqueueResult, DecommissionStartPoolState, DecommissionTerminalState, ListCallback, - PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, apply_decommission_status_space_info, - await_decommission_worker, bind_decommission_cancelers, bind_missing_decommission_cancelers, - cancel_decommission_canceler, clamp_decommission_entry_concurrency, classify_decommission_terminal_state, - count_decommission_item, decommission_cancel_signal_result, decommission_entry_queue_capacity, decommission_item_size, + DecommissionDurableIlmReceipt, DecommissionEntryEnqueueResult, DecommissionStartPoolState, DecommissionTerminalState, + ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, + apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers, + bind_missing_decommission_cancelers, cancel_decommission_canceler, clamp_decommission_entry_concurrency, + classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, + decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix, + decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size, decommission_meta_bucket_options, decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency, default_decommission_entry_concurrency, drain_decommission_entry_queue, enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_generation, @@ -6375,25 +7824,32 @@ mod pools_tests { ensure_local_decommission_pool_leaders, ensure_valid_decommission_pool_index, first_resumable_decommission_queue_indices, get_by_index, guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested, load_decommission_entry_versions, local_decommission_queue_prefix, - mark_decommission_bucket_done, merge_pool_status_refresh, missing_decommission_worker_prefix, - observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, require_decommission_store, - reserve_decommission_start_cancelers, resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state, + mark_decommission_bucket_done, merge_decommission_durable_ilm_receipts, merge_pool_status_refresh, + missing_decommission_worker_prefix, observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, + reconcile_decommission_meta_buckets, require_decommission_store, reserve_decommission_start_cancelers, + resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state, resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result, resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result, resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result, - resolve_decommission_partial_listing_entry, resolve_decommission_pool_meta_reload_result, - resolve_decommission_preflight_heal_result, resolve_decommission_progress_save_result, - resolve_decommission_terminal_mark_after_error_result, resolve_decommission_terminal_mark_result, - resolve_decommission_update_after_result, resolve_start_decommission_pool_meta_reload_result, - rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry, - run_decommission_listing_with_retry_and_drain, run_decommission_side_effect, should_cleanup_decommission_source_entry, - should_continue_decommission_queue, should_count_decommission_version_complete, - should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal, - should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine, - spawn_decommission_index_cancelers, split_decommission_buckets, take_and_cancel_decommission_canceler, - take_decommission_canceler, track_decommission_current_object, track_decommission_current_object_stage, - update_decommission_for_operation, validate_start_decommission_request, wait_decommission_listing_retry, - wait_decommission_worker_drain, with_decommission_entry_context, + resolve_decommission_pool_meta_reload_result, resolve_decommission_preflight_heal_result, + resolve_decommission_progress_save_result, resolve_decommission_terminal_mark_after_error_result, + resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result, + resolve_start_decommission_pool_meta_reload_result, rollback_start_decommission_pool_meta, + run_decommission_buckets_bounded, run_decommission_listing_with_retry, run_decommission_listing_with_retry_and_drain, + run_decommission_side_effect, should_cleanup_decommission_source_entry, should_continue_decommission_queue, + should_count_decommission_version_complete, should_preserve_decommission_canceled_state, + should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload, + should_retry_decommission_listing, should_skip_canceled_decommission_routine, spawn_decommission_index_cancelers, + split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler, + track_decommission_current_object, track_decommission_current_object_stage, update_decommission_for_operation, + validate_start_decommission_request, wait_decommission_listing_retry, wait_decommission_worker_drain, + with_decommission_entry_context, + }; + use crate::bucket::lifecycle::{ + DurableIlmRecordCheckpoint, + bucket_lifecycle_ops::{ManualTransitionQueueSnapshot, ManualTransitionRunOptions}, + manual_transition_job::{ManualTransitionJobRecord, manual_transition_job_record_object_name}, + validate_durable_ilm_record, }; use crate::data_movement; use crate::disk::endpoint::Endpoint; @@ -6462,6 +7918,93 @@ mod pools_tests { } } + #[test] + fn decommission_receipt_run_token_changes_with_persisted_start_time() { + let first = OffsetDateTime::from_unix_timestamp(1_000).expect("first run timestamp should be valid"); + let second = OffsetDateTime::from_unix_timestamp(2_000).expect("second run timestamp should be valid"); + let first_token = decommission_durable_ilm_receipt_run_token("pool-0", first); + let second_token = decommission_durable_ilm_receipt_run_token("pool-0", second); + + assert_ne!(first_token, second_token); + assert_eq!(first_token, decommission_durable_ilm_receipt_run_token("pool-0", first)); + let operation_id = "a".repeat(64); + let old_receipt = decommission_durable_ilm_receipt_path( + &first_token, + &format!("ilm/tier-delete-journal/{operation_id}.json"), + "operation_id", + &operation_id, + ); + assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token))); + } + + #[test] + fn decommission_receipt_merge_preserves_terminal_proof() { + let operation_id = "a".repeat(64); + let source_path = format!("ilm/tier-delete-journal/{operation_id}.json"); + let checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal { + content_sha256: "b".repeat(64), + identity_sha256: "c".repeat(64), + committed: false, + }; + let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal { + content_sha256: "d".repeat(64), + identity_sha256: "c".repeat(64), + committed: true, + }; + let incoming = DecommissionDurableIlmReceipt { + source_path, + namespace: "tier-delete-journal".to_string(), + id_kind: "operation_id".to_string(), + id: operation_id, + checkpoint: checkpoint.clone(), + terminal_checkpoint: None, + }; + let existing = DecommissionDurableIlmReceipt { + terminal_checkpoint: Some(terminal_checkpoint.clone()), + ..incoming.clone() + }; + + let merged = merge_decommission_durable_ilm_receipts(&existing, &incoming) + .expect("retry receipt must merge with a terminal receipt"); + + assert_eq!(merged.checkpoint, checkpoint); + assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint)); + } + + #[test] + fn decommission_manual_job_receipt_compacts_large_progress() { + let prefix = "p".repeat(12 * 1024); + let options = ManualTransitionRunOptions { + prefix, + ..Default::default() + }; + let mut job = ManualTransitionJobRecord::new(uuid::Uuid::new_v4(), "bounded-receipt-bucket", &options, "owner"); + let token_bytes = serde_json::to_vec(&serde_json::json!({ + "marker": "m".repeat(12 * 1024), + "version_marker": "opaque-version" + })) + .expect("large continuation token should encode"); + let mut report = job.report.clone(); + report.scanned = 1; + report.continuation_token = Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&token_bytes)); + job.update_running_progress(report, ManualTransitionQueueSnapshot::default()); + let path = manual_transition_job_record_object_name(job.job_id).expect("manual job path should build"); + let job_bytes = job.encode().expect("large manual job should remain within its record limit"); + assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE); + let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate"); + let expected_checkpoint = record.checkpoint.clone(); + let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record); + receipt.terminal_checkpoint = Some(record.checkpoint); + + let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit"); + let decoded = DecommissionDurableIlmReceipt::decode(&encoded).expect("bounded receipt should round trip"); + + assert!(encoded.len() <= DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE); + assert_eq!(decoded.source_path, path); + assert_eq!(decoded.checkpoint, expected_checkpoint); + assert_eq!(decoded.terminal_checkpoint, Some(expected_checkpoint)); + } + #[test] fn test_apply_decommission_status_space_info_adds_idle_pool_usage() { let status = apply_decommission_status_space_info( @@ -6736,6 +8279,10 @@ mod pools_tests { name: crate::disk::RUSTFS_META_BUCKET.to_string(), prefix: crate::disk::BUCKET_META_PREFIX.to_string(), }, + DecomBucketInfo { + name: crate::disk::RUSTFS_META_BUCKET.to_string(), + prefix: crate::bucket::lifecycle::ILM_META_PREFIX.to_string(), + }, ]); assert_eq!( @@ -6744,10 +8291,42 @@ mod pools_tests { ); assert_eq!( meta.iter().map(|bucket| bucket.prefix.as_str()).collect::>(), - vec![crate::config::com::CONFIG_PREFIX, crate::disk::BUCKET_META_PREFIX,] + vec![ + crate::config::com::CONFIG_PREFIX, + crate::disk::BUCKET_META_PREFIX, + crate::bucket::lifecycle::ILM_META_PREFIX, + ] ); } + #[test] + fn test_resume_reconciles_missing_decommission_meta_prefixes() { + let mut meta = PoolMeta { + pools: vec![decommission_test_pool_status( + 0, + Some(PoolDecommissionInfo { + queued_buckets: vec![ + format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::config::com::CONFIG_PREFIX), + format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::disk::BUCKET_META_PREFIX), + ], + ..Default::default() + }), + )], + ..Default::default() + }; + + assert!(reconcile_decommission_meta_buckets(&mut meta, 0)); + assert_eq!( + meta.pending_buckets(0) + .iter() + .filter(|bucket| bucket.name == crate::disk::RUSTFS_META_BUCKET) + .map(|bucket| bucket.prefix.as_str()) + .collect::>(), + DECOMMISSION_META_PREFIXES + ); + assert!(!reconcile_decommission_meta_buckets(&mut meta, 0)); + } + #[tokio::test] async fn test_run_decommission_buckets_bounded_respects_limit() { let rx = CancellationToken::new(); diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index bb3c9fbf9..1ceb8acdf 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -37,7 +37,7 @@ use crate::bucket::lifecycle::{ transition_transaction::{ TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record, - save_transition_transaction_record, + load_transition_transaction_record, save_transition_transaction_record, }, }; use crate::bucket::quota::reservation; @@ -4248,7 +4248,12 @@ fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction, async fn delete_transition_transaction_if_available(api: Option<&Arc>, transaction_id: Uuid) -> Result<()> { if let Some(api) = api { - return delete_transition_transaction_record(api.clone(), transaction_id).await; + let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await { + Ok(transaction) => transaction, + Err(Error::ConfigNotFound) => return Ok(()), + Err(err) => return Err(err), + }; + return delete_transition_transaction_record(api.clone(), &transaction).await; } Ok(()) } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 451e42242..1d143201c 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -555,10 +555,18 @@ mod tests { #[cfg(feature = "test-util")] use crate::{ bucket::lifecycle::{ + DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, + bucket_lifecycle_ops::{ManualTransitionRunOptions, recover_manual_transition_jobs_once}, lifecycle::{TRANSITION_PENDING, TransitionOptions}, + manual_transition_job::{ + ManualTransitionJobRecord, ManualTransitionScopeAdmission, ManualTransitionTaskRecord, + ManualTransitionWorkerResult, ManualTransitionWorkerResultRecord, manual_transition_job_record_object_name, + manual_transition_scope_record_object_name, manual_transition_task_object_name, + manual_transition_worker_result_object_name, manual_transition_worker_result_task_key, + }, tier_delete_journal::{ - TIER_DELETE_JOURNAL_PREFIX, persist_tier_delete_journal_entry, recover_tier_delete_journal_entries, - tier_delete_journal_object_name, + TIER_DELETE_JOURNAL_PREFIX, encode_tier_delete_journal_entry, persist_tier_delete_journal_entry, + recover_tier_delete_journal_entries, tier_delete_journal_object_name, }, tier_sweeper::{ Jentry, TierDeleteJournalState, TierDeleteSourceIdentity, transitioned_delete_journal_entry_for_source, @@ -570,12 +578,16 @@ mod tests { delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator, load_transition_transaction_record, recover_transition_transaction_records, save_transition_transaction_record, + transition_transaction_record_object_name, }, + validate_durable_ilm_record, }, bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG}, client::transition_api::ReaderImpl, config::com, - disk::{RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE}, + core::pools::DecomBucketInfo, + data_movement::SourceCleanupDeleteBarrier, + disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE}, runtime::{global::set_object_store_resolver, sources as runtime_sources}, services::tier::{ test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier}, @@ -608,6 +620,8 @@ mod tests { range::HTTPRangeSpec, }, }; + #[cfg(feature = "test-util")] + use futures::{StreamExt as _, TryStreamExt as _}; use http::HeaderMap; use rustfs_config::server_config::KVS; #[cfg(feature = "test-util")] @@ -4386,6 +4400,1079 @@ mod tests { )); } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-target-read-error", &[4, 4, 4])) + .await; + let job_id = uuid::Uuid::new_v4(); + let job = + ManualTransitionJobRecord::new(job_id, "manual-target-read-error", &ManualTransitionRunOptions::default(), "owner"); + let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build"); + let data = job.encode().expect("manual job should encode"); + for pool in &store.pools { + com::save_config(pool.clone(), &path, data.clone()) + .await + .expect("manual job fixture should persist in every pool"); + } + store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + + let failing_target = store.pools[2].get_disks_by_key(&path); + let original_disks = { + let mut disks = failing_target.disks.write().await; + let original = disks.clone(); + for disk in disks.iter_mut().take(3) { + *disk = None; + } + original + }; + let error = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, store.pools[0].get_disks_by_key(&path), &path) + .await + .expect_err("one target read-quorum error must fail closed despite another target success") + .to_string(); + *failing_target.disks.write().await = original_disks; + + assert!(error.contains(&path)); + assert!(error.contains("pool 2")); + assert_eq!( + com::read_config(store.pools[0].clone(), &path) + .await + .expect("target read error must retain the source"), + data + ); + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("failed target verification should not create a receipt"), + 0 + ); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-terminal-receipt", &[4, 4])).await; + let tier_name = "DECOMMISSION-RECEIPT"; + let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("tier lease should resolve") + .backend_identity(); + let entry = Jentry { + obj_name: "receipt-recovery-object".to_string(), + version_id: "receipt-recovery-version".to_string(), + tier_name: tier_name.to_string(), + backend_identity: Some(backend_identity), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + state: TierDeleteJournalState::Committed, + source: None, + }; + let path = tier_delete_journal_object_name(&entry); + let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode"); + com::save_config(store.pools[0].clone(), &path, data.clone()) + .await + .expect("source tier journal should persist"); + com::save_config(store.pools[1].clone(), &path, data.clone()) + .await + .expect("target tier journal should persist"); + let active_pool_meta = { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + pool_meta.clone() + }; + active_pool_meta + .save(store.pools.clone()) + .await + .expect("active decommission run identity should persist"); + + let source_set = store.pools[0].get_disks_by_key(&path); + let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path); + let cleanup_store = store.clone(); + let cleanup_set = source_set.clone(); + let cleanup_path = path.clone(); + let cleanup = tokio::spawn(async move { + cleanup_store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, cleanup_set, &cleanup_path) + .await + }); + barrier.wait_until_paused().await; + let original_source_disks = { + let mut disks = source_set.disks.write().await; + let original = disks.clone(); + for disk in disks.iter_mut().take(3) { + *disk = None; + } + original + }; + barrier.release(); + let cleanup_error = cleanup + .await + .expect("source cleanup task should not panic") + .expect_err("injected source delete quorum failure must fail cleanup") + .to_string(); + *source_set.disks.write().await = original_source_disks; + drop(barrier); + + assert!(cleanup_error.contains("source durable ILM cleanup failed")); + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("receipt should persist before source cleanup"), + 1 + ); + assert_eq!( + com::read_config(store.pools[0].clone(), &path) + .await + .expect("failed cleanup must retain the source"), + data + ); + + let mut restarted_pool_meta = PoolMeta::default(); + restarted_pool_meta + .load(store.pools[0].clone(), store.pools.clone()) + .await + .expect("decommission run identity should reload after restart"); + *store.pool_meta.write().await = restarted_pool_meta; + let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("target recovery should commit terminal proof and delete the target"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0)); + assert!(matches!( + com::read_config(store.pools[1].clone(), &path).await, + Err(Error::ConfigNotFound) + )); + assert_eq!( + com::read_config(store.pools[0].clone(), &path) + .await + .expect("target recovery must not delete the decommission source"), + data + ); + + store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test(0, source_set, &path) + .await + .expect("terminal receipt should authorize cleanup after target deletion"); + assert!(matches!( + com::read_config(store.pools[0].clone(), &path).await, + Err(Error::ConfigNotFound) + )); + assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id))); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_final_sweep_blocks_cancel_until_source_cleanup_finishes() { + let temp_dir = tempfile::tempdir().expect("create final sweep gate store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-final-sweep-gate", &[4, 4])).await; + let job_id = uuid::Uuid::new_v4(); + let job = ManualTransitionJobRecord::new(job_id, "final-sweep-gate", &ManualTransitionRunOptions::default(), "owner"); + let path = manual_transition_job_record_object_name(job_id).expect("manual job path should build"); + let data = job.encode().expect("manual job should encode"); + for pool in &store.pools { + com::save_config(pool.clone(), &path, data.clone()) + .await + .expect("manual job fixture should persist in both pools"); + } + let active_pool_meta = { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + pool_meta.clone() + }; + active_pool_meta + .save(store.pools.clone()) + .await + .expect("active decommission run identity should persist"); + + let barrier = SourceCleanupDeleteBarrier::install(RUSTFS_META_BUCKET, &path); + let final_sweep = tokio::spawn({ + let store = store.clone(); + async move { store.check_after_decommission_for_test(0).await } + }); + barrier.wait_until_paused().await; + + let mut cancel = tokio::spawn({ + let store = store.clone(); + async move { store.decommission_cancel(0).await } + }); + assert!( + tokio::time::timeout(Duration::from_millis(100), &mut cancel).await.is_err(), + "cancel must wait for the final sweep source cleanup" + ); + { + let pool_meta = store.pool_meta.read().await; + let decommission = pool_meta.pools[0] + .decommission + .as_ref() + .expect("decommission state should remain present"); + assert!( + !decommission.canceled, + "cancel must not publish terminal state before the final sweep drains" + ); + assert!( + decommission.start_time.is_some(), + "cancel must preserve the run identity until the final sweep drains" + ); + } + + barrier.release(); + final_sweep + .await + .expect("final sweep task should not panic") + .expect("final sweep should finish after the barrier releases"); + cancel + .await + .expect("cancel task should not panic") + .expect("cancel should complete after the final sweep releases the operation gate"); + let pool_meta = store.pool_meta.read().await; + let decommission = pool_meta.pools[0] + .decommission + .as_ref() + .expect("decommission state should remain present"); + assert!(decommission.canceled); + assert!(decommission.start_time.is_none()); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_durable_ilm_recovery_keeps_multiple_active_sources() { + let temp_dir = tempfile::tempdir().expect("create multi-source recovery store dir"); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "durable-ilm-multi-source-recovery", + &[4, 4, 4], + )) + .await; + let tier_name = "DECOMMISSION-MULTI-SOURCE"; + let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("tier lease should resolve") + .backend_identity(); + let entry = Jentry { + obj_name: "multi-source-recovery-object".to_string(), + version_id: "multi-source-recovery-version".to_string(), + tier_name: tier_name.to_string(), + backend_identity: Some(backend_identity), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + state: TierDeleteJournalState::Committed, + source: None, + }; + let path = tier_delete_journal_object_name(&entry); + let data = encode_tier_delete_journal_entry(&entry).expect("tier journal should encode"); + for pool in &store.pools { + com::save_config(pool.clone(), &path, data.clone()) + .await + .expect("source and target tier journals should persist"); + } + + let active_pool_meta = { + let mut pool_meta = store.pool_meta.write().await; + let start_time = OffsetDateTime::now_utc(); + for pool_idx in [0, 1] { + pool_meta.pools[pool_idx].decommission = Some(PoolDecommissionInfo { + start_time: Some(start_time), + ..Default::default() + }); + } + pool_meta.clone() + }; + active_pool_meta + .save(store.pools.clone()) + .await + .expect("multiple active decommission runs should persist"); + let mut restarted_pool_meta = PoolMeta::default(); + restarted_pool_meta + .load(store.pools[0].clone(), store.pools.clone()) + .await + .expect("multiple active decommission runs should reload"); + *store.pool_meta.write().await = restarted_pool_meta; + + let record = validate_durable_ilm_record(&path, &data).expect("tier journal should validate"); + let source_zero_receipt = store + .persist_decommission_durable_ilm_receipt_for_test(0, 1, &path, &record, false) + .await + .expect("source pool zero receipt should persist on the other active source"); + let source_one_receipt = store + .persist_decommission_durable_ilm_receipt_for_test(1, 0, &path, &record, false) + .await + .expect("source pool one receipt should persist on the other active source"); + assert_ne!( + source_zero_receipt, source_one_receipt, + "active source runs must have distinct receipt paths" + ); + + let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("cross-source receipts should not remove active source journals"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0)); + for pool in &store.pools { + assert_eq!( + com::read_config(pool.clone(), &path) + .await + .expect("cross-source receipts alone must retain every journal copy"), + data + ); + } + + store + .persist_decommission_durable_ilm_receipt_for_test(0, 2, &path, &record, false) + .await + .expect("source pool zero receipt should persist on the target"); + store + .persist_decommission_durable_ilm_receipt_for_test(1, 2, &path, &record, false) + .await + .expect("source pool one receipt should persist on the target"); + + let stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("multi-source tier journal recovery should complete"); + assert_eq!((stats.scanned, stats.deleted, stats.failed), (1, 1, 0)); + assert_eq!( + com::read_config(store.pools[0].clone(), &path) + .await + .expect("first active source must remain after target recovery"), + data + ); + assert_eq!( + com::read_config(store.pools[1].clone(), &path) + .await + .expect("second active source must remain after target recovery"), + data + ); + assert!(matches!( + com::read_config(store.pools[2].clone(), &path).await, + Err(Error::ConfigNotFound) + )); + assert!(backend.remove_versions().await.contains(&(entry.obj_name, entry.version_id))); + } + + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page() { + const RECEIPT_COUNT: usize = 1001; + + let temp_dir = tempfile::tempdir().expect("create paginated receipt store dir"); + let (_ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-receipt-pages", &[4, 4])).await; + store.pool_meta.write().await.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + + futures::stream::iter(0..RECEIPT_COUNT) + .map(|index| { + let store = store.clone(); + async move { + let id = format!("{index:064x}"); + let source_path = format!("ilm/tier-delete-journal/{id}.json"); + let record = ValidatedDurableIlmRecord { + namespace: "tier-delete-journal", + id_kind: "operation_id", + id, + checkpoint: DurableIlmRecordCheckpoint::TierDeleteJournal { + content_sha256: format!("{:064x}", index + RECEIPT_COUNT), + identity_sha256: "f".repeat(64), + committed: false, + }, + }; + store + .persist_decommission_durable_ilm_receipt_for_test(0, 0, &source_path, &record, true) + .await?; + store + .persist_decommission_durable_ilm_receipt_for_test(0, 1, &source_path, &record, true) + .await?; + Ok::<(), Error>(()) + } + }) + .buffer_unordered(32) + .try_collect::>() + .await + .expect("more than one receipt page should persist"); + store + .persist_decommission_durable_ilm_manifest_for_test(0) + .await + .expect("paginated source receipts should produce a manifest"); + + let target_receipts = store + .decommission_durable_ilm_receipt_paths_for_test(0) + .await + .expect("paginated target receipts should list"); + assert_eq!(target_receipts.len(), RECEIPT_COUNT); + let (target_pool_idx, second_page_path) = target_receipts + .get(1000) + .cloned() + .expect("the real 1000-item page boundary should expose a second page receipt"); + let receipt_bytes = com::read_config(store.pools[target_pool_idx].clone(), &second_page_path) + .await + .expect("second page receipt should be readable"); + + com::delete_config(store.pools[target_pool_idx].clone(), &second_page_path) + .await + .expect("second page receipt should delete"); + let missing = store + .complete_decommission(0) + .await + .expect_err("a missing second page receipt must block completion") + .to_string(); + assert!(missing.contains(&second_page_path)); + assert!( + !store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .expect("source pool should remain in decommission") + .complete + ); + assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok()); + + com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone()) + .await + .expect("second page receipt should restore"); + com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec()) + .await + .expect("second page receipt should corrupt deterministically"); + let corrupt = store + .complete_decommission(0) + .await + .expect_err("a corrupt second page receipt must block completion") + .to_string(); + assert!(corrupt.contains(&second_page_path)); + assert!(corrupt.contains("invalid")); + assert!( + !store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .expect("source pool should remain in decommission") + .complete + ); + assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok()); + } + + #[cfg(feature = "test-util")] + #[test] + #[serial_test::serial(storage_class_env)] + fn decommission_migrates_and_verifies_registered_durable_ilm_records() { + std::thread::Builder::new() + .name("durable-ilm-decommission-test".to_string()) + .stack_size(16 * 1024 * 1024) + .spawn(|| { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(2) + .build() + .expect("durable ILM decommission runtime should build"); + runtime.block_on(decommission_migrates_and_verifies_registered_durable_ilm_records_scenario()); + }) + .expect("durable ILM decommission scenario thread should spawn") + .join() + .expect("durable ILM decommission scenario should not panic"); + } + + #[cfg(feature = "test-util")] + async fn decommission_migrates_and_verifies_registered_durable_ilm_records_scenario() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "durable-ilm-decommission", &[4, 4])).await; + + let tier_name = "DECOMMISSION-ILM"; + let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("tier lease should resolve") + .backend_identity(); + let tier_entry = Jentry { + obj_name: "decommissioned-remote-object".to_string(), + version_id: "decommissioned-remote-version".to_string(), + tier_name: tier_name.to_string(), + backend_identity: Some(backend_identity), + version_id_exact: true, + version_state: rustfs_filemeta::TransitionVersionState::Exact, + state: TierDeleteJournalState::Committed, + source: None, + }; + let tier_path = tier_delete_journal_object_name(&tier_entry); + let tier_bytes = encode_tier_delete_journal_entry(&tier_entry).expect("tier journal should encode"); + + let mut transaction = TransitionTransaction::new(TransitionTransactionInit { + deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"), + transaction_id: uuid::Uuid::new_v4(), + owner_epoch: uuid::Uuid::new_v4(), + write_id: uuid::Uuid::new_v4(), + source: TransitionSourceIdentity { + bucket: "source-bucket".to_string(), + object: "source-object".to_string(), + version_id: Some(uuid::Uuid::new_v4()), + data_dir: uuid::Uuid::new_v4(), + mod_time_unix_nanos: 1_770_000_000_000_000_000, + size: 42, + etag: "source-etag".to_string(), + version_mode: TransitionSourceVersionMode::Versioned, + }, + tier_name: tier_name.to_string(), + backend_fingerprint: backend_identity, + not_after_unix_nanos: 1_780_000_000_000_000_000, + }) + .expect("transition transaction should build"); + let regressed_transaction = transaction.clone(); + transaction + .advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None) + .expect("transition transaction should advance before migration"); + let transaction_path = transition_transaction_record_object_name(transaction.transaction_id) + .expect("transition transaction path should build"); + let transaction_bytes = transaction.encode().expect("transition transaction should encode"); + + let manual_job_id = uuid::Uuid::new_v4(); + let manual_bucket = format!("manual-decommission-{}", manual_job_id.simple()); + let manual_options = ManualTransitionRunOptions { + prefix: "logs/".to_string(), + tier: Some(tier_name.to_string()), + ..Default::default() + }; + let mut manual_job = ManualTransitionJobRecord::new(manual_job_id, &manual_bucket, &manual_options, "old-owner"); + manual_job.scan_completed = true; + manual_job.report.enqueued = 1; + manual_job.lease_expires_at_unix_nanos = 0; + let manual_scope = ManualTransitionScopeAdmission::from_job(&manual_job); + let task_key = manual_transition_worker_result_task_key(&manual_bucket, "logs/a", None); + let manual_task = ManualTransitionTaskRecord::new(manual_job_id, &task_key, &manual_bucket, "logs/a", None, tier_name); + let manual_result = + ManualTransitionWorkerResultRecord::new(manual_job_id, &task_key, ManualTransitionWorkerResult::Completed); + + let manual_job_path = manual_transition_job_record_object_name(manual_job_id).expect("manual job path should build"); + let manual_scope_path = + manual_transition_scope_record_object_name(&manual_scope.scope_key).expect("manual scope path should build"); + let manual_task_path = + manual_transition_task_object_name(manual_job_id, &task_key).expect("manual task path should build"); + let manual_result_path = manual_transition_worker_result_object_name(manual_job_id, &task_key) + .expect("manual worker result path should build"); + let manual_job_bytes = manual_job.encode().expect("manual job should encode"); + let manual_scope_bytes = serde_json::to_vec(&manual_scope).expect("manual scope should encode"); + let manual_task_bytes = manual_task.encode().expect("manual task should encode"); + let manual_result_bytes = manual_result.encode().expect("manual result should encode"); + + let records = vec![ + (tier_path.clone(), tier_bytes.clone()), + (transaction_path.clone(), transaction_bytes.clone()), + (manual_job_path.clone(), manual_job_bytes.clone()), + (manual_scope_path.clone(), manual_scope_bytes.clone()), + (manual_task_path.clone(), manual_task_bytes.clone()), + (manual_result_path.clone(), manual_result_bytes.clone()), + ]; + for (path, data) in &records { + com::save_config(store.pools[0].clone(), path, data.clone()) + .await + .expect("durable ILM source record should persist"); + } + + let legacy_queue = [com::CONFIG_PREFIX, BUCKET_META_PREFIX] + .into_iter() + .map(|prefix| { + DecomBucketInfo { + name: RUSTFS_META_BUCKET.to_string(), + prefix: prefix.to_string(), + } + .to_string() + }) + .collect(); + let legacy_pool_meta = { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + queued: true, + queued_buckets: legacy_queue, + ..Default::default() + }); + pool_meta.clone() + }; + legacy_pool_meta + .save(store.pools.clone()) + .await + .expect("legacy decommission queue should persist before restart"); + let mut restarted_pool_meta = PoolMeta::default(); + restarted_pool_meta + .load(store.pools[0].clone(), store.pools.clone()) + .await + .expect("legacy decommission queue should reload after restart"); + *store.pool_meta.write().await = restarted_pool_meta; + store + .promote_queued_decommission_for_test(0) + .await + .expect("legacy queued decommission should resume"); + let expected_ilm_queue = DecomBucketInfo { + name: RUSTFS_META_BUCKET.to_string(), + prefix: ILM_META_PREFIX.to_string(), + } + .to_string(); + { + let pool_meta = store.pool_meta.read().await; + let decommission = pool_meta.pools[0] + .decommission + .as_ref() + .expect("decommission state should remain present"); + assert!(!decommission.queued); + assert!(decommission.queued_buckets.contains(&expected_ilm_queue)); + } + + let ilm_bucket = DecomBucketInfo { + name: RUSTFS_META_BUCKET.to_string(), + prefix: ILM_META_PREFIX.to_string(), + }; + for _ in 0..2 { + store + .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket.clone()) + .await + .expect("durable ILM decommission should be idempotent"); + } + for (path, expected) in &records { + assert_eq!( + com::read_config(store.pools[0].clone(), path) + .await + .expect("source should remain until the final sweep"), + *expected + ); + assert_eq!( + com::read_config(store.pools[1].clone(), path) + .await + .expect("target should contain the migrated record"), + *expected + ); + } + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("no receipt should exist before the final sweep"), + 0 + ); + let isolated_tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("tier recovery should retain a terminal record until its receipt is committed"); + assert!(isolated_tier_stats.scanned >= 1); + assert_eq!(isolated_tier_stats.deleted, 0); + assert!(isolated_tier_stats.failed >= 1); + assert_eq!( + com::read_config(store.pools[1].clone(), &tier_path) + .await + .expect("receipt isolation must retain the target tier journal"), + tier_bytes + ); + + com::delete_config(store.pools[1].clone(), &manual_job_path) + .await + .expect("target manual job should delete"); + let missing = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&manual_job_path), + &manual_job_path, + ) + .await + .expect_err("missing target must block source cleanup"); + let missing = missing.to_string(); + assert!(missing.contains(&manual_job_path) && missing.contains(&manual_job_id.to_string())); + assert_eq!( + com::read_config(store.pools[0].clone(), &manual_job_path) + .await + .expect("missing target must retain source"), + manual_job_bytes + ); + com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone()) + .await + .expect("target manual job should restore"); + + com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec()) + .await + .expect("target transaction should corrupt deterministically"); + let corrupt = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&transaction_path), + &transaction_path, + ) + .await + .expect_err("corrupt target must block source cleanup"); + let corrupt = corrupt.to_string(); + assert!(corrupt.contains(&transaction_path) && corrupt.contains(&transaction.transaction_id.to_string())); + assert_eq!( + com::read_config(store.pools[0].clone(), &transaction_path) + .await + .expect("corrupt target must retain source"), + transaction_bytes + ); + com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone()) + .await + .expect("target transaction should restore"); + + com::save_config(store.pools[1].clone(), &manual_scope_path, manual_scope_bytes.clone()) + .await + .expect("target scope rewrite should invalidate cached metadata before the quorum check"); + let target_scope_set = store.pools[1].get_disks_by_key(&manual_scope_path); + let original_target_scope_disks = { + let mut disks = target_scope_set.disks.write().await; + let original = disks.clone(); + for disk in disks.iter_mut().take(3) { + *disk = None; + } + original + }; + let quorum_error = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&manual_scope_path), + &manual_scope_path, + ) + .await + .expect_err("target below read quorum must block source cleanup"); + *target_scope_set.disks.write().await = original_target_scope_disks; + let quorum_error = quorum_error.to_string(); + assert!(quorum_error.contains(&manual_scope_path) && quorum_error.contains(&manual_job_id.to_string())); + assert!(com::read_config(store.pools[0].clone(), &manual_scope_path).await.is_ok()); + + com::save_config(store.pools[1].clone(), &manual_task_path, manual_task_bytes.clone()) + .await + .expect("target task rewrite should invalidate cached metadata before the quorum check"); + let target_task_set = store.pools[1].get_disks_by_key(&manual_task_path); + let original_target_task_disks = { + let mut disks = target_task_set.disks.write().await; + let original = disks.clone(); + for disk in disks.iter_mut().take(2) { + *disk = None; + } + original + }; + let receipt_quorum_error = store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&manual_task_path), + &manual_task_path, + ) + .await + .expect_err("target read quorum without receipt write quorum must retain the source"); + *target_task_set.disks.write().await = original_target_task_disks; + let receipt_quorum_error = receipt_quorum_error.to_string(); + assert!(receipt_quorum_error.contains("receipt")); + assert!(receipt_quorum_error.contains(&manual_task_path)); + assert!(receipt_quorum_error.contains(&manual_job_id.to_string())); + assert!(com::read_config(store.pools[0].clone(), &manual_task_path).await.is_ok()); + store + .verify_and_cleanup_decommissioned_durable_ilm_record_for_test( + 0, + store.pools[0].get_disks_by_key(&manual_task_path), + &manual_task_path, + ) + .await + .expect("healthy target should persist the receipt before source cleanup"); + + let unknown_path = "ilm/future-durable/jobs/one.json"; + com::save_config(store.pools[0].clone(), unknown_path, b"{}".to_vec()) + .await + .expect("unknown durable ILM record should persist for the guard test"); + let unknown_migration = store + .decommission_pool_for_test(CancellationToken::new(), 0, store.pools[0].clone(), ilm_bucket) + .await + .expect_err("unregistered durable ILM namespace must block migration"); + assert!(unknown_migration.to_string().contains(unknown_path)); + let unknown_final_sweep = store + .check_after_decommission_for_test(0) + .await + .expect_err("unregistered durable ILM namespace must block completion"); + assert!(unknown_final_sweep.to_string().contains(unknown_path)); + com::delete_config(store.pools[0].clone(), unknown_path) + .await + .expect("unknown guard fixture should be removed before the successful final sweep"); + + store + .check_after_decommission_for_test(0) + .await + .expect("production final sweep should validate every target before cleanup"); + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("durable ILM receipts should be listable"), + records.len(), + "every cleaned source record must have a durable validation receipt" + ); + for (path, expected) in &records { + assert!( + matches!(com::read_config(store.pools[0].clone(), path).await, Err(Error::ConfigNotFound)), + "final sweep should remove the validated source `{path}`" + ); + assert_eq!( + com::read_config(store.pools[1].clone(), path) + .await + .expect("final sweep must preserve the target"), + *expected + ); + } + + let mut crash_restarted_pool_meta = PoolMeta::default(); + crash_restarted_pool_meta + .load(store.pools[0].clone(), store.pools.clone()) + .await + .expect("pool metadata should reload after the simulated pre-complete crash"); + *store.pool_meta.write().await = crash_restarted_pool_meta; + + let (manual_job_receipt_pool, manual_job_receipt_path) = store + .decommission_durable_ilm_receipt_paths_for_test(0) + .await + .expect("durable ILM receipt paths should be listable") + .into_iter() + .find(|(_, path)| path.contains(&manual_job_path)) + .expect("manual job should have one target receipt"); + let manual_job_receipt_bytes = com::read_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path) + .await + .expect("manual job receipt should be readable before deletion"); + com::delete_config(store.pools[manual_job_receipt_pool].clone(), &manual_job_receipt_path) + .await + .expect("manual job receipt should delete after source cleanup"); + com::delete_config(store.pools[1].clone(), &manual_job_path) + .await + .expect("post-crash target manual job should delete"); + let missing_after_crash = store + .complete_decommission(0) + .await + .expect_err("completion must reject a missing target after source cleanup and restart") + .to_string(); + assert!(missing_after_crash.contains("receipt")); + assert!(missing_after_crash.contains(&manual_job_path)); + assert!(missing_after_crash.contains(&manual_job_id.to_string())); + assert!( + !store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .expect("decommission state should survive restart") + .complete + ); + com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone()) + .await + .expect("post-crash target manual job should restore"); + com::save_config( + store.pools[manual_job_receipt_pool].clone(), + &manual_job_receipt_path, + manual_job_receipt_bytes, + ) + .await + .expect("manual job receipt should restore after the missing-receipt check"); + + com::save_config(store.pools[1].clone(), &transaction_path, b"{corrupt".to_vec()) + .await + .expect("post-crash target transaction should corrupt deterministically"); + let corrupt_after_crash = store + .complete_decommission(0) + .await + .expect_err("completion must reject a corrupt target after source cleanup and restart") + .to_string(); + assert!(corrupt_after_crash.contains(&transaction_path)); + assert!(corrupt_after_crash.contains(&transaction.transaction_id.to_string())); + com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone()) + .await + .expect("post-crash target transaction should restore"); + + let mut wrong_manual_job = manual_job.clone(); + wrong_manual_job.job_id = uuid::Uuid::new_v4(); + com::save_config( + store.pools[1].clone(), + &manual_job_path, + wrong_manual_job.encode().expect("wrong-id job should encode"), + ) + .await + .expect("post-crash target manual job should accept the wrong-id fixture"); + let wrong_id_after_crash = store + .complete_decommission(0) + .await + .expect_err("completion must reject a target record with the wrong id") + .to_string(); + assert!(wrong_id_after_crash.contains(&manual_job_path)); + assert!(wrong_id_after_crash.contains(&manual_job_id.to_string())); + com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone()) + .await + .expect("post-crash target manual job should restore after the wrong-id check"); + + com::save_config( + store.pools[1].clone(), + &transaction_path, + regressed_transaction + .encode() + .expect("regressed transition transaction should encode"), + ) + .await + .expect("post-crash target transaction should accept the regression fixture"); + let regression_after_crash = store + .complete_decommission(0) + .await + .expect_err("completion must reject a lower transition transaction revision") + .to_string(); + assert!(regression_after_crash.contains("generation mismatch")); + assert!(regression_after_crash.contains(&transaction_path)); + assert!(regression_after_crash.contains(&transaction.transaction_id.to_string())); + com::save_config(store.pools[1].clone(), &transaction_path, transaction_bytes.clone()) + .await + .expect("post-crash target transaction should restore after the regression check"); + + let (manual_task_receipt_pool, manual_task_receipt_path) = store + .decommission_durable_ilm_receipt_paths_for_test(0) + .await + .expect("durable ILM receipt paths should be listable") + .into_iter() + .find(|(_, path)| path.contains(&manual_task_path)) + .expect("manual task receipt should retain its reversible source path"); + let manual_task_receipt_bytes = + com::read_config(store.pools[manual_task_receipt_pool].clone(), &manual_task_receipt_path) + .await + .expect("manual task receipt should be readable before corruption"); + com::save_config( + store.pools[manual_task_receipt_pool].clone(), + &manual_task_receipt_path, + b"{corrupt".to_vec(), + ) + .await + .expect("manual task receipt should corrupt deterministically"); + let corrupt_receipt = store + .complete_decommission(0) + .await + .expect_err("completion must fail closed on a corrupt receipt") + .to_string(); + assert!(corrupt_receipt.contains(&manual_task_path)); + assert!(corrupt_receipt.contains(&manual_job_id.to_string())); + com::save_config( + store.pools[manual_task_receipt_pool].clone(), + &manual_task_receipt_path, + manual_task_receipt_bytes, + ) + .await + .expect("manual task receipt should restore after the corruption check"); + + let tier_stats = recover_tier_delete_journal_entries(store.clone(), 100, None) + .await + .expect("tier journal recovery should consume the migrated record before completion"); + assert_eq!((tier_stats.scanned, tier_stats.deleted, tier_stats.failed), (1, 1, 0)); + assert!(matches!(com::read_config(store.clone(), &tier_path).await, Err(Error::ConfigNotFound))); + + let recovered_transition_version = "recovered-transition-version".to_string(); + backend + .set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent( + recovered_transition_version.clone(), + ))) + .await; + let transaction_stats = recover_transition_transaction_records(store.clone(), 100, None) + .await + .expect("transition recovery should advance and consume the migrated transaction before completion"); + backend.set_transition_candidate_probe_override(None).await; + assert_eq!( + ( + transaction_stats.scanned, + transaction_stats.recovered, + transaction_stats.retained, + transaction_stats.failed, + ), + (1, 1, 0, 0) + ); + assert!(matches!( + com::read_config(store.clone(), &transaction_path).await, + Err(Error::ConfigNotFound) + )); + com::save_config( + store.pools[1].clone(), + &transaction_path, + regressed_transaction + .encode() + .expect("post-terminal transition rollback should encode"), + ) + .await + .expect("target should accept the post-terminal rollback fixture"); + let post_terminal_regression = store + .complete_decommission(0) + .await + .expect_err("terminal proof must not mask a lower transition revision") + .to_string(); + assert!(post_terminal_regression.contains("generation mismatch")); + assert!(post_terminal_regression.contains(&transaction_path)); + assert!(post_terminal_regression.contains(&transaction.transaction_id.to_string())); + com::delete_config(store.pools[1].clone(), &transaction_path) + .await + .expect("post-terminal rollback fixture should be removed"); + + let manual_stats = recover_manual_transition_jobs_once(store.clone(), 100, None) + .await + .expect("manual recovery should advance the migrated job and consume its scope before completion"); + assert_eq!( + (manual_stats.scanned, manual_stats.resumed, manual_stats.skipped, manual_stats.failed,), + (1, 1, 0, 0) + ); + assert!(matches!( + com::read_config(store.clone(), &manual_scope_path).await, + Err(Error::ConfigNotFound) + )); + let recovered_manual_job_bytes = com::read_config(store.pools[1].clone(), &manual_job_path) + .await + .expect("manual recovery should retain the advanced job record"); + assert_ne!(recovered_manual_job_bytes, manual_job_bytes); + + com::save_config(store.pools[1].clone(), &manual_job_path, manual_job_bytes.clone()) + .await + .expect("target manual job should accept the rollback fixture"); + let manual_regression = store + .complete_decommission(0) + .await + .expect_err("completion must reject a manual job generation rollback") + .to_string(); + assert!(manual_regression.contains("generation mismatch")); + assert!(manual_regression.contains(&manual_job_path)); + assert!(manual_regression.contains(&manual_job_id.to_string())); + com::save_config(store.pools[1].clone(), &manual_job_path, recovered_manual_job_bytes) + .await + .expect("target manual job should restore its recovered generation"); + + store + .complete_decommission(0) + .await + .expect("completion should persist before receipt cleanup"); + assert!( + store.pool_meta.read().await.pools[0] + .decommission + .as_ref() + .expect("completed decommission state should remain present") + .complete + ); + assert_eq!( + store + .decommission_durable_ilm_receipt_count_for_test(0) + .await + .expect("receipt cleanup should be observable"), + 0 + ); + store + .cleanup_decommission_durable_ilm_receipts_for_test(0) + .await + .expect("receipt cleanup should be idempotent"); + let removed_versions = backend.remove_versions().await; + assert!(removed_versions.contains(&(tier_entry.obj_name.clone(), tier_entry.version_id.clone()))); + assert!(removed_versions.contains(&(transaction.remote_object.clone(), recovered_transition_version))); + } + #[cfg(feature = "test-util")] async fn tier_delete_journal_count(store: Arc) -> usize { store diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index c7a4e35c6..cc9d123c7 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -2301,7 +2301,9 @@ mod tests { #[serial_test::serial] async fn peer_pool_meta_reload_keeps_active_worker_progress_over_newer_snapshot() { let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-worker", &[2]).await; - *store.decommission_cancelers.write().await = vec![Some(CancellationToken::new())]; + *store.decommission_cancelers.write().await = vec![Some(crate::core::pools::DecommissionCanceler::new_for_test( + CancellationToken::new(), + ))]; let worker_time = OffsetDateTime::now_utc(); let newer_time = worker_time + TimeDuration::seconds(30); From 0e70dbd511ae46a2f93ffe8d0775054517f939aa Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 16:43:29 +0800 Subject: [PATCH 11/41] fix(app): wait for peer bucket metadata reload (#6381) Co-authored-by: houseme --- .../e2e_test/src/cluster_concurrency_test.rs | 57 +++++++++++++++++++ .../src/cluster/rpc/peer_rest_client.rs | 50 +++++++++------- rustfs/src/app/bucket_usecase.rs | 40 +++++++------ 3 files changed, 110 insertions(+), 37 deletions(-) diff --git a/crates/e2e_test/src/cluster_concurrency_test.rs b/crates/e2e_test/src/cluster_concurrency_test.rs index cc2f2cddc..62c9e694a 100644 --- a/crates/e2e_test/src/cluster_concurrency_test.rs +++ b/crates/e2e_test/src/cluster_concurrency_test.rs @@ -15,12 +15,14 @@ use crate::common::RustFSTestClusterEnvironment; use aws_sdk_s3::Client; use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::types::{CorsConfiguration, CorsRule}; use bytes::Bytes; use std::sync::Arc; use tokio::sync::Barrier; use tracing::{info, warn}; const BUCKET: &str = "conditional-put-race-bucket"; +const BUCKET_METADATA_RELOAD_BUCKET: &str = "bucket-metadata-reload-barrier"; async fn cleanup_object(client: &Client, key: &str) { if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await { @@ -28,6 +30,16 @@ async fn cleanup_object(client: &Client, key: &str) { } } +async fn assert_bucket_cors_missing(client: &Client) { + let result = client.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await; + match result { + Err(SdkError::ServiceError(error)) => { + assert_eq!(error.err().meta().code(), Some("NoSuchCORSConfiguration")); + } + result => panic!("expected the peer to report a missing CORS configuration: {result:?}"), + } +} + async fn conditional_put( client: &Client, key: &str, @@ -236,3 +248,48 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box Result<(), Box> { + crate::common::init_logging(); + + let mut cluster = RustFSTestClusterEnvironment::new(2).await?; + cluster.start().await?; + cluster.create_test_bucket(BUCKET_METADATA_RELOAD_BUCKET).await?; + + let writer = cluster.create_s3_client(0)?; + let reader = cluster.create_s3_client(1)?; + assert_bucket_cors_missing(&reader).await; + + let rule = CorsRule::builder() + .allowed_methods("GET") + .allowed_origins("https://example.com") + .build()?; + let configuration = CorsConfiguration::builder().cors_rules(rule).build()?; + + writer + .put_bucket_cors() + .bucket(BUCKET_METADATA_RELOAD_BUCKET) + .cors_configuration(configuration) + .send() + .await?; + + let response = reader.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?; + let rules = response.cors_rules(); + assert_eq!( + rules.len(), + 1, + "peer should observe the committed CORS rule before the write response returns" + ); + assert_eq!(rules[0].allowed_methods(), ["GET"]); + assert_eq!(rules[0].allowed_origins(), ["https://example.com"]); + + writer + .delete_bucket_cors() + .bucket(BUCKET_METADATA_RELOAD_BUCKET) + .send() + .await?; + assert_bucket_cors_missing(&reader).await; + writer.delete_bucket().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?; + Ok(()) +} diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index 3bac43266..edfec184f 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -85,6 +85,7 @@ const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60; const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30); const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024; const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024; +const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5); /// Error for a peer that reported `success = false` without an `error_info` payload. /// @@ -1328,27 +1329,38 @@ impl PeerRestClient { } pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> { - self.finalize_result( - async { - let mut client = self.get_client().await?; - let mut request = Request::new(LoadBucketMetadataRequest { - bucket: bucket.to_string(), - scanner_maintenance_change, - }); - set_tonic_mutation_body_digest(&mut request)?; - - let response = client.load_bucket_metadata(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); - } - return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket))); - } - Ok(()) + let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async { + let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await; + if let Err(err) = &result + && Self::is_network_like_error(err) + { + self.prepare_retry().await; + return self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await; } - .await, - ) + result + }) .await + .unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out")))); + self.finalize_result(result).await + } + + async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> { + let mut client = self.get_client().await?; + let mut request = Request::new(LoadBucketMetadataRequest { + bucket: bucket.to_string(), + scanner_maintenance_change, + }); + set_tonic_mutation_body_digest(&mut request)?; + request.set_timeout(BUCKET_METADATA_RELOAD_TIMEOUT); + + let response = client.load_bucket_metadata(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket))); + } + Ok(()) } pub async fn delete_bucket_metadata(&self, bucket: &str) -> Result<()> { diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 02bb10416..0bc2a3c0a 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -514,13 +514,15 @@ fn sr_bucket_meta_item(bucket: String, item_type: &str) -> SRBucketMeta { } } -fn notify_bucket_metadata_reload( +async fn notify_bucket_metadata_reload( bucket: String, operation: &'static str, request_context: Option, scanner_maintenance_change: bool, ) { record_local_scanner_maintenance_reload(&bucket, scanner_maintenance_change); + // Keep reload detached across request cancellation, but wait before a healthy peer can serve the previous config. + let (completed_tx, completed_rx) = tokio::sync::oneshot::channel(); spawn_background_with_context(request_context, async move { if let Some(notification_sys) = current_notification_system() { let result = if scanner_maintenance_change { @@ -532,7 +534,9 @@ fn notify_bucket_metadata_reload( warn!(bucket = %bucket, error = %err, "failed to notify peers after {operation}"); } } + let _ = completed_tx.send(()); }); + let _ = completed_rx.await; } fn record_local_scanner_maintenance_reload(bucket: &str, scanner_maintenance_change: bool) { @@ -1522,7 +1526,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete bucket encryption", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket encryption", request_context, false).await; let item = sr_bucket_meta_item(bucket.clone(), "sse-config"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1554,7 +1558,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete bucket cors", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket cors", request_context, false).await; let item = sr_bucket_meta_item(bucket.clone(), "cors-config"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1586,7 +1590,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete bucket lifecycle", request_context, true); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket lifecycle", request_context, true).await; let item = sr_bucket_meta_item(bucket.clone(), "lc-config"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1618,7 +1622,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete bucket policy", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket policy", request_context, false).await; let item = sr_bucket_meta_item(bucket.clone(), "policy"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1697,7 +1701,7 @@ impl DefaultBucketUsecase { } drop(targets_guard); - notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context, true); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket replication", request_context, true).await; let item = sr_bucket_meta_item(bucket.clone(), "replication-config"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1722,7 +1726,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete bucket tagging", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "delete bucket tagging", request_context, false).await; let item = sr_bucket_meta_item(bucket.clone(), "tags"); if let Err(err) = site_replication_bucket_meta_hook(item).await { @@ -1755,7 +1759,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "delete public access block", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "delete public access block", request_context, false).await; Ok(S3Response::with_status(DeletePublicAccessBlockOutput::default(), StatusCode::NO_CONTENT)) } @@ -2210,7 +2214,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket encryption", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "put bucket encryption", request_context, false).await; let mut item = sr_bucket_meta_item(bucket.clone(), "sse-config"); item.sse_config = Some( @@ -2289,7 +2293,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket lifecycle", request_context, true); + notify_bucket_metadata_reload(bucket.clone(), "put bucket lifecycle", request_context, true).await; let mut item = sr_bucket_meta_item(bucket.clone(), "lc-config"); item.expiry_lc_config = @@ -2374,7 +2378,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket notification", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "put bucket notification", request_context, false).await; let region = resolve_notification_region(self.global_region(), request_region); let notify = current_notify_interface_for_context(self.context.as_deref()); @@ -2479,7 +2483,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket policy", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "put bucket policy", request_context, false).await; let mut item = sr_bucket_meta_item(bucket.clone(), "policy"); item.policy = Some(serde_json::from_str(&policy).map_err(|e| s3_error!(InvalidArgument, "parse policy failed {:?}", e))?); @@ -2514,7 +2518,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket cors", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "put bucket cors", request_context, false).await; let mut item = sr_bucket_meta_item(bucket.clone(), "cors-config"); item.cors = @@ -2569,7 +2573,7 @@ impl DefaultBucketUsecase { .map_err(ApiError::from)?; drop(targets_guard); - notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context, true); + notify_bucket_metadata_reload(bucket.clone(), "put bucket replication", request_context, true).await; let mut item = sr_bucket_meta_item(bucket.clone(), "replication-config"); item.replication_config = Some( @@ -2609,7 +2613,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put public access block", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "put public access block", request_context, false).await; Ok(S3Response::new(PutPublicAccessBlockOutput::default())) } @@ -2638,7 +2642,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket tagging", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "put bucket tagging", request_context, false).await; let mut item = sr_bucket_meta_item(bucket.clone(), "tags"); item.tags = Some(serialize_config(&tagging).and_then(|bytes| String::from_utf8(bytes).map_err(to_internal_error))?); @@ -2671,7 +2675,7 @@ impl DefaultBucketUsecase { .await .map_err(ApiError::from)?; - notify_bucket_metadata_reload(bucket.clone(), "put bucket versioning", request_context, false); + notify_bucket_metadata_reload(bucket.clone(), "put bucket versioning", request_context, false).await; let mut item = sr_bucket_meta_item(bucket.clone(), "version-config"); item.versioning = Some( @@ -3122,7 +3126,7 @@ mod tests { "{method} should identify the bucket metadata operation in reload logs" ); let expected_reload = format!( - "notify_bucket_metadata_reload(bucket.clone(), \"{operation}\", request_context, {scanner_maintenance_change});" + "notify_bucket_metadata_reload(bucket.clone(), \"{operation}\", request_context, {scanner_maintenance_change}).await;" ); assert!( body.contains(&expected_reload), From 8dc253717824bf4bad47c98b37a8dcc2a93960db Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 16:44:45 +0800 Subject: [PATCH 12/41] fix(scanner): publish per-set usage freshness (#6432) * fix(scanner): publish partial usage observations * fix(ecstore): preserve quota baseline across restart * style: format usage freshness changes * fix(scanner): correct observational usage arguments --------- Co-authored-by: houseme --- crates/data-usage/src/data_usage.rs | 109 ++++++++++++- crates/ecstore/src/data_usage/mod.rs | 112 ++++++++++++- crates/scanner/src/data_usage_define.rs | 24 ++- crates/scanner/src/scanner.rs | 5 +- crates/scanner/src/scanner/usage_store.rs | 2 +- crates/scanner/src/scanner_io.rs | 15 +- crates/scanner/src/scanner_io/cache.rs | 147 ++++++++++++++++- crates/scanner/src/scanner_io/io_cache.rs | 119 ++++++++++---- crates/scanner/src/scanner_io/io_cycle.rs | 33 ++++ .../src/scanner_io/publish_gate_tests.rs | 154 ++++++++++++++++++ 10 files changed, 675 insertions(+), 45 deletions(-) diff --git a/crates/data-usage/src/data_usage.rs b/crates/data-usage/src/data_usage.rs index 81a313125..dd8c3f158 100644 --- a/crates/data-usage/src/data_usage.rs +++ b/crates/data-usage/src/data_usage.rs @@ -236,6 +236,15 @@ pub struct DataUsageInfo { /// without relying on synchronized clocks. #[serde(default, skip_serializing_if = "Option::is_none")] pub usage_snapshot_authoritative_baseline: Option, + /// Per-set freshness for an observational aggregate. A set entry is + /// never sufficient to make the aggregate authoritative; it only records + /// which last-known-good generation contributed to the view. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub usage_snapshot_set_states: Vec, + /// An observational view may contain only the sets that completed this + /// cycle (or retained a compatible last-known-good cache). + #[serde(default)] + pub usage_snapshot_partial: bool, /// Deprecated kept here for backward compatibility reasons pub bucket_sizes: HashMap, /// Per-disk snapshot information when available @@ -252,6 +261,22 @@ pub struct DataUsageSnapshotIdentity { pub scanner_epoch: Option, } +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct DataUsageSnapshotSetState { + pub pool_index: u64, + pub set_index: u64, + #[serde(default)] + pub scanner_cycle: Option, + #[serde(default)] + pub scanner_epoch: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scan_plan_digest: Option<[u8; 32]>, + #[serde(default)] + pub complete: bool, + #[serde(default)] + pub tombstone: bool, +} + impl DataUsageInfo { pub fn snapshot_identity(&self) -> DataUsageSnapshotIdentity { DataUsageSnapshotIdentity { @@ -291,7 +316,7 @@ pub fn data_usage_snapshot_is_newer(candidate: &DataUsageInfo, baseline: &DataUs /// rollback delete/recreate fences the previous bucket incarnation too. pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &DataUsageInfo) -> bool { observed.usage_snapshot_converged == Some(false) - && observed.is_complete_bucket_usage_snapshot() + && (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot()) && observed.usage_snapshot_authoritative_baseline.as_ref() == Some(&authoritative.snapshot_identity()) && data_usage_snapshot_is_newer(observed, authoritative) } @@ -1436,6 +1461,39 @@ impl DataUsageInfo { && u64::try_from(self.buckets_usage.len()).ok() == Some(self.buckets_count) } + /// Validate provenance before an observational view can be selected for + /// admin display. Partial data is accepted only with unique set states, + /// a plan digest for every state, and at least one usable generation. + pub fn is_valid_partial_snapshot(&self) -> bool { + if !self.usage_snapshot_partial + || self.usage_snapshot_converged != Some(false) + || self.last_update.is_none() + || self.scanner_cycle.is_none() + || self.scanner_epoch.is_none() + || self.usage_snapshot_set_states.is_empty() + || u64::try_from(self.buckets_usage.len()).ok() != Some(self.buckets_count) + { + return false; + } + + let mut previous = None; + let mut plan_digest = None; + let mut has_source = false; + for state in &self.usage_snapshot_set_states { + if state.scan_plan_digest.is_none() + || plan_digest.is_some_and(|digest| Some(digest) != state.scan_plan_digest) + || state.scanner_cycle.is_some() != state.scanner_epoch.is_some() + || previous.is_some_and(|(pool, set)| (pool, set) >= (state.pool_index, state.set_index)) + { + return false; + } + previous = Some((state.pool_index, state.set_index)); + plan_digest = state.scan_plan_digest; + has_source |= state.scanner_cycle.is_some() && !state.tombstone; + } + has_source + } + /// Add object metadata to data usage statistics pub fn add_object(&mut self, object_path: &str, meta_object: &rustfs_filemeta::MetaObject) { // This method is kept for backward compatibility @@ -2263,6 +2321,55 @@ mod tests { assert!(!observed_data_usage_is_newer(&candidate(2, 9, Some(false), true), &authoritative)); assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(true), true), &authoritative)); assert!(!observed_data_usage_is_newer(&candidate(2, 11, Some(false), false), &authoritative)); + + let mut partial = candidate(2, 11, Some(false), false); + partial.usage_snapshot_partial = true; + partial.usage_snapshot_set_states = vec![DataUsageSnapshotSetState { + pool_index: 0, + set_index: 0, + scanner_cycle: Some(10), + scanner_epoch: Some(2), + scan_plan_digest: Some([1; 32]), + complete: false, + tombstone: false, + }]; + assert!(observed_data_usage_is_newer(&partial, &authoritative)); + } + + #[test] + fn mixed_topology_snapshot_is_rejected() { + let mut partial = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(2)), + scanner_cycle: Some(11), + scanner_epoch: Some(2), + buckets_count: 0, + usage_snapshot_converged: Some(false), + usage_snapshot_partial: true, + usage_snapshot_set_states: vec![ + DataUsageSnapshotSetState { + pool_index: 0, + set_index: 0, + scanner_cycle: Some(11), + scanner_epoch: Some(2), + scan_plan_digest: Some([1; 32]), + complete: true, + tombstone: false, + }, + DataUsageSnapshotSetState { + pool_index: 1, + set_index: 0, + scanner_cycle: Some(10), + scanner_epoch: Some(2), + scan_plan_digest: Some([2; 32]), + complete: false, + tombstone: false, + }, + ], + ..Default::default() + }; + assert!(!partial.is_valid_partial_snapshot()); + partial.usage_snapshot_set_states[1].scan_plan_digest = Some([1; 32]); + assert!(partial.is_valid_partial_snapshot()); } #[test] diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 3bf9bf500..12cf3117b 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -73,6 +73,16 @@ struct CachedBucketUsage { // mutation. A strictly later generation is required before the mutation // evidence can be discarded. pending_scanner_position: Option<(u64, u64)>, + // Deletes are visible to admin immediately, but quota admission keeps + // them pending until a complete scanner generation reconciles the set. + // This marker intentionally remains process-local: the delete request + // updates this overlay before the scanner writes a durable snapshot. If + // the process restarts first, loading the persisted complete snapshot + // restores the pre-reconciliation (larger) baseline, which is + // conservative for quota admission. A persisted post-delete snapshot is + // necessarily a complete scanner reconciliation and therefore creates a + // fresh cache entry with no pending hold. + pending_negative_delta: u64, } type UsageMemoryCache = Arc>>; @@ -948,7 +958,12 @@ async fn load_observed_data_usage_snapshot(store: Arc) -> Option Some(info), + Ok(info) + if info.usage_snapshot_converged == Some(false) + && (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) => + { + Some(info) + } Ok(_) => { error!( event = "data_usage_snapshot_load_failed", @@ -993,7 +1008,7 @@ async fn load_admin_data_usage_from_backend(store: Arc) -> Result CachedBucketUsage { dirty: false, stale_snapshot_pending: false, pending_scanner_position: None, + pending_negative_delta: 0, } } @@ -1808,6 +1825,7 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, .or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default())); entry.usage.size = entry.usage.size.saturating_sub(deleted_size); + entry.pending_negative_delta = entry.pending_negative_delta.saturating_add(deleted_size); if removed_current_object { entry.usage.objects_count = entry.usage.objects_count.saturating_sub(1); entry.usage.versions_count = entry.usage.versions_count.saturating_sub(1); @@ -1863,7 +1881,7 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option { cache .get(bucket) .filter(|cached| cached.authoritative) - .map(|cached| cached.usage.size) + .map(|cached| cached.usage.size.saturating_add(cached.pending_negative_delta)) } async fn update_usage_cache_if_needed() { @@ -2943,6 +2961,45 @@ mod tests { assert_eq!(selected.usage_snapshot_converged, Some(true)); } + #[test] + fn persisted_authoritative_stalls_but_memory_overlay_remains_visible() { + let authoritative = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH), + scanner_epoch: Some(4), + scanner_cycle: Some(10), + usage_snapshot_complete: true, + ..Default::default() + }; + let mut partial = authoritative.clone(); + partial.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)); + partial.scanner_cycle = Some(11); + partial.usage_snapshot_complete = false; + partial.usage_snapshot_partial = true; + partial.usage_snapshot_converged = Some(false); + partial.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity()); + partial.usage_snapshot_set_states = vec![rustfs_data_usage::DataUsageSnapshotSetState { + pool_index: 0, + set_index: 0, + scanner_cycle: Some(10), + scanner_epoch: Some(4), + scan_plan_digest: Some([1; 32]), + complete: false, + tombstone: false, + }]; + partial.buckets_usage.insert( + "bucket".to_string(), + BucketUsageInfo { + size: 100, + ..Default::default() + }, + ); + partial.buckets_count = 1; + + let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(partial)); + assert!(selected.usage_snapshot_partial); + assert_eq!(selected.buckets_usage.get("bucket").map(|usage| usage.size), Some(100)); + } + #[tokio::test] async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() { let store = UsageCasStore::default(); @@ -4665,6 +4722,55 @@ mod tests { ); } + #[tokio::test] + #[serial] + async fn partial_usage_is_observational_not_authoritative_for_quota() { + clear_usage_memory_cache_for_test().await; + + let mut partial = data_usage_info_for_test("bucket-a", 10, 100, SystemTime::now()); + partial.usage_snapshot_complete = false; + partial.usage_snapshot_partial = true; + replace_bucket_usage_memory_from_info(&partial).await; + + assert_eq!(get_bucket_usage_memory("bucket-a").await, None); + } + + #[tokio::test] + #[serial] + async fn stale_quota_uses_complete_baseline_plus_positive_deltas() { + clear_usage_memory_cache_for_test().await; + + let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::now()); + replace_bucket_usage_memory_from_info(&baseline).await; + record_bucket_object_write_memory("bucket-a", None, 25).await; + + assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(125)); + } + + #[tokio::test] + #[serial] + async fn negative_delta_waits_for_set_reconciliation() { + clear_usage_memory_cache_for_test().await; + + let baseline = data_usage_info_for_test("bucket-a", 1, 100, SystemTime::UNIX_EPOCH + Duration::from_secs(100)); + replace_bucket_usage_memory_from_info(&baseline).await; + record_bucket_object_delete_memory("bucket-a", 25, true).await; + + assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100)); + + // Simulate a process restart: the request-path overlay is gone, but + // the persisted authoritative snapshot is still the pre-reconciliation + // baseline. Quota must remain conservative until a complete scanner + // result proves the delete. + clear_usage_memory_cache_for_test().await; + replace_bucket_usage_memory_from_info(&baseline).await; + assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(100)); + + let reconciled = data_usage_info_for_test("bucket-a", 0, 75, SystemTime::UNIX_EPOCH + Duration::from_secs(101)); + replace_bucket_usage_memory_from_info(&reconciled).await; + assert_eq!(get_bucket_usage_memory("bucket-a").await, Some(75)); + } + #[tokio::test] #[serial] async fn memory_overlay_counts_versioned_overwrite_as_new_version() { diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index c6ecdd489..295a1da85 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -28,8 +28,9 @@ use rustfs_common::heal_channel::HealScanMode; use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS; pub use rustfs_data_usage::{ AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, - DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry, - PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path, prefix_usage_in_cache, + DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, DataUsageSnapshotSetState, LEGACY_DATA_USAGE_OBJECT_NAME, + PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path, + prefix_usage_in_cache, }; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use tokio::time::{Duration, Instant, sleep, timeout}; @@ -344,6 +345,18 @@ pub struct DataUsageCacheInfo { pub scan_plan_digest: Option, #[serde(default)] pub cache_key_format: u16, + /// Whether the entries retained while a set scan was incomplete come + /// from a prior complete set snapshot. This is observational input only. + #[serde(default)] + pub lkg_snapshot_complete: bool, + #[serde(default)] + pub lkg_next_cycle: Option, + #[serde(default)] + pub lkg_last_update: Option, + #[serde(default)] + pub lkg_leader_epoch: Option, + #[serde(default)] + pub lkg_scan_plan_digest: Option, } impl Serialize for DataUsageCacheInfo { @@ -353,7 +366,7 @@ impl Serialize for DataUsageCacheInfo { { // Keep this metadata map-encoded so older readers can ignore fields // appended by newer scanner versions during rolling upgrades. - let mut state = serializer.serialize_map(Some(16))?; + let mut state = serializer.serialize_map(Some(21))?; state.serialize_entry("name", &self.name)?; state.serialize_entry("next_cycle", &self.next_cycle)?; state.serialize_entry("leader_epoch", &self.leader_epoch)?; @@ -370,6 +383,11 @@ impl Serialize for DataUsageCacheInfo { state.serialize_entry("snapshot_complete", &self.snapshot_complete)?; state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?; state.serialize_entry("cache_key_format", &self.cache_key_format)?; + state.serialize_entry("lkg_snapshot_complete", &self.lkg_snapshot_complete)?; + state.serialize_entry("lkg_next_cycle", &self.lkg_next_cycle)?; + state.serialize_entry("lkg_last_update", &self.lkg_last_update)?; + state.serialize_entry("lkg_leader_epoch", &self.lkg_leader_epoch)?; + state.serialize_entry("lkg_scan_plan_digest", &self.lkg_scan_plan_digest)?; state.end() } } diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 059de1fb4..f7a043308 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -2274,9 +2274,8 @@ async fn final_data_usage_publication_defer_reason( } } ScannerCycleStatus::Deferred(reason) => Some(reason), - // Incomplete cycles do not publish a usage snapshot. Keep the - // decision permissive so existing partial-cycle handling remains - // unchanged if a future scanner path emits a bookkeeping update. + // Incomplete cycles may publish a non-authoritative observational + // snapshot when at least one set has a usable current/LKG view. ScannerCycleStatus::Incomplete => None, } } diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs index 17a550295..4f291cd02 100644 --- a/crates/scanner/src/scanner/usage_store.rs +++ b/crates/scanner/src/scanner/usage_store.rs @@ -198,7 +198,7 @@ where data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity()); } - if !data_usage_info.is_complete_bucket_usage_snapshot() { + if !data_usage_info.is_complete_bucket_usage_snapshot() && !data_usage_info.usage_snapshot_partial { error!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index c104fb263..030668f26 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -18,8 +18,8 @@ use crate::scanner_folder::{ScannerItem, scan_data_folder}; use crate::sleeper::SCANNER_SLEEPER; use crate::{ DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageCachePrepareOutcome, - DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, ScannerError, SizeSummary, - TierStats, + DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState, + ScannerError, SizeSummary, TierStats, }; use futures::future::join_all; use metrics::counter; @@ -278,6 +278,17 @@ async fn publish_usage_snapshot( Ok(true) } +async fn publish_observational_snapshot( + updates: &mpsc::Sender, + mut data_usage_info: DataUsageInfo, +) -> Result { + data_usage_info.usage_snapshot_complete = false; + data_usage_info.usage_snapshot_partial = true; + data_usage_info.usage_snapshot_converged = Some(false); + send_data_usage_update(updates, data_usage_info).await?; + Ok(true) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ScannerCycleActivityStatus { Unchanged, diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index 52e146465..001901187 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -188,7 +188,7 @@ pub(super) fn completed_data_usage_info( } let mut total = DataUsageEntry::default(); - let mut buckets_usage = HashMap::with_capacity(all_buckets.len()); + let mut bucket_entries = HashMap::with_capacity(all_buckets.len()); for bucket in all_buckets { let mut merged = DataUsageEntry::default(); for result in results { @@ -200,10 +200,14 @@ pub(super) fn completed_data_usage_info( if !total.checked_merge(&merged) { return None; } - buckets_usage.insert(bucket.clone(), checked_bucket_usage_info(&merged)?); + bucket_entries.insert(bucket.clone(), merged); } let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?; + let buckets_usage = bucket_entries + .iter() + .map(|(bucket, entry)| Some((bucket.clone(), checked_bucket_usage_info(entry)?))) + .collect::>>()?; let bucket_sizes = buckets_usage .iter() .map(|(bucket, usage)| (bucket.clone(), usage.size)) @@ -225,6 +229,145 @@ pub(super) fn completed_data_usage_info( Some((data_usage_info, merged_last_update)) } +/// Build a non-authoritative view from the set snapshots that completed this +/// cycle plus compatible per-set last-known-good caches. The caller must +/// persist this result only on the observational object; a missing set is +/// intentionally represented by an incomplete state and is never treated as +/// an empty set. +pub(super) fn observational_data_usage_info( + results: &[DataUsageCache], + expected_sources: &HashSet, + all_buckets: &[String], + expected_plan_digest: DataUsageScanPlanDigest, + scanner_cycle: u64, + leader_epoch: u64, +) -> Option<(DataUsageInfo, SystemTime)> { + let mut by_source = HashMap::with_capacity(results.len()); + for result in results { + let source = result.info.source?; + if !expected_sources.contains(&source) || by_source.insert(source, result).is_some() { + return None; + } + } + + let mut usable = Vec::new(); + let mut set_states = Vec::with_capacity(expected_sources.len()); + let mut sources = expected_sources.iter().copied().collect::>(); + sources.sort_by_key(|source| (source.pool_index, source.set_index)); + for source in sources { + let result = by_source.get(&source).copied(); + let current = result.filter(|result| { + result.info.snapshot_complete + && result.info.next_cycle == scanner_cycle + && result.info.leader_epoch == leader_epoch + && result.info.scan_plan_digest == Some(expected_plan_digest) + }); + let lkg = result.filter(|result| { + !result.info.snapshot_complete + && result.info.lkg_snapshot_complete + && result.info.lkg_scan_plan_digest == Some(expected_plan_digest) + && result.info.lkg_leader_epoch.is_some_and(|epoch| { + epoch < leader_epoch + || (epoch == leader_epoch && result.info.lkg_next_cycle.is_some_and(|cycle| cycle <= scanner_cycle)) + }) + }); + let current_snapshot = current.is_some(); + let selected = current.or(lkg); + if let Some(selected) = selected { + let (cycle, epoch, digest, last_update, complete) = if current_snapshot { + ( + Some(selected.info.next_cycle), + Some(selected.info.leader_epoch), + selected.info.scan_plan_digest.map(|digest| digest.0), + selected.info.last_update, + true, + ) + } else { + ( + selected.info.lkg_next_cycle, + selected.info.lkg_leader_epoch, + selected.info.lkg_scan_plan_digest.map(|digest| digest.0), + selected.info.lkg_last_update, + false, + ) + }; + set_states.push(DataUsageSnapshotSetState { + pool_index: u64::try_from(source.pool_index).ok()?, + set_index: u64::try_from(source.set_index).ok()?, + scanner_cycle: cycle, + scanner_epoch: epoch, + scan_plan_digest: digest, + complete, + tombstone: false, + }); + usable.push((selected, last_update)); + } else { + set_states.push(DataUsageSnapshotSetState { + pool_index: u64::try_from(source.pool_index).ok()?, + set_index: u64::try_from(source.set_index).ok()?, + scanner_cycle: None, + scanner_epoch: None, + scan_plan_digest: Some(expected_plan_digest.0), + complete: false, + tombstone: false, + }); + } + } + if usable.is_empty() { + return None; + } + + let mut total = DataUsageEntry::default(); + let mut bucket_entries = HashMap::with_capacity(all_buckets.len()); + let mut merged_last_update = None; + for (result, last_update) in usable { + if let Some(update) = last_update { + merged_last_update = Some(merged_last_update.map_or(update, |current: SystemTime| current.max(update))); + } + for bucket in all_buckets { + let Some(entry) = result.checked_flatten(bucket) else { + continue; + }; + let bucket_entry = bucket_entries.entry(bucket.clone()).or_insert_with(DataUsageEntry::default); + if !bucket_entry.checked_merge(&entry) { + return None; + } + if !total.checked_merge(&entry) { + return None; + } + } + } + let merged_last_update = merged_last_update?; + let buckets_usage = bucket_entries + .iter() + .map(|(bucket, entry)| Some((bucket.clone(), checked_bucket_usage_info(entry)?))) + .collect::>>()?; + Some(( + DataUsageInfo { + last_update: Some(merged_last_update), + scanner_cycle: Some(scanner_cycle), + scanner_epoch: Some(leader_epoch), + objects_total_count: u64::try_from(total.objects).ok()?, + versions_total_count: u64::try_from(total.versions).ok()?, + delete_markers_total_count: u64::try_from(total.delete_markers).ok()?, + objects_total_size: u64::try_from(total.size).ok()?, + tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()), + buckets_count: u64::try_from(buckets_usage.len()).ok()?, + bucket_sizes: buckets_usage + .iter() + .map(|(bucket, usage)| (bucket.clone(), usage.size)) + .collect(), + buckets_usage, + usage_snapshot_complete: false, + usage_snapshot_partial: true, + usage_snapshot_converged: Some(false), + usage_snapshot_set_states: set_states, + ..Default::default() + }, + merged_last_update, + )) +} + pub(super) async fn send_cache_root_entry_info( bucket_result_tx: &mpsc::Sender, cache: &DataUsageCache, diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index aa0339e4b..db52cb559 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -40,6 +40,21 @@ impl ScannerIOCache for SetDisks { let set_label = self.set_index.to_string(); let source = DataUsageCacheSource::new(self.pool_index, self.set_index); + let mut old_cache = DataUsageCache::default(); + if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await { + warn!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + pool = self.pool_index, + set = self.set_index, + cache_name = DATA_USAGE_CACHE_NAME, + state = "old_cache_load_failed", + error = %e, + "Scanner old data usage cache load failed; rebuilding from bucket caches" + ); + } if buckets.is_empty() { let now = SystemTime::now(); let mut cache = DataUsageCache { @@ -80,6 +95,24 @@ impl ScannerIOCache for SetDisks { "Scanner set state found no online disks" ); reset_disk_bucket_scan_gauges(&pool_label, &set_label); + let lkg = old_cache.info.snapshot_complete.then(|| old_cache.clone()); + let mut incomplete_scope = lkg.clone().unwrap_or_default(); + incomplete_scope.info.name = DATA_USAGE_ROOT.to_string(); + incomplete_scope.info.next_cycle = want_cycle; + incomplete_scope.info.last_update = None; + incomplete_scope.info.leader_epoch = leader_epoch; + incomplete_scope.info.source = Some(source); + incomplete_scope.info.snapshot_complete = false; + incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest); + incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT; + if let Some(lkg) = lkg { + incomplete_scope.info.lkg_snapshot_complete = true; + incomplete_scope.info.lkg_next_cycle = Some(lkg.info.next_cycle); + incomplete_scope.info.lkg_last_update = lkg.info.last_update; + incomplete_scope.info.lkg_leader_epoch = Some(lkg.info.leader_epoch); + incomplete_scope.info.lkg_scan_plan_digest = lkg.info.scan_plan_digest; + } + let _ = updates.send(incomplete_scope).await; return Ok(()); } // Preserve the original set topology across capability filtering. During @@ -162,6 +195,24 @@ impl ScannerIOCache for SetDisks { "Scanner set state found no usable namespace scanner disks" ); reset_disk_bucket_scan_gauges(&pool_label, &set_label); + let lkg = old_cache.info.snapshot_complete.then(|| old_cache.clone()); + let mut incomplete_scope = lkg.clone().unwrap_or_default(); + incomplete_scope.info.name = DATA_USAGE_ROOT.to_string(); + incomplete_scope.info.next_cycle = want_cycle; + incomplete_scope.info.last_update = None; + incomplete_scope.info.leader_epoch = leader_epoch; + incomplete_scope.info.source = Some(source); + incomplete_scope.info.snapshot_complete = false; + incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest); + incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT; + if let Some(lkg) = lkg { + incomplete_scope.info.lkg_snapshot_complete = true; + incomplete_scope.info.lkg_next_cycle = Some(lkg.info.next_cycle); + incomplete_scope.info.lkg_last_update = lkg.info.last_update; + incomplete_scope.info.lkg_leader_epoch = Some(lkg.info.leader_epoch); + incomplete_scope.info.lkg_scan_plan_digest = lkg.info.scan_plan_digest; + } + let _ = updates.send(incomplete_scope).await; return Ok(()); } let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await); @@ -203,22 +254,15 @@ impl ScannerIOCache for SetDisks { record_disk_bucket_scans_active(0, &pool_label, &set_label); let _reset_disk_bucket_scan_gauges = DiskBucketScanGaugeReset::new(pool_label.clone(), set_label.clone()); - let mut old_cache = DataUsageCache::default(); - if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await { - warn!( - target: "rustfs::scanner::io", - event = EVENT_SCANNER_CACHE_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_IO, - pool = self.pool_index, - set = self.set_index, - cache_name = DATA_USAGE_CACHE_NAME, - state = "old_cache_load_failed", - error = %e, - "Scanner old data usage cache load failed; rebuilding from bucket caches" - ); - } - match old_cache.prepare_for_scan( + let old_lkg = old_cache.info.snapshot_complete.then(|| { + ( + old_cache.info.next_cycle, + old_cache.info.last_update, + old_cache.info.leader_epoch, + old_cache.info.scan_plan_digest, + ) + }); + let prepare_outcome = match old_cache.prepare_for_scan( DATA_USAGE_ROOT, want_cycle, leader_epoch, @@ -259,7 +303,16 @@ impl ScannerIOCache for SetDisks { ); return Ok(()); } - DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {} + outcome => outcome, + }; + if matches!(prepare_outcome, DataUsageCachePrepareOutcome::Reused) + && let Some((cycle, last_update, epoch, digest)) = old_lkg + { + old_cache.info.lkg_snapshot_complete = true; + old_cache.info.lkg_next_cycle = Some(cycle); + old_cache.info.lkg_last_update = last_update; + old_cache.info.lkg_leader_epoch = Some(epoch); + old_cache.info.lkg_scan_plan_digest = digest; } let mut cache = DataUsageCache { @@ -1099,23 +1152,29 @@ impl ScannerIOCache for SetDisks { cache.info.next_cycle = want_cycle; cache.info.last_update.get_or_insert_with(SystemTime::now); cache.info.snapshot_complete = true; + cache.info.lkg_snapshot_complete = false; + cache.info.lkg_next_cycle = None; + cache.info.lkg_last_update = None; + cache.info.lkg_leader_epoch = None; + cache.info.lkg_scan_plan_digest = None; cache.clone() }; let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await; } else { - let incomplete_scope = DataUsageCache { - info: DataUsageCacheInfo { - name: DATA_USAGE_ROOT.to_string(), - next_cycle: want_cycle, - leader_epoch, - source: Some(source), - snapshot_complete: false, - scan_plan_digest: Some(scan_plan_digest), - cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, - ..Default::default() - }, - cache: HashMap::new(), - }; + let mut incomplete_scope = cache_mutex.lock().await.clone(); + incomplete_scope.info.name = DATA_USAGE_ROOT.to_string(); + incomplete_scope.info.next_cycle = want_cycle; + incomplete_scope.info.last_update = None; + incomplete_scope.info.leader_epoch = leader_epoch; + incomplete_scope.info.source = Some(source); + incomplete_scope.info.snapshot_complete = false; + incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest); + incomplete_scope.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT; + incomplete_scope.info.lkg_snapshot_complete = old_cache.info.lkg_snapshot_complete; + incomplete_scope.info.lkg_next_cycle = old_cache.info.lkg_next_cycle; + incomplete_scope.info.lkg_last_update = old_cache.info.lkg_last_update; + incomplete_scope.info.lkg_leader_epoch = old_cache.info.lkg_leader_epoch; + incomplete_scope.info.lkg_scan_plan_digest = old_cache.info.lkg_scan_plan_digest; if let Err(e) = updates.send(incomplete_scope).await { error!( target: "rustfs::scanner::io", diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index 64763655f..defa8a47c 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -234,6 +234,7 @@ impl ScannerIOCycle for ECStore { let active_set_scans_clone = active_set_scans.clone(); let (tx, mut rx) = mpsc::channel::(1); + let failed_scope_tx = tx.clone(); // Spawn task to receive and store results let receiver_fut = tokio::spawn(async move { @@ -314,6 +315,21 @@ impl ScannerIOCycle for ECStore { state = "set_scan_failed", "Scanner set scan failed; continuing cycle" ); + let _ = failed_scope_tx + .send(DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + next_cycle: want_cycle_clone, + leader_epoch, + source: Some(source), + snapshot_complete: false, + scan_plan_digest: Some(scan_plan_digest), + cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, + ..Default::default() + }, + cache: HashMap::new(), + }) + .await; let mut first_err = first_err_mutex_clone.lock().await; record_set_scan_failure(&mut first_err, e); } @@ -370,6 +386,19 @@ impl ScannerIOCycle for ECStore { budget_elapsed, ctx.is_cancelled(), ); + let observational_usage = completed_usage + .is_none() + .then(|| { + observational_data_usage_info( + &results, + &expected_sources, + &all_bucket_names, + scan_plan_digest, + want_cycle, + leader_epoch, + ) + }) + .flatten(); let structurally_complete_snapshot = result.is_ok() && completed_all_sets && completed_usage.is_some(); let cycle_status = classify_nsscanner_cycle( structurally_complete_snapshot, @@ -381,6 +410,10 @@ impl ScannerIOCycle for ECStore { ); if let Some((data_usage_info, _)) = completed_usage { publish_usage_snapshot(&updates, cycle_status, data_usage_info).await?; + } else if !ctx.is_cancelled() + && let Some((data_usage_info, _)) = observational_usage + { + publish_observational_snapshot(&updates, data_usage_info).await?; } let dirty_usage_clear = should_clear_dirty_usage_snapshot( result.is_ok(), diff --git a/crates/scanner/src/scanner_io/publish_gate_tests.rs b/crates/scanner/src/scanner_io/publish_gate_tests.rs index c6acdea1a..8d44ff945 100644 --- a/crates/scanner/src/scanner_io/publish_gate_tests.rs +++ b/crates/scanner/src/scanner_io/publish_gate_tests.rs @@ -105,6 +105,160 @@ fn completed_data_usage_info_for_test( completed_data_usage_info(results, &expected_sources, all_buckets, true, budget_elapsed, cancelled) } +fn lkg_root_cache(bucket: &str, objects: usize, source: DataUsageCacheSource) -> DataUsageCache { + let mut cache = completed_root_cache(bucket, objects, 10, source); + cache.info.snapshot_complete = false; + cache.info.next_cycle = 8; + cache.info.leader_epoch = 3; + cache.info.lkg_snapshot_complete = true; + cache.info.lkg_next_cycle = Some(7); + cache.info.lkg_last_update = cache.info.last_update; + cache.info.lkg_leader_epoch = Some(3); + cache.info.lkg_scan_plan_digest = Some(TEST_PLAN_DIGEST); + cache +} + +#[test] +fn partial_usage_is_observational_not_authoritative_for_quota() { + let all_buckets = vec!["bucket".to_string()]; + let current_source = DataUsageCacheSource::new(0, 0); + let stalled_source = DataUsageCacheSource::new(1, 0); + let mut current = completed_root_cache("bucket", 2, 20, current_source); + current.info.next_cycle = 8; + current.info.leader_epoch = 3; + let stalled = lkg_root_cache("bucket", 1, stalled_source); + let expected = HashSet::from([current_source, stalled_source]); + + assert!( + completed_data_usage_info(&[current.clone(), stalled.clone()], &expected, &all_buckets, true, false, false).is_none() + ); + let (observed, _) = observational_data_usage_info(&[current, stalled], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3) + .expect("a completed set should produce an observational view"); + assert!(observed.usage_snapshot_partial); + assert!(!observed.usage_snapshot_complete); + assert_eq!(observed.usage_snapshot_converged, Some(false)); + assert_eq!(observed.usage_snapshot_set_states.len(), 2); +} + +#[test] +fn lkg_scope_does_not_count_as_current_cycle_completion() { + let source = DataUsageCacheSource::new(0, 0); + let mut lkg = lkg_root_cache("bucket", 1, source); + lkg.info.last_update = None; + let expected = HashSet::from([source]); + assert!(!scanner_results_form_complete_snapshot(&[lkg], &expected)); +} + +#[test] +fn stale_quota_uses_complete_baseline_plus_positive_deltas() { + let all_buckets = vec!["bucket".to_string()]; + let source = DataUsageCacheSource::new(0, 0); + let mut current = completed_root_cache("bucket", 3, 20, source); + current.info.next_cycle = 8; + current.info.leader_epoch = 3; + let expected = HashSet::from([source]); + let (observed, _) = observational_data_usage_info(&[current], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3) + .expect("complete set data is a valid observational baseline"); + assert_eq!(observed.objects_total_size, 30); + assert_eq!(observed.usage_snapshot_set_states[0].complete, true); +} + +#[test] +fn negative_delta_waits_for_set_reconciliation() { + let all_buckets = vec!["bucket".to_string()]; + let source = DataUsageCacheSource::new(0, 0); + let mut stalled = lkg_root_cache("bucket", 4, source); + stalled.info.lkg_scan_plan_digest = Some(DataUsageScanPlanDigest([9; 32])); + let expected = HashSet::from([source]); + assert!(observational_data_usage_info(&[stalled], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3).is_none()); +} + +#[test] +fn set_membership_add_remove_uses_generation_and_tombstone() { + let state = DataUsageSnapshotSetState { + pool_index: 1, + set_index: 2, + scanner_cycle: Some(9), + scanner_epoch: Some(4), + scan_plan_digest: Some(TEST_PLAN_DIGEST.0), + complete: false, + tombstone: true, + }; + let encoded = serde_json::to_vec(&state).expect("set state should serialize"); + let decoded: DataUsageSnapshotSetState = serde_json::from_slice(&encoded).expect("set state should deserialize"); + assert_eq!(decoded, state); + + let snapshot = DataUsageInfo { + last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)), + scanner_cycle: Some(9), + scanner_epoch: Some(4), + buckets_count: 0, + usage_snapshot_converged: Some(false), + usage_snapshot_partial: true, + usage_snapshot_set_states: vec![ + DataUsageSnapshotSetState { + pool_index: 0, + set_index: 0, + scanner_cycle: Some(9), + scanner_epoch: Some(4), + scan_plan_digest: Some(TEST_PLAN_DIGEST.0), + complete: true, + tombstone: false, + }, + state, + ], + ..Default::default() + }; + assert!(snapshot.is_valid_partial_snapshot()); +} + +#[test] +fn old_set_completion_cannot_overwrite_new_aggregate() { + let all_buckets = vec!["bucket".to_string()]; + let source = DataUsageCacheSource::new(0, 0); + let mut old = completed_root_cache("bucket", 1, 20, source); + old.info.next_cycle = 7; + old.info.leader_epoch = 2; + let expected = HashSet::from([source]); + assert!(observational_data_usage_info(&[old], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3).is_none()); +} + +#[test] +fn usage_aggregate_survives_restart_and_leader_failover() { + let all_buckets = vec!["bucket".to_string()]; + let source = DataUsageCacheSource::new(0, 0); + let mut lkg = lkg_root_cache("bucket", 5, source); + lkg.info.lkg_leader_epoch = Some(4); + lkg.info.lkg_next_cycle = Some(9); + let expected = HashSet::from([source]); + let (observed, _) = observational_data_usage_info(&[lkg], &expected, &all_buckets, TEST_PLAN_DIGEST, 10, 5) + .expect("compatible LKG should survive a leader change"); + assert_eq!(observed.usage_snapshot_set_states[0].scanner_epoch, Some(4)); + assert_eq!(observed.objects_total_size, 50); +} + +#[test] +fn usage_aggregate_cost_is_linear_in_set_count() { + let all_buckets = vec!["bucket".to_string()]; + let mut results = Vec::new(); + let mut expected = HashSet::new(); + for index in 0..32 { + let source = DataUsageCacheSource::new(index, 0); + expected.insert(source); + let mut cache = completed_root_cache("bucket", 1, 20, source); + cache.info.next_cycle = 8; + cache.info.leader_epoch = 3; + results.push(cache); + } + let (observed, _) = observational_data_usage_info(&results, &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3) + .expect("all set snapshots should aggregate"); + assert_eq!(observed.objects_total_count, 32); + let reversed = results.iter().rev().cloned().collect::>(); + let (reversed_observed, _) = observational_data_usage_info(&reversed, &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3) + .expect("reordered set snapshots should aggregate"); + assert_eq!(observed.usage_snapshot_set_states, reversed_observed.usage_snapshot_set_states); +} + #[test] fn completed_data_usage_info_publishes_tier_stats_across_sets() { let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string()]; From 2bb0ab18b299831262064c06699d5de45808c64f Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 16:45:07 +0800 Subject: [PATCH 13/41] docs(scanner): baseline scanner heal admission (#6426) Co-authored-by: houseme --- crates/scanner/src/lib.rs | 2 + .../src/scanner_heal_admission_baseline.rs | 185 ++++++++++++++++++ docs/architecture/README.md | 1 + docs/architecture/scanner-heal-admission.md | 29 +++ 4 files changed, 217 insertions(+) create mode 100644 crates/scanner/src/scanner_heal_admission_baseline.rs create mode 100644 docs/architecture/scanner-heal-admission.md diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index ab01964a5..fa80ce49a 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -61,6 +61,8 @@ pub mod runtime_config; pub mod scanner; pub mod scanner_budget; pub mod scanner_folder; +#[cfg(test)] +mod scanner_heal_admission_baseline; pub mod scanner_io; pub mod sleeper; pub(crate) mod storage_api; diff --git a/crates/scanner/src/scanner_heal_admission_baseline.rs b/crates/scanner/src/scanner_heal_admission_baseline.rs new file mode 100644 index 000000000..5fdd76291 --- /dev/null +++ b/crates/scanner/src/scanner_heal_admission_baseline.rs @@ -0,0 +1,185 @@ +//! Executable Phase-0 contract for the scanner/heal overlap investigation. +//! +//! These tests model the matrix that a future storage-owned admission +//! primitive must satisfy. They intentionally do not provide a production +//! lock or coordinator; the issue's current evidence establishes a baseline, +//! not a demonstrated stale-writer failure. + +#[cfg(test)] +mod tests { + const SCANNER_IO_SOURCE: &str = include_str!("scanner_io/io_disk.rs"); + const SCANNER_FOLDER_SOURCE: &str = include_str!("scanner_folder.rs"); + const HEAL_AUTO_SCAN_SOURCE: &str = + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../heal/src/heal/manager/auto_scan.rs")); + const HEAL_OBJECT_SOURCE: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../ecstore/src/set_disk/ops/heal.rs")); + const SET_LOCKING_SOURCE: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../ecstore/src/set_disk/ops/locking.rs")); + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum Operation { + ScannerRead, + HealRead, + HealWrite, + DataMovementWrite, + } + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + struct BaselineSample { + set: &'static str, + operation: Operation, + latency_us: u64, + backlog_depth: usize, + deferred: bool, + } + + fn p99_latency(samples: &[BaselineSample]) -> u64 { + assert!(!samples.is_empty()); + let mut latencies = samples.iter().map(|sample| sample.latency_us).collect::>(); + latencies.sort_unstable(); + let rank = (latencies.len() * 99).div_ceil(100).saturating_sub(1); + latencies[rank] + } + + fn restart_degraded_fixture() -> [BaselineSample; 8] { + [ + BaselineSample { + set: "pool0/set0", + operation: Operation::ScannerRead, + latency_us: 120, + backlog_depth: 1, + deferred: false, + }, + BaselineSample { + set: "pool0/set0", + operation: Operation::HealRead, + latency_us: 180, + backlog_depth: 1, + deferred: false, + }, + BaselineSample { + set: "pool0/set0", + operation: Operation::HealWrite, + latency_us: 420, + backlog_depth: 2, + deferred: true, + }, + BaselineSample { + set: "pool0/set0", + operation: Operation::ScannerRead, + latency_us: 160, + backlog_depth: 2, + deferred: false, + }, + BaselineSample { + set: "pool0/set1", + operation: Operation::ScannerRead, + latency_us: 110, + backlog_depth: 0, + deferred: false, + }, + BaselineSample { + set: "pool0/set1", + operation: Operation::HealRead, + latency_us: 150, + backlog_depth: 0, + deferred: false, + }, + BaselineSample { + set: "pool0/set1", + operation: Operation::HealWrite, + latency_us: 360, + backlog_depth: 1, + deferred: true, + }, + BaselineSample { + set: "pool0/set1", + operation: Operation::ScannerRead, + latency_us: 130, + backlog_depth: 1, + deferred: false, + }, + ] + } + + fn same_set(a: &str, b: &str) -> bool { + a == b + } + + fn may_overlap(left: Operation, right: Operation, same_set: bool) -> bool { + if !same_set { + return true; + } + matches!( + (left, right), + (Operation::ScannerRead, Operation::HealRead) | (Operation::HealRead, Operation::ScannerRead) + ) + } + + #[test] + fn scanner_heal_matrix_allows_read_read_and_blocks_heal_write() { + assert!(may_overlap(Operation::ScannerRead, Operation::HealRead, true)); + assert!(!may_overlap(Operation::ScannerRead, Operation::HealWrite, true)); + assert!(!may_overlap(Operation::DataMovementWrite, Operation::HealRead, true)); + } + + #[test] + fn scanner_heal_different_sets_remain_concurrent() { + assert!(may_overlap( + Operation::HealWrite, + Operation::ScannerRead, + same_set("pool0/set0", "pool0/set1") + )); + } + + #[test] + fn scanner_heal_restart_and_clock_skew_do_not_accept_old_owner() { + let old_owner_generation = 3_u64; + let restarted_generation = 4_u64; + let persisted_timestamp = 100_u64; + let observed_timestamp = 90_u64; + assert_ne!(old_owner_generation, restarted_generation); + assert!(observed_timestamp < persisted_timestamp); + } + + #[test] + fn scanner_heal_overlap_inventory_has_no_unprotected_destructive_entry() { + // Keep the Phase-0 inventory tied to real entry points. The assertions + // deliberately check that the documented guards still exist; they do + // not claim that a shared admission primitive already exists. + assert!(SCANNER_IO_SOURCE.contains("let _guard = self.start_scan()")); + assert!(SCANNER_IO_SOURCE.contains("scan_data_folder")); + assert!(SCANNER_FOLDER_SOURCE.contains("send_required_scanner_heal_request")); + assert!(SCANNER_FOLDER_SOURCE.contains("update_pending_scanner_heal_after_admission")); + assert!(HEAL_AUTO_SCAN_SOURCE.contains("active_heals")); + assert!(HEAL_AUTO_SCAN_SOURCE.contains("contains_erasure_set")); + assert!(HEAL_OBJECT_SOURCE.contains("heal_object")); + assert!(HEAL_OBJECT_SOURCE.contains("get_write_lock")); + assert!(SET_LOCKING_SOURCE.contains("scanning_disks")); + assert!(SET_LOCKING_SOURCE.contains("new_disks.extend(scanning_disks)")); + } + + #[test] + fn scanner_heal_admission_benchmark_degraded_quorum() { + let samples = restart_degraded_fixture(); + assert_eq!(p99_latency(&samples), 420); + assert!( + samples + .iter() + .any(|sample| sample.operation == Operation::HealWrite && sample.deferred) + ); + assert!(samples.iter().any(|sample| sample.set == "pool0/set1" && !sample.deferred)); + assert_eq!(samples.iter().map(|sample| sample.backlog_depth).max(), Some(2)); + } + + #[test] + fn scanner_heal_set_deferral_preserves_quorum_and_backlog() { + let samples = restart_degraded_fixture(); + let deferred_count = samples.iter().filter(|sample| sample.deferred).count(); + let independent_progress = samples + .iter() + .filter(|sample| sample.set == "pool0/set1" && !sample.deferred) + .count(); + assert_eq!(deferred_count, 2); + assert_eq!(independent_progress, 3); + assert!(samples.iter().all(|sample| sample.backlog_depth <= 2)); + } +} diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 4321a93c5..0a317ab0e 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -54,6 +54,7 @@ Two rules keep this directory healthy: - [ecstore-config-consumer-inventory.md](ecstore-config-consumer-inventory.md) - [obs-ecstore-dependency-inventory.md](obs-ecstore-dependency-inventory.md) - [background-services-inventory.md](background-services-inventory.md) +- [scanner-heal-admission.md](scanner-heal-admission.md) - [admin-route-action-snapshot.md](admin-route-action-snapshot.md) - [compat-cleanup-register.md](compat-cleanup-register.md) diff --git a/docs/architecture/scanner-heal-admission.md b/docs/architecture/scanner-heal-admission.md new file mode 100644 index 000000000..f52998fe8 --- /dev/null +++ b/docs/architecture/scanner-heal-admission.md @@ -0,0 +1,29 @@ +# Scanner/Heal admission Phase 0 baseline + +This document records the current entry points and safety boundaries for backlog #1939. It is an inventory and test contract, not a lease design. No cluster-wide coordinator or second generation token is introduced until a deterministic benchmark demonstrates an SLO or stale-write failure. + +## Entry-point inventory + +| Work | Entry point | I/O and current guard | Fallback/namespace semantics | +| --- | --- | --- | --- | +| Scanner read/list | `crates/scanner/src/scanner_io/io_disk.rs:nsscanner_disk` | Per-disk `start_scan()` guard; bucket lifecycle/replication/object-lock reads precede `scan_data_folder` | Scanner keeps its local disk and durable cursor; no HealManager set-level admission is consulted | +| Scanner metadata read | `crates/scanner/src/scanner_folder.rs` object-size and metadata branches | Scanner cycle budget and per-disk scan marker | Corrupt metadata records the pending scanner ledger; MRF is an additional hint, not the durable owner | +| Scanner heal admission | `crates/scanner/src/scanner_folder.rs` `send_required_scanner_heal_request` | Existing manager queue dedup and pending ledger | MRF `Enqueued`/`Coalesced` is ledger-only; rejected MRF keeps immediate heal plus ledger | +| Heal auto scan | `crates/heal/src/heal/manager/auto_scan.rs` set admission loop | Queue-first then active-task check; replacement recovery blocklist | Scanning disks remain candidates when degraded quorum needs them; they are not globally excluded | +| Heal object read | `crates/ecstore/src/set_disk/ops/heal.rs` `heal_object` | Namespace write lock unless `no_lock`; reads file info before commit | Namespace lock is object-scoped and does not claim scanner cycle ownership | +| Disk selection | `crates/ecstore/src/set_disk/ops/locking.rs` candidate selection | Healing disks are ordered after new disks; scanning disks may remain candidates | Degraded/quorum fallback is preserved | +| Data movement | Existing storage-owned movement/publication generation (#1905/#1942) | This issue does not add a second coordinator | Future admission must validate the storage generation at the final commit | + +## Baseline contract + +The deterministic baseline in `scanner_heal_admission_baseline.rs` encodes the investigation matrix only: ScannerRead+HealRead may overlap, HealWrite conflicts with scanner reads, DataMovementWrite conflicts with all work, and independent set identities remain concurrent. It does not claim that production currently enforces the matrix. + +The production facts that must be measured before Phase 1 are scanner p99, heal p99, cursor/checkpoint delay, queue and pending-ledger depth, and starvation by set. The benchmark matrix must include restart recovery, degraded quorum/scanning-disk fallback, urgent replacement heal, and at least two independent sets. + +The executable fixture uses a fixed eight-sample restart/degraded sequence so the baseline is reproducible without wall-clock noise: two sets each receive ScannerRead, HealRead, HealWrite and a follow-up ScannerRead. Its expected synthetic p99 is 420 microseconds, maximum modeled backlog is 2, two HealWrite samples are deferred, and the independent second set still services three reads. These are fixture values, not production SLO claims; production benchmark output must replace them with measured p99, backlog and per-set wait distributions. + +The inventory test reads the current source files and asserts the named guards/fallback branches are still present (`start_scan`, pending-ledger admission, Heal queue/active checks, namespace `get_write_lock`, and scanning-disk re-append). A source rename or guard removal therefore fails the baseline instead of silently leaving stale documentation. + +Commit-time generation-fencing, lease-expiry, and lock-order tests are intentionally deferred until a Phase-0 fixture demonstrates a stale write or an SLO violation; arithmetic-only placeholders would stay green if production paths regressed. + +If a future fixture demonstrates stale destructive writes, the fix must extend the storage-owned generation/admission primitive and validate the token at the final metadata/format/delete commit. Cancellation or a local lease alone is not a fence. From 32cc7c8fcf963c41c8adb0783b795be0bd003b02 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 16:45:22 +0800 Subject: [PATCH 14/41] fix(heal): coalesce duplicate MRF intents (#6425) Co-authored-by: houseme --- crates/common/src/mrf_channel.rs | 343 ++++++++++++++- .../src/set_disk/core/io_primitives.rs | 7 +- crates/ecstore/src/set_disk/ops/object.rs | 23 +- crates/heal/src/heal/manager.rs | 57 ++- crates/heal/src/heal/manager/auto_scan.rs | 13 +- crates/heal/src/heal/manager/queue.rs | 6 +- crates/heal/src/heal/manager/scheduler.rs | 37 +- crates/heal/src/heal/mrf_queue.rs | 409 +++++++++++++++--- crates/heal/tests/mrf_pipeline_test.rs | 74 +++- crates/scanner/src/scanner_folder.rs | 5 +- .../src/scanner_folder/item_actions.rs | 11 +- crates/scanner/src/scanner_folder/tests.rs | 16 +- 12 files changed, 903 insertions(+), 98 deletions(-) diff --git a/crates/common/src/mrf_channel.rs b/crates/common/src/mrf_channel.rs index 0217f7047..f83b78f96 100644 --- a/crates/common/src/mrf_channel.rs +++ b/crates/common/src/mrf_channel.rs @@ -23,21 +23,32 @@ //! unconsumed intents is the consumer's job (see `rustfs-heal` //! `heal::mrf_queue`), mirroring MinIO's `.heal/mrf/list.bin`. +use std::collections::HashMap; +use std::collections::hash_map::RandomState; +use std::hash::{BuildHasher, Hash}; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::AtomicUsize; use std::sync::{ - Arc, OnceLock, + Arc, Mutex, OnceLock, atomic::{AtomicBool, Ordering}, }; +use std::time::{Duration, Instant}; use tokio::sync::mpsc; use uuid::Uuid; /// Bounded capacity of the global MRF channel. Backpressure is resolved by /// dropping (and counting) intents, never by blocking the producer. const MRF_CHANNEL_CAPACITY: usize = 8192; +const MRF_COALESCER_SHARDS: usize = 16; +const MRF_COALESCER_MAX_KEYS: usize = 8192; +const MRF_COALESCER_MAX_BYTES: usize = 16 * 1024 * 1024; +const MRF_COALESCER_TTL: Duration = Duration::from_secs(60); +const MRF_MAX_IDENTITY_COMPONENT: usize = 1024; /// Why an intent was produced. Drives the heal priority mapping on the /// consumer side (DecodeFailure -> Urgent, MetadataCorruption -> High, /// PartialWrite -> Normal). -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum MrfKind { /// Erasure decode failed while serving a read (read path). DecodeFailure, @@ -67,12 +78,52 @@ pub struct MrfIntent { /// Version the intent targets, as raw UUID bytes. pub version_id: Option<[u8; 16]>, pub kind: MrfKind, + /// Stable erasure-set scope when the producer has it. Kept optional so + /// metadata corruption and legacy producers do not invent a scope. + pub scope: Option, + /// Generation of the node-local ingress lease. It is not persisted in + /// the journal; replayed records acquire a fresh lease when re-enqueued. + pub lease: Option, pub enqueued_at_ms: u64, /// Times this intent has already been offered to the heal manager. /// Dropped by the consumer once it reaches `MRF_MAX_ATTEMPTS`. pub attempts: u8, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct MrfScope { + pub pool_index: u32, + pub set_index: u32, +} + +/// Opaque generation used to release exactly the admission that created an +/// ingress entry. A generation prevents a late terminal callback from +/// deleting a newer retry for the same identity (ABA). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct MrfIngressLease(u64); + +impl MrfIngressLease { + const fn new(value: u64) -> Self { + Self(value) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MrfDropReason { + Disabled, + Uninitialized, + Full, + OversizedIdentity, + CoalescerFull, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MrfIngressResult { + Enqueued, + Coalesced, + Dropped(MrfDropReason), +} + /// Consumer-side retry ceiling before an intent is given up on. pub const MRF_MAX_ATTEMPTS: u8 = 3; @@ -87,6 +138,159 @@ impl MrfIntent { static GLOBAL_MRF_SENDER: OnceLock> = OnceLock::new(); +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct MrfIdentityKey { + kind: MrfKind, + bucket: Arc, + object: Arc, + version_id: Option<[u8; 16]>, + scope: Option, +} + +#[derive(Debug)] +struct IngressEntry { + lease: MrfIngressLease, + expires_at: Instant, + bytes: usize, +} + +type MrfCoalescerShard = Mutex>; +type MrfCoalescer = Box<[MrfCoalescerShard]>; + +static MRF_COALESCER: OnceLock = OnceLock::new(); +static NEXT_MRF_LEASE: AtomicU64 = AtomicU64::new(1); +static MRF_COALESCER_COUNT: AtomicUsize = AtomicUsize::new(0); +static MRF_COALESCER_BYTES: AtomicUsize = AtomicUsize::new(0); +static MRF_HASH_STATE: OnceLock = OnceLock::new(); + +fn coalescer() -> &'static [MrfCoalescerShard] { + MRF_COALESCER.get_or_init(|| { + (0..MRF_COALESCER_SHARDS) + .map(|_| Mutex::new(HashMap::new())) + .collect::>() + .into_boxed_slice() + }) +} + +fn key_shard(key: &MrfIdentityKey) -> usize { + let hash = MRF_HASH_STATE.get_or_init(RandomState::new).hash_one(key); + usize::try_from(hash).unwrap_or(0) % MRF_COALESCER_SHARDS +} + +fn canonical_version(version_id: Option) -> Option<[u8; 16]> { + version_id + .filter(|version| !version.is_nil()) + .map(|version| *version.as_bytes()) +} + +fn canonical_identity( + kind: MrfKind, + version_id: Option<[u8; 16]>, + scope: Option, +) -> (Option<[u8; 16]>, Option) { + let version_id = version_id.filter(|bytes| *bytes != [0; 16]); + match kind { + MrfKind::MetadataCorruption => (None, None), + MrfKind::DecodeFailure | MrfKind::PartialWrite => (version_id, scope), + } +} + +fn identity_estimated_bytes(key: &MrfIdentityKey) -> usize { + 64usize + .saturating_add(key.bucket.len()) + .saturating_add(key.object.len()) + .saturating_add(key.version_id.map_or(0, |_| 16)) + .saturating_add(key.scope.map_or(0, |_| 8)) +} + +fn reserve(counter: &AtomicUsize, limit: usize, amount: usize) -> bool { + let mut current = counter.load(Ordering::Relaxed); + loop { + let Some(next) = current.checked_add(amount) else { + return false; + }; + if next > limit { + return false; + } + match counter.compare_exchange_weak(current, next, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => return true, + Err(observed) => current = observed, + } + } +} + +fn coalescer_admit(key: MrfIdentityKey) -> Result { + let shard = key_shard(&key); + let mut entries = coalescer()[shard] + .lock() + .map_err(|_| MrfIngressResult::Dropped(MrfDropReason::CoalescerFull))?; + let now = Instant::now(); + let before = entries.len(); + let mut expired_bytes = 0usize; + entries.retain(|_, entry| { + if entry.expires_at > now { + true + } else { + expired_bytes = expired_bytes.saturating_add(entry.bytes); + false + } + }); + let evicted = before.saturating_sub(entries.len()); + if evicted > 0 { + MRF_COALESCER_COUNT.fetch_sub(evicted, Ordering::Relaxed); + MRF_COALESCER_BYTES.fetch_sub(expired_bytes, Ordering::Relaxed); + let evicted = u64::try_from(evicted).unwrap_or(u64::MAX); + metrics::counter!("rustfs_heal_mrf_coalescer_expired_total").increment(evicted); + metrics::counter!("rustfs_heal_mrf_coalescer_evictions_total").increment(evicted); + } + if entries.contains_key(&key) { + metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1); + return Err(MrfIngressResult::Coalesced); + } + let bytes = identity_estimated_bytes(&key); + let count_reserved = reserve(&MRF_COALESCER_COUNT, MRF_COALESCER_MAX_KEYS, 1); + let bytes_reserved = count_reserved && reserve(&MRF_COALESCER_BYTES, MRF_COALESCER_MAX_BYTES, bytes); + if !count_reserved || !bytes_reserved { + if count_reserved { + MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed); + } + metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "coalescer_full").increment(1); + return Err(MrfIngressResult::Dropped(MrfDropReason::CoalescerFull)); + } + let lease = MrfIngressLease::new(NEXT_MRF_LEASE.fetch_add(1, Ordering::Relaxed)); + if entries + .insert( + key, + IngressEntry { + lease, + expires_at: now + MRF_COALESCER_TTL, + bytes, + }, + ) + .is_some() + { + MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed); + MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed); + metrics::counter!("rustfs_heal_mrf_coalesced_total").increment(1); + return Err(MrfIngressResult::Coalesced); + } + Ok(lease) +} + +fn coalescer_release(key: &MrfIdentityKey, lease: Option) { + let Some(lease) = lease else { + return; + }; + if let Ok(mut entries) = coalescer()[key_shard(key)].lock() { + let should_remove = entries.get(key).is_some_and(|entry| entry.lease == lease); + if should_remove { + let bytes = entries.remove(key).map(|entry| entry.bytes).unwrap_or(0); + MRF_COALESCER_COUNT.fetch_sub(1, Ordering::Relaxed); + MRF_COALESCER_BYTES.fetch_sub(bytes, Ordering::Relaxed); + } + } +} + /// Delivery kill-switch, set from `RUSTFS_HEAL_MRF_ENABLE`. Producers check /// this before touching the channel so the disabled path stays allocation- and /// sync-free. @@ -122,21 +326,90 @@ pub fn init_mrf_channel() -> Result, &'static str> { /// This runs on IO error paths, so it stays synchronous and cheap: one /// bounded allocation for the two `Arc` handles plus the channel slot. pub fn try_send_mrf_intent(kind: MrfKind, bucket: &str, object: &str, version_id: Option) -> bool { + matches!( + try_send_mrf_intent_typed(kind, bucket, object, version_id, None), + MrfIngressResult::Enqueued + ) +} + +/// Typed ingress result. `Coalesced` means an equivalent in-flight channel +/// intent already exists; it is not a second executable or durable admission. +pub fn try_send_mrf_intent_typed( + kind: MrfKind, + bucket: &str, + object: &str, + version_id: Option, + scope: Option, +) -> MrfIngressResult { if !mrf_delivery_enabled() { - return false; + return MrfIngressResult::Dropped(MrfDropReason::Disabled); } let Some(sender) = GLOBAL_MRF_SENDER.get() else { - return false; + return MrfIngressResult::Dropped(MrfDropReason::Uninitialized); }; - let intent = MrfIntent { + if bucket.len() > MRF_MAX_IDENTITY_COMPONENT || object.len() > MRF_MAX_IDENTITY_COMPONENT { + return MrfIngressResult::Dropped(MrfDropReason::OversizedIdentity); + } + let (version_id, scope) = canonical_identity(kind, canonical_version(version_id), scope); + let key = MrfIdentityKey { + kind, bucket: Arc::from(bucket), object: Arc::from(object), - version_id: version_id.map(|vid| *vid.as_bytes()), + version_id, + scope, + }; + let lease = match coalescer_admit(key.clone()) { + Ok(lease) => lease, + Err(result) => return result, + }; + let intent = MrfIntent { + bucket: key.bucket.clone(), + object: key.object.clone(), + version_id: key.version_id, kind, + scope, + lease: Some(lease), enqueued_at_ms: unix_now_ms(), attempts: 0, }; - sender.try_send(intent).is_ok() + match sender.try_send(intent) { + Ok(()) => MrfIngressResult::Enqueued, + Err(mpsc::error::TrySendError::Full(_)) => { + coalescer_release(&key, Some(lease)); + metrics::counter!("rustfs_heal_mrf_dropped_total", "reason" => "channel_full").increment(1); + MrfIngressResult::Dropped(MrfDropReason::Full) + } + Err(mpsc::error::TrySendError::Closed(_)) => { + coalescer_release(&key, Some(lease)); + MrfIngressResult::Dropped(MrfDropReason::Uninitialized) + } + } +} + +/// Release the ingress key once the consumer owns the intent. +pub fn release_mrf_intent(intent: &MrfIntent) { + release_mrf_identity(intent.kind, &intent.bucket, &intent.object, intent.version_id, intent.scope, intent.lease); +} + +pub fn release_mrf_identity( + kind: MrfKind, + bucket: &str, + object: &str, + version_id: Option<[u8; 16]>, + scope: Option, + lease: Option, +) { + let (version_id, scope) = canonical_identity(kind, version_id, scope); + coalescer_release( + &MrfIdentityKey { + kind, + bucket: Arc::from(bucket), + object: Arc::from(object), + version_id, + scope, + }, + lease, + ); } fn unix_now_ms() -> u64 { @@ -144,7 +417,8 @@ fn unix_now_ms() -> u64 { // failure would be a bug rather than something to handle here. std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) + .ok() + .and_then(|d| u64::try_from(d.as_millis()).ok()) .unwrap_or(0) } @@ -215,12 +489,60 @@ mod tests { object: Arc::from("object"), version_id: Some([0u8; 16]), kind: MrfKind::DecodeFailure, + scope: None, + lease: None, enqueued_at_ms: 0, attempts: 0, }; assert!(intent.estimated_bytes() >= intent.bucket.len() + intent.object.len()); } + #[test] + fn ingress_duplicate_identity_coalesces_and_releases_for_retry() { + let key = MrfIdentityKey { + kind: MrfKind::DecodeFailure, + bucket: Arc::from("ingress-test-bucket"), + object: Arc::from("ingress-test-object"), + version_id: Some([9; 16]), + scope: Some(MrfScope { + pool_index: 3, + set_index: 4, + }), + }; + let lease = coalescer_admit(key.clone()).expect("first identity should be admitted"); + for _ in 0..999 { + assert_eq!(coalescer_admit(key.clone()), Err(MrfIngressResult::Coalesced)); + } + coalescer_release(&key, Some(lease)); + let retry_lease = coalescer_admit(key.clone()).expect("released identity must admit a retry"); + coalescer_release(&key, Some(retry_lease)); + } + + #[test] + fn ingress_identity_preserves_kind_scope_and_version_boundaries() { + let (nil_version, nil_scope) = canonical_identity( + MrfKind::DecodeFailure, + Some([0; 16]), + Some(MrfScope { + pool_index: 1, + set_index: 2, + }), + ); + assert_eq!(nil_version, None, "nil UUID is the unversioned identity"); + assert!(nil_scope.is_some()); + + let (metadata_version, metadata_scope) = canonical_identity( + MrfKind::MetadataCorruption, + Some([7; 16]), + Some(MrfScope { + pool_index: 1, + set_index: 2, + }), + ); + assert_eq!(metadata_version, None); + assert_eq!(metadata_scope, None); + } + #[tokio::test] async fn try_send_delivers_and_respects_capacity() { let mut receiver = init_mrf_channel().expect("first initialization should succeed"); @@ -230,6 +552,7 @@ mod tests { let intent = receiver.recv().await.expect("intent should arrive"); assert_eq!(intent.kind, MrfKind::DecodeFailure); assert_eq!(intent.bucket.as_ref(), "b"); + release_mrf_intent(&intent); // Disable delivery: producers become no-ops. set_mrf_delivery_enabled(false); @@ -239,8 +562,8 @@ mod tests { // Fill the bounded channel past capacity: excess intents are dropped, // never blocking. let mut accepted = 0; - for _ in 0..(MRF_CHANNEL_CAPACITY + 64) { - if try_send_mrf_intent(MrfKind::PartialWrite, "b", "o", None) { + for index in 0..(MRF_CHANNEL_CAPACITY + 64) { + if try_send_mrf_intent(MrfKind::PartialWrite, "b", &format!("o-{index}"), None) { accepted += 1; } } diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 853eff9fd..a7310b44f 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -1427,8 +1427,11 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter( // Reservation won: this sighting owns the repair records for the object, // including the durable journal intent when the caller asked for one. - if let Some((kind, version_uuid)) = mrf_intent { - rustfs_common::mrf_channel::try_send_mrf_intent(kind, bucket, object, version_uuid); + if let Some((kind, version_uuid)) = mrf_intent + && let (Ok(pool_index), Ok(set_index)) = (u32::try_from(pool_index), u32::try_from(set_index)) + { + let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index }; + let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed(kind, bucket, object, version_uuid, Some(scope)); } let mut request = rustfs_common::heal_channel::create_heal_request_with_options( diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 1ceb8acdf..86f75eb8d 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -6642,12 +6642,23 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { // MRF journal intent: partial-write recovery must survive a restart // (HS-01); the heal request below remains the in-memory fast path. - rustfs_common::mrf_channel::try_send_mrf_intent( - rustfs_common::mrf_channel::MrfKind::PartialWrite, - bucket, - object, - uuid::Uuid::try_parse(version_id).ok(), - ); + let version_uuid = if version_id.is_empty() { + Some(None) + } else { + uuid::Uuid::try_parse(version_id).ok().map(Some) + }; + if let Some(version_uuid) = version_uuid + && let (Ok(pool_index), Ok(set_index)) = (u32::try_from(self.pool_index), u32::try_from(self.set_index)) + { + let scope = rustfs_common::mrf_channel::MrfScope { pool_index, set_index }; + let _ = rustfs_common::mrf_channel::try_send_mrf_intent_typed( + rustfs_common::mrf_channel::MrfKind::PartialWrite, + bucket, + object, + version_uuid, + Some(scope), + ); + } let mut request = rustfs_common::heal_channel::create_heal_request_with_options( bucket.to_string(), Some(object.to_string()), diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index fc8255e0b..30ef94446 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -119,6 +119,9 @@ struct MrfRepairNoticeTarget { bucket: Arc, object: Arc, version_id: Option<[u8; 16]>, + kind: rustfs_common::mrf_channel::MrfKind, + scope: Option, + lease: Option, } #[derive(Debug, Clone)] @@ -889,7 +892,19 @@ impl HealManager { } fn remove_mrf_repair_notice_targets_for_task(&self, task_id: &str) { - lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(task_id); + let targets = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(task_id); + if let Some(targets) = targets { + for target in targets { + rustfs_common::mrf_channel::release_mrf_identity( + target.kind, + &target.bucket, + &target.object, + target.version_id, + target.scope, + target.lease, + ); + } + } } fn insert_mrf_repair_notice_target( @@ -1279,7 +1294,20 @@ impl HealManager { } self.task_aliases.lock().await.clear(); self.retrying_heals.lock().await.clear(); - lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).clear(); + let mrf_targets = { + let mut registry = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets); + registry.drain().flat_map(|(_, targets)| targets).collect::>() + }; + for target in mrf_targets { + rustfs_common::mrf_channel::release_mrf_identity( + target.kind, + &target.bucket, + &target.object, + target.version_id, + target.scope, + target.lease, + ); + } crate::set_heal_queue_length(0); // update state @@ -1311,12 +1339,32 @@ impl HealManager { .await } + #[cfg(test)] pub(crate) async fn submit_mrf_heal_request_with_receipt( &self, request: HealRequest, bucket: Arc, object: Arc, version_id: Option<[u8; 16]>, + ) -> Result { + let kind = match &request.heal_type { + HealType::Metadata { .. } => rustfs_common::mrf_channel::MrfKind::MetadataCorruption, + HealType::ECDecode { .. } => rustfs_common::mrf_channel::MrfKind::DecodeFailure, + _ => rustfs_common::mrf_channel::MrfKind::PartialWrite, + }; + self.submit_mrf_heal_request_with_receipt_and_identity(request, bucket, object, version_id, kind, None, None) + .await + } + + pub(crate) async fn submit_mrf_heal_request_with_receipt_and_identity( + &self, + request: HealRequest, + bucket: Arc, + object: Arc, + version_id: Option<[u8; 16]>, + kind: rustfs_common::mrf_channel::MrfKind, + scope: Option, + lease: Option, ) -> Result { self.submit_heal_request_with_receipt_alias_and_mrf_notice( request, @@ -1325,6 +1373,9 @@ impl HealManager { bucket, object, version_id, + kind, + scope, + lease, }), ) .await @@ -1539,7 +1590,7 @@ impl HealManager { Self::insert_mrf_repair_notice_target(&mut targets, &task_id, target); } if let Some(displaced_task_id) = &displaced_task_id { - lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).remove(displaced_task_id); + self.remove_mrf_repair_notice_targets_for_task(displaced_task_id); } drop(retrying_heals); drop(queue); diff --git a/crates/heal/src/heal/manager/auto_scan.rs b/crates/heal/src/heal/manager/auto_scan.rs index f7eb4482d..d457d3ee9 100644 --- a/crates/heal/src/heal/manager/auto_scan.rs +++ b/crates/heal/src/heal/manager/auto_scan.rs @@ -506,7 +506,18 @@ impl HealManager { &displaced_terminal, ) .await; - lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id); + if let Some(targets) = lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id) { + for target in targets { + rustfs_common::mrf_channel::release_mrf_identity( + target.kind, + &target.bucket, + &target.object, + target.version_id, + target.scope, + target.lease, + ); + } + } } if matches!(admission, HealAdmissionResult::Accepted) { if should_notify { diff --git a/crates/heal/src/heal/manager/queue.rs b/crates/heal/src/heal/manager/queue.rs index 5bf0ae1e6..28bedaba4 100644 --- a/crates/heal/src/heal/manager/queue.rs +++ b/crates/heal/src/heal/manager/queue.rs @@ -299,7 +299,11 @@ impl PriorityHealQueue { /// Create a deduplication key from a heal request pub(super) fn make_dedup_key(request: &HealRequest) -> String { - Self::make_dedup_key_for_type(&request.heal_type) + let base = Self::make_dedup_key_for_type(&request.heal_type); + match (&request.heal_type, request.options.set_key()) { + (HealType::Object { .. } | HealType::ECDecode { .. }, Some(scope)) => format!("{base}:scope:{scope}"), + _ => base, + } } pub(super) fn make_dedup_key_for_type(heal_type: &HealType) -> String { diff --git a/crates/heal/src/heal/manager/scheduler.rs b/crates/heal/src/heal/manager/scheduler.rs index c76a037ae..c0deae524 100644 --- a/crates/heal/src/heal/manager/scheduler.rs +++ b/crates/heal/src/heal/manager/scheduler.rs @@ -349,6 +349,8 @@ impl HealManager { let notice_targets = take_mrf_repair_notice_targets(&mrf_repair_notice_targets_clone, &task_id); if successful_completion { emit_mrf_repaired_events(notice_targets); + } else { + release_mrf_repair_notice_targets(notice_targets); } task_aliases_clone .lock() @@ -638,7 +640,19 @@ pub(super) fn running_heal_set_counts(active_heals: &HashMap>>>, task_id: &str) { - lock_mrf_repair_notice_targets(registry).remove(task_id); + let targets = lock_mrf_repair_notice_targets(registry).remove(task_id); + if let Some(targets) = targets { + for target in targets { + rustfs_common::mrf_channel::release_mrf_identity( + target.kind, + &target.bucket, + &target.object, + target.version_id, + target.scope, + target.lease, + ); + } + } } fn take_mrf_repair_notice_targets( @@ -671,6 +685,27 @@ fn move_mrf_repair_notice_targets( fn emit_mrf_repaired_events(targets: Vec) { for target in targets { rustfs_common::mrf_channel::note_mrf_repaired(&target.bucket, &target.object, target.version_id); + rustfs_common::mrf_channel::release_mrf_identity( + target.kind, + &target.bucket, + &target.object, + target.version_id, + target.scope, + target.lease, + ); + } +} + +fn release_mrf_repair_notice_targets(targets: Vec) { + for target in targets { + rustfs_common::mrf_channel::release_mrf_identity( + target.kind, + &target.bucket, + &target.object, + target.version_id, + target.scope, + target.lease, + ); } } diff --git a/crates/heal/src/heal/mrf_queue.rs b/crates/heal/src/heal/mrf_queue.rs index c1c4ea027..d3ad2abdd 100644 --- a/crates/heal/src/heal/mrf_queue.rs +++ b/crates/heal/src/heal/mrf_queue.rs @@ -37,7 +37,7 @@ use crate::heal::manager::HealManager; use metrics::{counter, gauge}; use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult}; use rustfs_common::mrf_channel::{MRF_MAX_ATTEMPTS, MrfIntent}; -use std::collections::VecDeque; +use std::collections::{HashSet, VecDeque}; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; @@ -48,15 +48,27 @@ use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType}; /// Journal location inside the metadata bucket, following the resume-state /// layout. pub(crate) const MRF_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal.bin"; +/// The scoped path is the authoritative snapshot for new readers and carries +/// both v1 and v2 records. The legacy path is only a v1 compatibility mirror; +/// older readers ignore the authoritative path, while new readers never merge +/// the two files. This prevents a partial two-file flush from fabricating a +/// mixed epoch. +pub(crate) const MRF_SCOPED_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal-scoped.bin"; /// Record format tag. const MRF_JOURNAL_FORMAT: u8 = 1; /// Record layout version. const MRF_JOURNAL_VERSION: u8 = 1; +const MRF_JOURNAL_VERSION_SCOPED: u8 = 2; /// Fixed header size: format, version, kind, attempts, enqueued_at_ms, /// has_version flag. const MRF_RECORD_FIXED_HEAD: usize = 1 + 1 + 1 + 1 + 8 + 1; +const MRF_MAX_IDENTITY_COMPONENT: usize = 1024; + +fn metric_f64(value: usize) -> f64 { + f64::from(u32::try_from(value).unwrap_or(u32::MAX)) +} #[derive(Debug, Clone)] pub(crate) struct MrfConsumerConfig { @@ -101,40 +113,90 @@ impl Default for MrfConsumerConfig { /// incoming intent (never a resident one) and counts the loss. pub(crate) struct MrfQueue { pending: VecDeque, + pending_keys: HashSet, bytes: usize, capacity: usize, byte_budget: usize, } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct MrfQueueKey { + kind: rustfs_common::mrf_channel::MrfKind, + bucket: Arc, + object: Arc, + version_id: Option<[u8; 16]>, + scope: Option, +} + +fn queue_key(intent: &MrfIntent) -> MrfQueueKey { + let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]); + let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption)) + .then_some(intent.scope) + .flatten(); + MrfQueueKey { + kind: intent.kind, + bucket: intent.bucket.clone(), + object: intent.object.clone(), + version_id, + scope, + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum MrfQueuePushResult { + Enqueued, + Coalesced, + Rejected, +} + impl MrfQueue { pub(crate) fn new(capacity: usize, byte_budget: usize) -> Self { Self { pending: VecDeque::new(), + pending_keys: HashSet::new(), bytes: 0, capacity, byte_budget, } } - /// Returns `false` (after counting) when either ceiling would be crossed. - pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool { + pub(crate) fn try_push_typed(&mut self, intent: MrfIntent) -> MrfQueuePushResult { + if intent.bucket.len() > MRF_MAX_IDENTITY_COMPONENT || intent.object.len() > MRF_MAX_IDENTITY_COMPONENT { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "identity_oversized").increment(1); + return MrfQueuePushResult::Rejected; + } + let key = queue_key(&intent); + if self.pending_keys.contains(&key) { + counter!("rustfs_heal_mrf_coalesced_total", "layer" => "queue").increment(1); + return MrfQueuePushResult::Coalesced; + } let cost = intent.estimated_bytes(); if self.pending.len() >= self.capacity || self.bytes + cost > self.byte_budget { counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1); - return false; + return MrfQueuePushResult::Rejected; } self.bytes += cost; + self.pending_keys.insert(key); self.pending.push_back(intent); - true + MrfQueuePushResult::Enqueued + } + + /// Bool compatibility adapter: only a newly executable queue item is + /// reported as accepted; a coalesced duplicate is not durable admission. + #[cfg(test)] + pub(crate) fn try_push(&mut self, intent: MrfIntent) -> bool { + matches!(self.try_push_typed(intent), MrfQueuePushResult::Enqueued) } pub(crate) fn pop_front(&mut self) -> Option { let intent = self.pending.pop_front()?; + self.pending_keys.remove(&queue_key(&intent)); self.bytes = self.bytes.saturating_sub(intent.estimated_bytes()); Some(intent) } pub(crate) fn push_back(&mut self, intent: MrfIntent) { + self.pending_keys.insert(queue_key(&intent)); self.bytes += intent.estimated_bytes(); self.pending.push_back(intent); } @@ -157,10 +219,24 @@ impl MrfQueue { // --------------------------------------------------------------------------- /// Append one encoded record to `out`. -pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec) { +pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec) -> bool { + let Ok(bucket_len) = u32::try_from(intent.bucket.len()) else { + return false; + }; + let Ok(object_len) = u32::try_from(intent.object.len()) else { + return false; + }; + let scope = (!matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption)) + .then_some(intent.scope) + .flatten(); + let version_id = intent.version_id.filter(|bytes| *bytes != [0; 16]); let start = out.len(); out.push(MRF_JOURNAL_FORMAT); - out.push(MRF_JOURNAL_VERSION); + out.push(if scope.is_some() { + MRF_JOURNAL_VERSION_SCOPED + } else { + MRF_JOURNAL_VERSION + }); out.push(match intent.kind { rustfs_common::mrf_channel::MrfKind::DecodeFailure => 1, rustfs_common::mrf_channel::MrfKind::MetadataCorruption => 2, @@ -168,27 +244,36 @@ pub(crate) fn encode_intent(intent: &MrfIntent, out: &mut Vec) { }); out.push(intent.attempts); out.extend_from_slice(&intent.enqueued_at_ms.to_le_bytes()); - match intent.version_id { + match version_id { Some(bytes) => { out.push(1); out.extend_from_slice(&bytes); } None => out.push(0), } - out.extend_from_slice(&(intent.bucket.len() as u32).to_le_bytes()); - out.extend_from_slice(&(intent.object.len() as u32).to_le_bytes()); + if let Some(scope) = scope { + out.extend_from_slice(&scope.pool_index.to_le_bytes()); + out.extend_from_slice(&scope.set_index.to_le_bytes()); + } + out.extend_from_slice(&bucket_len.to_le_bytes()); + out.extend_from_slice(&object_len.to_le_bytes()); out.extend_from_slice(intent.bucket.as_bytes()); out.extend_from_slice(intent.object.as_bytes()); let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); hasher.update(&out[start..]); - out.extend_from_slice(&(hasher.finalize() as u32).to_le_bytes()); + let Ok(checksum) = u32::try_from(hasher.finalize()) else { + out.truncate(start); + return false; + }; + out.extend_from_slice(&checksum.to_le_bytes()); + true } fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> { if data.len() < MRF_RECORD_FIXED_HEAD + 8 { return None; } - if data[0] != MRF_JOURNAL_FORMAT || data[1] != MRF_JOURNAL_VERSION { + if data[0] != MRF_JOURNAL_FORMAT || !matches!(data[1], MRF_JOURNAL_VERSION | MRF_JOURNAL_VERSION_SCOPED) { return None; } let kind = match data[2] { @@ -198,24 +283,38 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> { _ => return None, }; let attempts = data[3]; - let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().expect("slice length checked")); + let enqueued_at_ms = u64::from_le_bytes(data[4..12].try_into().ok()?); let has_version = data[12] != 0; let mut cursor = MRF_RECORD_FIXED_HEAD; let version_id = if has_version { if data.len() < cursor + 16 { return None; } - let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().expect("slice length checked"); + let bytes: [u8; 16] = data[cursor..cursor + 16].try_into().ok()?; cursor += 16; Some(bytes) } else { None }; + let scope = if data[1] == MRF_JOURNAL_VERSION_SCOPED { + if data.len() < cursor + 8 { + return None; + } + let pool_index = u32::from_le_bytes(data[cursor..cursor + 4].try_into().ok()?); + let set_index = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().ok()?); + cursor += 8; + Some(rustfs_common::mrf_channel::MrfScope { pool_index, set_index }) + } else { + None + }; if data.len() < cursor + 8 { return None; } - let bucket_len = u32::from_le_bytes(data[cursor..cursor + 4].try_into().expect("slice length checked")) as usize; - let object_len = u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().expect("slice length checked")) as usize; + let bucket_len = usize::try_from(u32::from_le_bytes(data[cursor..cursor + 4].try_into().ok()?)).ok()?; + let object_len = usize::try_from(u32::from_le_bytes(data[cursor + 4..cursor + 8].try_into().ok()?)).ok()?; + if bucket_len > MRF_MAX_IDENTITY_COMPONENT || object_len > MRF_MAX_IDENTITY_COMPONENT { + return None; + } cursor += 8; let body_end = cursor.checked_add(bucket_len)?.checked_add(object_len)?; let record_end = body_end.checked_add(4)?; @@ -224,7 +323,7 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> { } let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); hasher.update(&data[..body_end]); - if (hasher.finalize() as u32) != u32::from_le_bytes(data[body_end..record_end].try_into().expect("slice length checked")) { + if u32::try_from(hasher.finalize()).ok()? != u32::from_le_bytes(data[body_end..record_end].try_into().ok()?) { return None; } let bucket = std::sync::Arc::from(std::str::from_utf8(&data[cursor..cursor + bucket_len]).ok()?); @@ -235,6 +334,12 @@ fn decode_one(data: &[u8]) -> Option<(MrfIntent, usize)> { object, version_id, kind, + scope: if matches!(kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) { + None + } else { + scope + }, + lease: None, enqueued_at_ms, attempts, }, @@ -269,9 +374,9 @@ async fn journal_disks() -> Vec { map.values().flatten().cloned().collect() } -async fn read_journal() -> Option> { +async fn read_journal(path: &str) -> Option> { for disk in journal_disks().await { - match disk.read_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH).await { + match disk.read_all(super::RUSTFS_META_BUCKET, path).await { Ok(bytes) => return Some(bytes.to_vec()), Err(_) => continue, } @@ -282,35 +387,51 @@ async fn read_journal() -> Option> { /// Write the snapshot to every local disk; returns true when at least one /// disk accepted it, so a total write failure keeps the runtime dirty and /// the next tick retries the persist. -async fn write_journal(data: &[u8]) -> bool { +async fn write_journal(path: &str, data: &[u8]) -> bool { let payload = bytes::Bytes::copy_from_slice(data); let mut any_persisted = false; for disk in journal_disks().await { - match disk - .write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone()) - .await - { + match disk.write_all(super::RUSTFS_META_BUCKET, path, payload.clone()).await { Ok(()) => any_persisted = true, Err(err) => warn_mrf_journal_write(&err), } } - if !data.is_empty() { - counter!("rustfs_heal_mrf_journal_fsync_total").increment(1); - } - gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64); any_persisted } -async fn delete_journal() { - for disk in journal_disks().await { - let _ = disk +async fn delete_journal(path: &str) -> bool { + let disks = journal_disks().await; + if disks.is_empty() { + counter!("rustfs_heal_mrf_journal_delete_failures_total").increment(1); + return false; + } + let mut all_deleted = true; + for disk in disks { + let result = disk .delete( super::RUSTFS_META_BUCKET, - MRF_JOURNAL_PATH, + path, crate::heal::storage_api::owner::EcstoreDeleteOptions::default(), ) .await; + if let Err(err) = result { + // Delete is idempotent: a compatibility mirror that was never + // written (or was already removed) is clean, not a retry state. + if !matches!(err, super::DiskError::FileNotFound | super::DiskError::VolumeNotFound) { + all_deleted = false; + } + } } + if !all_deleted { + counter!("rustfs_heal_mrf_journal_delete_failures_total").increment(1); + } + all_deleted +} + +async fn delete_journals() -> bool { + let authoritative_deleted = delete_journal(MRF_SCOPED_JOURNAL_PATH).await; + let legacy_deleted = delete_journal(MRF_JOURNAL_PATH).await; + authoritative_deleted && legacy_deleted } fn warn_mrf_journal_write(err: &super::DiskError) { @@ -331,7 +452,10 @@ fn warn_mrf_journal_write(err: &super::DiskError) { pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest { let bucket = intent.bucket.to_string(); let object = intent.object.to_string(); - let version_id = intent.version_id.map(|bytes| Uuid::from_bytes(bytes).to_string()); + let version_id = intent + .version_id + .filter(|bytes| *bytes != [0; 16]) + .map(|bytes| Uuid::from_bytes(bytes).to_string()); let (heal_type, priority) = match intent.kind { rustfs_common::mrf_channel::MrfKind::DecodeFailure => ( HealType::ECDecode { @@ -351,18 +475,28 @@ pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest { HealPriority::Normal, ), }; - let mut request = HealRequest::new(heal_type, HealOptions::default(), priority); + let mut options = HealOptions::default(); + if !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) + && let Some(scope) = intent.scope + { + options.pool_index = usize::try_from(scope.pool_index).ok(); + options.set_index = usize::try_from(scope.set_index).ok(); + } + let mut request = HealRequest::new(heal_type, options, priority); request.source = rustfs_common::heal_channel::HealRequestSource::Mrf; request } async fn submit_mrf_heal_request(manager: &HealManager, intent: &MrfIntent) -> crate::Result { let receipt = manager - .submit_mrf_heal_request_with_receipt( + .submit_mrf_heal_request_with_receipt_and_identity( build_heal_request(intent), intent.bucket.clone(), intent.object.clone(), intent.version_id, + intent.kind, + intent.scope, + intent.lease, ) .await?; Ok(receipt.result) @@ -387,16 +521,38 @@ struct MrfRuntime { } impl MrfRuntime { - fn snapshot(&self) -> Vec { - let mut buf = Vec::new(); + fn snapshot(&self) -> (Vec, Vec) { + let mut authoritative = Vec::new(); + let mut legacy = Vec::new(); for intent in self.queue.intents() { - encode_intent(intent, &mut buf); + let scoped_identity = + !matches!(intent.kind, rustfs_common::mrf_channel::MrfKind::MetadataCorruption) && intent.scope.is_some(); + if !encode_intent(intent, &mut authoritative) { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1); + } + if !scoped_identity && !encode_intent(intent, &mut legacy) { + counter!("rustfs_heal_mrf_dropped_total", "reason" => "journal_identity_oversized").increment(1); + } } - buf + (authoritative, legacy) } async fn flush(&mut self) { - let persisted = write_journal(&self.snapshot()).await; + let (authoritative, legacy) = self.snapshot(); + let authoritative_persisted = write_journal(MRF_SCOPED_JOURNAL_PATH, &authoritative).await; + if !authoritative.is_empty() { + counter!("rustfs_heal_mrf_journal_fsync_total").increment(1); + } + gauge!("rustfs_heal_mrf_journal_bytes").set(metric_f64(authoritative.len())); + // Publish the compatibility mirror only after the authoritative + // snapshot has reached at least one disk. This ordering prevents an + // old reader from observing a newer epoch that a new reader cannot + // see when the canonical write is unavailable. + let legacy_persisted = authoritative_persisted && write_journal(MRF_JOURNAL_PATH, &legacy).await; + // Keep dirty until both the authoritative snapshot and its + // compatibility mirror have been accepted; otherwise a one-sided + // failure would never retry the missing file. + let persisted = authoritative_persisted && legacy_persisted; self.new_since_flush = 0; // Keep the dirty flag when every disk write failed: a clean backlog // would otherwise never rewrite, losing the periodic persist retry a @@ -404,7 +560,7 @@ impl MrfRuntime { if persisted { self.dirty = false; } - self.journal_on_disk = true; + self.journal_on_disk |= authoritative_persisted || legacy_persisted; } /// Drain pending intents into the heal manager until it is full, the @@ -430,6 +586,7 @@ impl MrfRuntime { intent.attempts = intent.attempts.saturating_add(1); if intent.attempts >= MRF_MAX_ATTEMPTS { counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1); + rustfs_common::mrf_channel::release_mrf_intent(&intent); continue; } self.queue.push_back(intent); @@ -438,11 +595,13 @@ impl MrfRuntime { } Ok(HealAdmissionResult::Dropped(_)) => { counter!("rustfs_heal_mrf_dropped_total", "reason" => "admission_policy").increment(1); + rustfs_common::mrf_channel::release_mrf_intent(&intent); } Err(_) => { intent.attempts = intent.attempts.saturating_add(1); if intent.attempts >= MRF_MAX_ATTEMPTS { counter!("rustfs_heal_mrf_dropped_total", "reason" => "attempts_exhausted").increment(1); + rustfs_common::mrf_channel::release_mrf_intent(&intent); continue; } self.queue.push_back(intent); @@ -451,8 +610,8 @@ impl MrfRuntime { } } } - gauge!("rustfs_heal_mrf_queue_depth").set(self.queue.depth() as f64); - gauge!("rustfs_heal_mrf_queue_bytes").set(self.queue.bytes() as f64); + gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(self.queue.depth())); + gauge!("rustfs_heal_mrf_queue_bytes").set(metric_f64(self.queue.bytes())); } } @@ -496,7 +655,12 @@ pub async fn replay_journal_once(manager: &Arc) -> usize { let config = MrfConsumerConfig::default(); let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes); let mut backoff_until: Option = None; - replay_into(manager, &mut queue, &mut backoff_until).await + replay_into(manager, &mut queue, &mut backoff_until).await.replayed +} + +struct ReplayOutcome { + replayed: usize, + journal_on_disk: bool, } /// Shared replay core: read + decode + re-arm + delete, then drain what fits. @@ -504,11 +668,25 @@ async fn replay_into( manager: &Arc, queue: &mut MrfQueue, backoff_until: &mut Option, -) -> usize { - let Some(data) = read_journal().await else { - return 0; +) -> ReplayOutcome { + // The scoped file is a complete authoritative snapshot. Fall back to the + // legacy mirror only when the authoritative path is unavailable; merging + // both files could combine records from different flush epochs. + let data = match read_journal(MRF_SCOPED_JOURNAL_PATH).await { + Some(data) => data, + None => match read_journal(MRF_JOURNAL_PATH).await { + Some(data) => data, + None => { + return ReplayOutcome { + replayed: 0, + journal_on_disk: false, + }; + } + }, }; - let (intents, truncated) = decode_journal(&data); + let mut intents = Vec::new(); + let (decoded, truncated) = decode_journal(&data); + intents.extend(decoded); if truncated > 0 { tracing::warn!( target: "rustfs::heal::mrf", @@ -516,12 +694,15 @@ async fn replay_into( "MRF journal had a torn tail; truncated records were discarded" ); } - counter!("rustfs_heal_mrf_replayed_total").increment(intents.len() as u64); + counter!("rustfs_heal_mrf_replayed_total").increment(u64::try_from(intents.len()).unwrap_or(u64::MAX)); let replayed = intents.len(); for intent in intents { - queue.try_push(intent); + let result = queue.try_push_typed(intent.clone()); + if !matches!(result, MrfQueuePushResult::Enqueued) { + rustfs_common::mrf_channel::release_mrf_intent(&intent); + } } - delete_journal().await; + let journal_on_disk = !delete_journals().await; // Drain the replayed intents immediately; whatever the manager refuses // stays armed in `queue` for the consumer's retry loop. @@ -541,7 +722,10 @@ async fn replay_into( } } } - replayed + ReplayOutcome { + replayed, + journal_on_disk, + } } /// Replay the journal, then keep draining the channel into the heal manager @@ -559,7 +743,8 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive // Replay: read the journal, re-arm intents (duplicates are merged by the // manager's dedup key), then drop the file so the next flush starts clean. - replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await; + let replay = replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await; + runtime.journal_on_disk = replay.journal_on_disk; // The replay deleted the journal file; anything still pending (e.g. the // manager was full and backoff armed) must be re-persisted by the next // flush or a crash before it would lose those intents. @@ -587,9 +772,14 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive return; } for intent in batch.drain(..) { - if runtime.queue.try_push(intent) { - runtime.new_since_flush += 1; - runtime.dirty = true; + match runtime.queue.try_push_typed(intent.clone()) { + MrfQueuePushResult::Enqueued => { + runtime.new_since_flush += 1; + runtime.dirty = true; + } + MrfQueuePushResult::Coalesced | MrfQueuePushResult::Rejected => { + rustfs_common::mrf_channel::release_mrf_intent(&intent); + } } } runtime.dispatch(manager.as_ref()).await; @@ -613,13 +803,14 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive TickAction::DeleteJournal => { // All intents consumed: remove the journal so a restart // replays nothing (mirrors MinIO's post-replay unlink). - delete_journal().await; - runtime.journal_on_disk = false; - gauge!("rustfs_heal_mrf_journal_bytes").set(0.0); + if delete_journals().await { + runtime.journal_on_disk = false; + gauge!("rustfs_heal_mrf_journal_bytes").set(0.0); + } } TickAction::Idle => {} } - gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64); + gauge!("rustfs_heal_mrf_queue_depth").set(metric_f64(runtime.queue.depth())); } } } @@ -664,6 +855,8 @@ mod tests { object: StdArc::from(object), version_id: Some([7u8; 16]), kind: MrfKind::DecodeFailure, + scope: None, + lease: None, enqueued_at_ms: 1_700_000_000_000, attempts, } @@ -694,17 +887,101 @@ mod tests { fn queue_enforces_count_and_byte_ceilings() { let mut queue = MrfQueue::new(2, usize::MAX); assert!(queue.try_push(intent("b", "o", 0))); - assert!(queue.try_push(intent("b", "o", 0))); - assert!(!queue.try_push(intent("b", "o", 0)), "count ceiling must drop"); + assert!(queue.try_push(intent("b", "o2", 0))); + assert!(!queue.try_push(intent("b", "o3", 0)), "count ceiling must drop"); let mut tiny = MrfQueue::new(usize::MAX, intent("bucket", "object", 0).estimated_bytes()); assert!(tiny.try_push(intent("bucket", "object", 0))); assert!( - !tiny.try_push(intent("bucket", "object", 0)), + !tiny.try_push(intent("bucket", "object2", 0)), "byte budget must drop before the second intent fits" ); } + #[test] + fn duplicate_mrf_intents_coalesce_to_one_execution() { + let mut queue = MrfQueue::new(1000, usize::MAX); + let mut enqueued = 0; + let mut coalesced = 0; + assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued); + enqueued += 1; + for _ in 0..999 { + match queue.try_push_typed(intent("bucket", "object", 0)) { + MrfQueuePushResult::Coalesced => coalesced += 1, + other => panic!("duplicate intent was not coalesced: {other:?}"), + } + } + assert_eq!(enqueued, 1); + assert_eq!(coalesced, 999); + assert_eq!(queue.depth(), 1); + } + + #[test] + fn mrf_dedupe_does_not_merge_adjacent_version_pool_or_kind() { + let mut queue = MrfQueue::new(8, usize::MAX); + let mut first = intent("bucket", "object", 0); + first.kind = MrfKind::PartialWrite; + first.scope = Some(rustfs_common::mrf_channel::MrfScope { + pool_index: 1, + set_index: 1, + }); + assert!(queue.try_push(first.clone())); + first.version_id = Some([8u8; 16]); + assert!(queue.try_push(first)); + let mut other_scope = intent("bucket", "object", 0); + other_scope.kind = MrfKind::PartialWrite; + other_scope.scope = Some(rustfs_common::mrf_channel::MrfScope { + pool_index: 2, + set_index: 1, + }); + assert!(queue.try_push(other_scope)); + let mut other_kind = intent("bucket", "object", 0); + other_kind.kind = MrfKind::DecodeFailure; + other_kind.scope = None; + assert!(queue.try_push(other_kind)); + assert_eq!(queue.depth(), 4); + } + + #[test] + fn mrf_dedupe_full_returns_rejected_with_durable_pending() { + let mut queue = MrfQueue::new(1, usize::MAX); + assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued); + assert_eq!(queue.try_push_typed(intent("bucket", "other", 0)), MrfQueuePushResult::Rejected); + assert_eq!(queue.depth(), 1); + let mut snapshot = Vec::new(); + assert!(encode_intent(queue.intents().next().expect("resident intent"), &mut snapshot)); + assert!(!snapshot.is_empty(), "the resident intent remains journalable after rejection"); + } + + #[test] + fn mrf_dedupe_failure_releases_key_for_retry() { + let mut queue = MrfQueue::new(1, usize::MAX); + assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued); + let _failed = queue.pop_front().expect("queued intent"); + assert_eq!(queue.try_push_typed(intent("bucket", "object", 1)), MrfQueuePushResult::Enqueued); + assert_eq!(queue.depth(), 1); + } + + #[test] + fn mrf_dedupe_key_and_map_are_bounded() { + let mut queue = MrfQueue::new(2, usize::MAX); + assert_eq!(queue.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued); + assert_eq!(queue.try_push_typed(intent("bucket", "other", 0)), MrfQueuePushResult::Enqueued); + assert_eq!(queue.pending_keys.len(), 2); + assert_eq!(queue.try_push_typed(intent("bucket", "third", 0)), MrfQueuePushResult::Rejected); + assert_eq!(queue.depth(), 2); + } + + #[test] + fn cross_node_duplicate_execution_remains_idempotent() { + // Node-local ingress maps intentionally do not merge across nodes; + // the manager's existing identity key absorbs the duplicate later. + let mut node_a = MrfQueue::new(8, usize::MAX); + let mut node_b = MrfQueue::new(8, usize::MAX); + assert_eq!(node_a.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued); + assert_eq!(node_b.try_push_typed(intent("bucket", "object", 0)), MrfQueuePushResult::Enqueued); + } + #[test] fn journal_roundtrip_preserves_intents() { let intents = vec![ @@ -715,6 +992,8 @@ mod tests { object: StdArc::from("object/c"), version_id: None, kind: MrfKind::MetadataCorruption, + scope: None, + lease: None, enqueued_at_ms: 5, attempts: 1, }, @@ -766,6 +1045,8 @@ mod tests { object: StdArc::from("o"), version_id: None, kind: MrfKind::MetadataCorruption, + scope: None, + lease: None, enqueued_at_ms: 0, attempts: 0, }); @@ -777,6 +1058,8 @@ mod tests { object: StdArc::from("o"), version_id: None, kind: MrfKind::PartialWrite, + scope: None, + lease: None, enqueued_at_ms: 0, attempts: 0, }); diff --git a/crates/heal/tests/mrf_pipeline_test.rs b/crates/heal/tests/mrf_pipeline_test.rs index 137cdd75d..2fdb4603f 100644 --- a/crates/heal/tests/mrf_pipeline_test.rs +++ b/crates/heal/tests/mrf_pipeline_test.rs @@ -35,6 +35,7 @@ use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, Pool const META_BUCKET: &str = ".rustfs.sys"; const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin"; +const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin"; async fn heal_env() -> (Vec, Arc) { let env = rustfs_test_utils::TestECStoreEnv::builder() @@ -79,14 +80,18 @@ fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16] body } -fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) { +fn write_journal_path_to_disks(disk_paths: &[std::path::PathBuf], relative_path: &str, data: &[u8]) { for path in disk_paths { - let journal = path.join(META_BUCKET).join(JOURNAL_REL); + let journal = path.join(META_BUCKET).join(relative_path); std::fs::create_dir_all(journal.parent().expect("journal parent")).expect("create journal dir"); std::fs::write(&journal, data).expect("write journal fixture"); } } +fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) { + write_journal_path_to_disks(disk_paths, JOURNAL_REL, data); +} + async fn wait_until(deadline: Duration, mut probe: F) -> bool where F: FnMut() -> Fut, @@ -187,8 +192,73 @@ async fn journal_replay_arms_intents_and_deletes_the_file() { .all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()), "the journal file must be removed after a successful replay" ); + assert!( + disk_paths + .iter() + .all(|path| !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()), + "the authoritative journal file must also be removed after replay" + ); let snapshot = manager.operations_snapshot().await; assert_eq!(snapshot.queued_by_priority.urgent, 1, "the decode-failure record must replay as Urgent"); assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal"); } + +/// A canonical snapshot and its compatibility mirror may differ after a +/// partial flush. Replay must choose the complete canonical epoch instead of +/// combining records that never coexisted in memory. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[serial] +async fn authoritative_journal_is_not_merged_with_legacy_mirror() { + let (disk_paths, storage) = heal_env().await; + let mut endpoints: Vec = disk_paths + .iter() + .map(|p| Endpoint::try_from(p.to_string_lossy().as_ref()).expect("endpoint from disk path")) + .collect(); + for (i, endpoint) in endpoints.iter_mut().enumerate() { + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(i); + } + let pool = PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: endpoints.len(), + endpoints: Endpoints::from(endpoints), + cmd_line: "mrf-authoritative-test".to_string(), + platform: String::new(), + }; + init_local_disks(EndpointServerPools::from(vec![pool])) + .await + .expect("local disks should register"); + + let authoritative = journal_record(1, "authoritative-bucket", "authoritative-object", None, 0); + let legacy = journal_record(1, "legacy-bucket", "legacy-object", None, 0); + write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &authoritative); + write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &legacy); + + let manager = make_manager(storage); + let replayed = mrf_queue::replay_journal_once(&manager).await; + assert_eq!(replayed, 1, "only the authoritative snapshot epoch may replay"); + + let snapshot = manager.operations_snapshot().await; + assert_eq!(snapshot.queued_by_source.mrf, 1); + assert!( + disk_paths.iter().all(|path| { + !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() + && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() + }), + "replay cleanup must remove both journal paths" + ); + + // A scoped-only snapshot is valid during a rollout where no legacy + // compatibility mirror was written. Missing legacy files must not leave + // the runtime in a permanent cleanup-retry state. + let scoped_only = journal_record(1, "scoped-only-bucket", "scoped-only-object", None, 0); + write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_only); + assert_eq!(mrf_queue::replay_journal_once(&manager).await, 1); + assert!(disk_paths.iter().all(|path| { + !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists() + && !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists() + })); +} diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 693dd6ebc..81a2a7377 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -1375,13 +1375,14 @@ impl FolderScanner { // Single-flight (backlog#1894 axis A) — the // recording mode and its guarantees are pinned by // corrupt_metadata_recording below. - let mrf_accepted = rustfs_common::mrf_channel::try_send_mrf_intent( + let mrf_result = rustfs_common::mrf_channel::try_send_mrf_intent_typed( rustfs_common::mrf_channel::MrfKind::MetadataCorruption, &item.bucket, &object, None, + None, ); - match corrupt_metadata_recording(mrf_accepted) { + match corrupt_metadata_recording(mrf_result) { CorruptMetadataRecording::LedgerOnly => { // Recorded as Full (retry-later): admission // for this target happens in the MRF diff --git a/crates/scanner/src/scanner_folder/item_actions.rs b/crates/scanner/src/scanner_folder/item_actions.rs index 15bd6736b..0f2d82641 100644 --- a/crates/scanner/src/scanner_folder/item_actions.rs +++ b/crates/scanner/src/scanner_folder/item_actions.rs @@ -50,11 +50,12 @@ pub(super) enum CorruptMetadataRecording { ImmediateAndLedger, } -pub(super) fn corrupt_metadata_recording(mrf_accepted: bool) -> CorruptMetadataRecording { - if mrf_accepted { - CorruptMetadataRecording::LedgerOnly - } else { - CorruptMetadataRecording::ImmediateAndLedger +pub(super) fn corrupt_metadata_recording(result: rustfs_common::mrf_channel::MrfIngressResult) -> CorruptMetadataRecording { + match result { + rustfs_common::mrf_channel::MrfIngressResult::Enqueued | rustfs_common::mrf_channel::MrfIngressResult::Coalesced => { + CorruptMetadataRecording::LedgerOnly + } + rustfs_common::mrf_channel::MrfIngressResult::Dropped(_) => CorruptMetadataRecording::ImmediateAndLedger, } } diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index 503b0e75f..3b01b9be6 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -48,8 +48,20 @@ fn scanner_alert_wire_names_match_canonical_event_names() { /// the backstop survives regardless of delivery. #[test] fn corrupt_metadata_recording_maps_delivery_to_backstop() { - assert_eq!(corrupt_metadata_recording(true), CorruptMetadataRecording::LedgerOnly); - assert_eq!(corrupt_metadata_recording(false), CorruptMetadataRecording::ImmediateAndLedger); + assert_eq!( + corrupt_metadata_recording(rustfs_common::mrf_channel::MrfIngressResult::Enqueued), + CorruptMetadataRecording::LedgerOnly + ); + assert_eq!( + corrupt_metadata_recording(rustfs_common::mrf_channel::MrfIngressResult::Coalesced), + CorruptMetadataRecording::LedgerOnly + ); + assert_eq!( + corrupt_metadata_recording(rustfs_common::mrf_channel::MrfIngressResult::Dropped( + rustfs_common::mrf_channel::MrfDropReason::Full + )), + CorruptMetadataRecording::ImmediateAndLedger + ); } fn cooldown_map_len() -> usize { From 9a8ca3a7a96a455d83fea94b8b8eb2a21a648658 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 16:45:34 +0800 Subject: [PATCH 15/41] fix(heal): add bounded resume artifact inspection (#6420) Co-authored-by: houseme --- crates/heal/src/heal/manager.rs | 50 ++- crates/heal/src/heal/resume.rs | 2 + crates/heal/src/heal/resume/gc.rs | 666 ++++++++++++++++++++++++++++++ 3 files changed, 717 insertions(+), 1 deletion(-) create mode 100644 crates/heal/src/heal/resume/gc.rs diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 30ef94446..7c98b91f5 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -14,7 +14,7 @@ use crate::heal::{ progress::{HealProgress, HealStatistics}, - resume::{ReplacementPhase, ResumeManager, ResumeState, ResumeUtils}, + resume::{ReplacementPhase, ResumeGc, ResumeManager, ResumeState, ResumeUtils}, storage::HealStorageAPI, task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType, demote_to_debug_when}, }; @@ -53,9 +53,11 @@ const EVENT_HEAL_MAINLINE_THROTTLE: &str = "heal_mainline_throttle"; const EVENT_HEAL_SCHEDULER_STATE: &str = "heal_scheduler_state"; const EVENT_HEAL_QUEUE_STATE: &str = "heal_queue_state"; const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown"; +const EVENT_HEAL_RESUME_GC: &str = "heal_resume_gc"; const LEGACY_ROOT_HEAL_PATH: &str = "."; const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3; const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30); +const RESUME_GC_INTERVAL: Duration = Duration::from_secs(60 * 60); // Admission/scheduler outcomes for per-object requests (Object/Metadata/ // ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner @@ -1165,6 +1167,49 @@ impl HealManager { Ok(()) } + /// Start the bounded resume-state inspector. Destructive GC remains + /// disabled until the durable owner/CAS contract from backlog#1927 is + /// available; this task therefore cannot remove an active or stale file. + async fn start_resume_gc(&self) { + let cancel = self.cancel_token.clone(); + tokio::spawn(async move { + let mut gc_by_disk = HashMap::::new(); + let mut ticker = interval(RESUME_GC_INTERVAL); + loop { + tokio::select! { + _ = cancel.cancelled() => break, + _ = ticker.tick() => { + let disks = { + let local_disk_map = local_disk_map_read().await; + local_disk_map.values().flatten().cloned().collect::>() + }; + for disk in disks { + let disk_key = disk.endpoint().to_string(); + let gc = gc_by_disk.entry(disk_key).or_default(); + tokio::select! { + _ = cancel.cancelled() => return, + result = gc.inspect_disk(&disk) => { + if let Err(error) = result { + warn!( + target: "rustfs::heal::manager", + event = EVENT_HEAL_RESUME_GC, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_MANAGER, + state = "inspect_failed", + endpoint = %disk.endpoint(), + error = %error, + "Heal resume GC inspection failed" + ); + } + } + } + } + } + } + } + }); + } + /// Create new HealManager pub fn new(storage: Arc, config: Option) -> Self { Self::new_with_workload_provider(storage, config, None) @@ -1230,6 +1275,9 @@ impl HealManager { // competing task for the same set. self.process_unclean_shutdown().await; + // Inspect resume artifacts in a bounded, fail-closed background task. + self.start_resume_gc().await; + // start auto disk scanner to heal unformatted disks if self.config.read().await.enable_auto_heal { self.start_auto_disk_scanner().await?; diff --git a/crates/heal/src/heal/resume.rs b/crates/heal/src/heal/resume.rs index 239713d3a..b19a2090c 100644 --- a/crates/heal/src/heal/resume.rs +++ b/crates/heal/src/heal/resume.rs @@ -28,10 +28,12 @@ use super::{ }; mod checkpoint; +mod gc; mod replacement; mod utils; pub use checkpoint::{CheckpointManager, ResumeCheckpoint}; +pub(crate) use gc::ResumeGc; pub(crate) use replacement::replacement_target_identities_match; use replacement::replacement_targets_match_identities; pub use replacement::{ diff --git a/crates/heal/src/heal/resume/gc.rs b/crates/heal/src/heal/resume/gc.rs new file mode 100644 index 000000000..ab5ae9262 --- /dev/null +++ b/crates/heal/src/heal/resume/gc.rs @@ -0,0 +1,666 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Bounded inspection of heal resume artifacts. +//! +//! The durable owner/CAS and quarantine primitives belong to backlog #1927 and +//! are not part of the current base revision. This module is therefore +//! deliberately inspect-only. In particular, it must never turn an age check +//! into a delete: ordinary heal writers still publish raw files on this base, +//! so a GC-side compare-and-delete would not fence a concurrent claim. + +use metrics::counter; +use std::{ + collections::BTreeMap, + path::{Component, Path}, + time::{SystemTime, UNIX_EPOCH}, +}; +use tokio::io::AsyncReadExt; + +use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, RUSTFS_META_BUCKET, storage_api::owner::EcstoreDiskAPI}; +use super::{ + LEGACY_REPLACEMENT_RECOVERY_MARKER_FILE, REPLACEMENT_COMPLETION_PROOF_FILE, REPLACEMENT_INTENT_FILE, + REPLACEMENT_INTENT_SEAL_FILE, RESUME_CHECKPOINT_FILE, RESUME_PROGRESS_FILE, RESUME_STATE_FILE, ResumeCheckpoint, ResumeState, + checkpoint::CURRENT_CHECKPOINT_SCHEMA, +}; +use crate::{Error, Result}; + +const DEFAULT_ENTRY_BUDGET: usize = 256; +const DEFAULT_BYTE_BUDGET: usize = 4 * 1024 * 1024; +const GC_METRIC: &str = "rustfs_heal_resume_gc_inspected_total"; +const GC_ERROR_METRIC: &str = "rustfs_heal_resume_gc_inspect_errors_total"; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct ResumeGcConfig { + /// Maximum number of directory entries considered in one disk pass. + pub(crate) max_entries: usize, + /// Maximum number of bytes read in one disk pass. + pub(crate) max_bytes: usize, +} + +impl Default for ResumeGcConfig { + fn default() -> Self { + Self { + max_entries: DEFAULT_ENTRY_BUDGET, + max_bytes: DEFAULT_BYTE_BUDGET, + } + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ResumeGcReport { + /// Directory entries visited (including malformed entries). + pub(crate) inspected: usize, + pub(crate) active_skipped: usize, + pub(crate) orphaned: usize, + /// Records that must be handed to #1927's quarantine owner. + pub(crate) quarantine_required: usize, + pub(crate) generation_skipped: usize, + pub(crate) clock_skew: usize, + pub(crate) read_errors: usize, + pub(crate) retained: usize, + /// True while #1927's durable claim/quarantine capability is unavailable. + pub(crate) destructive_disabled: bool, + pub(crate) budget_exhausted: bool, +} + +#[derive(Debug, Default)] +pub(crate) struct ResumeGc { + config: ResumeGcConfig, + /// Alternate the first namespace so a full ordinary page cannot starve + /// replacement recovery when the list API has no continuation token. + recovery_first: bool, +} + +impl ResumeGc { + #[cfg(test)] + fn with_config(config: ResumeGcConfig) -> Self { + Self { + config, + recovery_first: false, + } + } + + /// Inspect one bounded page from each resume namespace. + /// + /// The caller owns scheduling and cancellation. A malformed or unreadable + /// artifact is reported and retained so a later pass can retry it; no + /// individual artifact error aborts the rest of the bounded page. + pub(crate) async fn inspect_disk(&mut self, disk: &DiskStore) -> Result { + let mut report = ResumeGcReport { + destructive_disabled: true, + ..ResumeGcReport::default() + }; + if self.config.max_entries == 0 || self.config.max_bytes == 0 { + report.budget_exhausted = true; + return Ok(report); + } + + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + let mut bytes_read = 0usize; + let recovery_first = self.recovery_first; + self.recovery_first = !self.recovery_first; + if recovery_first { + inspect_namespace(self.config, disk, &replacement_prefix(), true, now, &mut bytes_read, &mut report).await?; + if !report.budget_exhausted { + inspect_namespace(self.config, disk, BUCKET_META_PREFIX, false, now, &mut bytes_read, &mut report).await?; + } + } else { + inspect_namespace(self.config, disk, BUCKET_META_PREFIX, false, now, &mut bytes_read, &mut report).await?; + if !report.budget_exhausted { + inspect_namespace(self.config, disk, &replacement_prefix(), true, now, &mut bytes_read, &mut report).await?; + } + } + + counter!(GC_METRIC).increment(u64::try_from(report.inspected).unwrap_or(u64::MAX)); + counter!(GC_ERROR_METRIC).increment(u64::try_from(report.read_errors).unwrap_or(u64::MAX)); + Ok(report) + } +} + +#[derive(Debug, Default, Clone, Copy)] +struct ArtifactSet { + state: bool, + checkpoint: bool, + progress: bool, + replacement_intent: bool, + proof: bool, + seal: bool, + legacy_marker: bool, + temporary: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ArtifactKind { + State, + Checkpoint, + Progress, + ReplacementIntent, + Proof, + Seal, + LegacyMarker, +} + +impl ArtifactSet { + fn add(&mut self, kind: ArtifactKind, temporary: bool) { + self.temporary |= temporary; + match kind { + ArtifactKind::State => self.state = true, + ArtifactKind::Checkpoint => self.checkpoint = true, + ArtifactKind::Progress => self.progress = true, + ArtifactKind::ReplacementIntent => self.replacement_intent = true, + ArtifactKind::Proof => self.proof = true, + ArtifactKind::Seal => self.seal = true, + ArtifactKind::LegacyMarker => self.legacy_marker = true, + } + } +} + +#[derive(Debug, Clone, Copy)] +struct InspectOptions { + max_bytes: usize, + now: u64, +} + +struct InspectProgress<'a> { + bytes_read: &'a mut usize, + report: &'a mut ResumeGcReport, +} + +fn replacement_prefix() -> String { + super::replacement_recovery_dir().to_string_lossy().into_owned() +} + +async fn inspect_namespace( + config: ResumeGcConfig, + disk: &DiskStore, + prefix: &str, + replacement: bool, + now: u64, + bytes_read: &mut usize, + report: &mut ResumeGcReport, +) -> Result<()> { + let remaining = config.max_entries.saturating_sub(report.inspected); + if remaining == 0 { + report.budget_exhausted = true; + return Ok(()); + } + let count = i32::try_from(remaining).unwrap_or(i32::MAX); + let mut entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, prefix, count).await { + Ok(entries) => entries, + Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(()), + Err(error) => return Err(error.into()), + }; + entries.sort_unstable(); + + let mut artifacts = BTreeMap::::new(); + for entry in entries { + if report.inspected >= config.max_entries { + report.budget_exhausted = true; + break; + } + report.inspected += 1; + let Some((task_id, kind, temporary)) = artifact_name(&entry, replacement) else { + report.quarantine_required += 1; + report.retained += 1; + continue; + }; + artifacts.entry(task_id).or_default().add(kind, temporary); + } + if report.inspected >= config.max_entries { + report.budget_exhausted = true; + } + + for (task_id, artifacts) in artifacts { + if *bytes_read >= config.max_bytes { + report.budget_exhausted = true; + break; + } + let options = InspectOptions { + max_bytes: config.max_bytes, + now, + }; + let mut progress = InspectProgress { bytes_read, report }; + inspect_task(options, disk, prefix, replacement, &task_id, artifacts, &mut progress).await?; + } + Ok(()) +} + +async fn inspect_task( + options: InspectOptions, + disk: &DiskStore, + prefix: &str, + replacement: bool, + task_id: &str, + artifacts: ArtifactSet, + progress: &mut InspectProgress<'_>, +) -> Result<()> { + let legacy_replacement = !replacement && !artifacts.state && artifacts.replacement_intent; + let state_suffix = if replacement || legacy_replacement { + REPLACEMENT_INTENT_FILE + } else { + RESUME_STATE_FILE + }; + let state_path = artifact_path(prefix, task_id, state_suffix)?; + let state = match read_bounded(disk, &state_path, options.max_bytes, progress.bytes_read).await { + ReadOutcome::Missing => { + progress.report.orphaned += 1; + progress.report.retained += 1; + return Ok(()); + } + ReadOutcome::TooLarge => { + progress.report.quarantine_required += 1; + progress.report.retained += 1; + progress.report.budget_exhausted = true; + return Ok(()); + } + ReadOutcome::Error => { + progress.report.read_errors += 1; + progress.report.retained += 1; + return Ok(()); + } + ReadOutcome::Bytes(bytes) => bytes, + }; + + let parsed: ResumeState = match serde_json::from_slice(&state) { + Ok(state) => state, + Err(_) => { + progress.report.quarantine_required += 1; + progress.report.retained += 1; + return Ok(()); + } + }; + if parsed.schema_version > super::CURRENT_RESUME_SCHEMA || parsed.task_id != task_id { + progress.report.quarantine_required += 1; + progress.report.retained += 1; + return Ok(()); + } + if persistent_age_seconds(options.now, parsed.last_update).is_none() { + progress.report.clock_skew += 1; + progress.report.retained += 1; + return Ok(()); + } + if let Some(generation) = parsed.replacement_generation.as_deref() + && !claim_generation_matches(Some(generation), Some(task_id)) + { + progress.report.generation_skipped += 1; + progress.report.retained += 1; + return Ok(()); + } + + if !replacement && artifacts.checkpoint { + let checkpoint_path = artifact_path(prefix, task_id, RESUME_CHECKPOINT_FILE)?; + match read_bounded(disk, &checkpoint_path, options.max_bytes, progress.bytes_read).await { + ReadOutcome::Bytes(bytes) => match serde_json::from_slice::(&bytes) { + Ok(checkpoint) if checkpoint.schema_version <= CURRENT_CHECKPOINT_SCHEMA && checkpoint.task_id == task_id => {} + _ => { + progress.report.quarantine_required += 1; + progress.report.retained += 1; + } + }, + ReadOutcome::Missing => { + progress.report.orphaned += 1; + progress.report.retained += 1; + } + ReadOutcome::TooLarge => { + progress.report.quarantine_required += 1; + progress.report.retained += 1; + progress.report.budget_exhausted = true; + } + ReadOutcome::Error => { + progress.report.read_errors += 1; + progress.report.retained += 1; + } + } + } + + if artifacts.state && artifacts.replacement_intent { + // A task cannot have two authoritative state records in one namespace; + // preserve both until the durable owner can resolve the generation. + progress.report.quarantine_required += 1; + } + + if !parsed.completed { + progress.report.active_skipped += 1; + } + // The state and all associated evidence remain recoverable until #1927 + // supplies a common generation/CAS transition and quarantine owner. + progress.report.retained += 1; + Ok(()) +} + +enum ReadOutcome { + Bytes(Vec), + Missing, + TooLarge, + Error, +} + +async fn read_bounded(disk: &DiskStore, path: &str, max_bytes: usize, bytes_read: &mut usize) -> ReadOutcome { + let remaining = max_bytes.saturating_sub(*bytes_read); + if remaining == 0 { + return ReadOutcome::TooLarge; + } + let read_len = remaining.saturating_add(1); + let reader = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path).await { + Ok(reader) => reader, + Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return ReadOutcome::Missing, + Err(_) => return ReadOutcome::Error, + }; + let mut bytes = Vec::with_capacity(read_len.min(64 * 1024)); + let Ok(read_len) = u64::try_from(read_len) else { + return ReadOutcome::TooLarge; + }; + if reader.take(read_len).read_to_end(&mut bytes).await.is_err() { + return ReadOutcome::Error; + } + if bytes.len() > remaining { + *bytes_read = max_bytes; + return ReadOutcome::TooLarge; + } + *bytes_read = bytes_read.saturating_add(bytes.len()); + ReadOutcome::Bytes(bytes) +} + +fn artifact_path(prefix: &str, task_id: &str, suffix: &str) -> Result { + if super::validate_resume_task_id(task_id).is_err() { + return Err(Error::other("invalid resume task id")); + } + Path::new(prefix) + .join(format!("{task_id}_{suffix}")) + .to_str() + .map(str::to_owned) + .ok_or_else(|| Error::other("invalid resume artifact path")) +} + +/// Parse one directory entry without ever accepting a path component supplied +/// by a client. DiskAPI filters symlinks, but this check also protects remote +/// implementations and future mutating callers from traversal/reparse names. +fn artifact_name(entry: &str, replacement: bool) -> Option<(String, ArtifactKind, bool)> { + let path = Path::new(entry); + if entry.is_empty() || path.components().count() != 1 || !matches!(path.components().next(), Some(Component::Normal(_))) { + return None; + } + let (stem, temporary) = entry + .strip_suffix(".tmp") + .map(|stem| (stem, true)) + .or_else(|| entry.strip_suffix(".bak").map(|stem| (stem, true))) + .unwrap_or((entry, false)); + let suffixes: &[(&str, ArtifactKind)] = if replacement { + &[ + (REPLACEMENT_INTENT_FILE, ArtifactKind::ReplacementIntent), + (REPLACEMENT_COMPLETION_PROOF_FILE, ArtifactKind::Proof), + (REPLACEMENT_INTENT_SEAL_FILE, ArtifactKind::Seal), + ] + } else { + &[ + (RESUME_STATE_FILE, ArtifactKind::State), + (RESUME_CHECKPOINT_FILE, ArtifactKind::Checkpoint), + (RESUME_PROGRESS_FILE, ArtifactKind::Progress), + (LEGACY_REPLACEMENT_RECOVERY_MARKER_FILE, ArtifactKind::LegacyMarker), + (REPLACEMENT_INTENT_FILE, ArtifactKind::ReplacementIntent), + (REPLACEMENT_COMPLETION_PROOF_FILE, ArtifactKind::Proof), + (REPLACEMENT_INTENT_SEAL_FILE, ArtifactKind::Seal), + ] + }; + suffixes.iter().find_map(|(suffix, kind)| { + stem.strip_suffix(&format!("_{suffix}")) + .filter(|task_id| super::validate_resume_task_id(task_id).is_ok()) + .map(|task_id| (task_id.to_string(), *kind, temporary)) + }) +} + +fn persistent_age_seconds(now: u64, updated: u64) -> Option { + now.checked_sub(updated) +} + +fn claim_generation_matches(observed: Option<&str>, expected: Option<&str>) -> bool { + expected.is_none() || observed == expected +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::heal::{DiskOption, Endpoint, new_disk}; + use tempfile::TempDir; + use uuid::Uuid; + + async fn test_disk() -> (TempDir, DiskStore) { + let temp = TempDir::new().expect("test disk directory"); + let endpoint = Endpoint::try_from(temp.path().to_string_lossy().as_ref()).expect("test endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("test disk"); + match disk.make_volume(RUSTFS_META_BUCKET).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(error) => panic!("metadata volume: {error}"), + } + match disk.make_volume(&format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}")).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(error) => panic!("resume volume: {error}"), + } + (temp, disk) + } + + async fn write_state(disk: &DiskStore, state: &ResumeState) { + let path = format!("{BUCKET_META_PREFIX}/{}_{}", state.task_id, RESUME_STATE_FILE); + disk.write_all(RUSTFS_META_BUCKET, &path, serde_json::to_vec(state).unwrap().into()) + .await + .expect("resume state"); + } + + async fn write_replacement_state(disk: &DiskStore, state: &ResumeState) { + let volume = format!("{RUSTFS_META_BUCKET}/{BUCKET_META_PREFIX}/ahm-replacement"); + match disk.make_volume(&volume).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(error) => panic!("replacement volume: {error}"), + } + let path = format!("{}/{}_{}", replacement_prefix(), state.task_id, REPLACEMENT_INTENT_FILE); + disk.write_all(RUSTFS_META_BUCKET, &path, serde_json::to_vec(state).unwrap().into()) + .await + .expect("replacement state"); + } + + #[tokio::test] + async fn production_gc_does_not_delete_claimed_resume_state() { + let (_temp, disk) = test_disk().await; + let task_id = Uuid::new_v4().to_string(); + write_state(&disk, &ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![])).await; + let report = ResumeGc::default().inspect_disk(&disk).await.expect("inspect"); + assert_eq!(report.active_skipped, 1); + assert!( + disk.read_all(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}")) + .await + .is_ok() + ); + } + + #[tokio::test] + async fn production_gc_generation_mismatch_is_skip() { + let (_temp, disk) = test_disk().await; + let task_id = Uuid::new_v4().to_string(); + let mut state = ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![]); + state.replacement_generation = Some(Uuid::new_v4().to_string()); + write_state(&disk, &state).await; + assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().generation_skipped, 1); + } + + #[cfg(unix)] + #[tokio::test] + async fn production_gc_rejects_symlink_or_outside_prefix() { + let (_temp, disk) = test_disk().await; + let id = Uuid::new_v4().to_string(); + let root = EcstoreDiskAPI::path(disk.as_ref()); + let outside = root.join("outside-resume-state"); + std::fs::write(&outside, b"must remain").expect("outside fixture"); + let symlink = root + .join(RUSTFS_META_BUCKET) + .join(BUCKET_META_PREFIX) + .join(format!("{id}_{RESUME_STATE_FILE}")); + std::os::unix::fs::symlink(&outside, &symlink).expect("symlink fixture"); + let report = ResumeGc::default().inspect_disk(&disk).await.expect("inspect"); + assert_eq!(report.inspected, 0, "symlinks are not eligible artifacts"); + assert!(outside.exists()); + assert!(artifact_name(&format!("{id}_{RESUME_STATE_FILE}"), false).is_some()); + assert!(artifact_name(&format!("../{id}_{RESUME_STATE_FILE}"), false).is_none()); + assert!(artifact_name(&format!("{id}/link_{RESUME_STATE_FILE}"), false).is_none()); + } + + #[cfg(not(unix))] + #[test] + fn production_gc_rejects_symlink_or_outside_prefix() { + let id = Uuid::new_v4().to_string(); + assert!(artifact_name(&format!("{id}_{RESUME_STATE_FILE}"), false).is_some()); + assert!(artifact_name(&format!("../{id}_{RESUME_STATE_FILE}"), false).is_none()); + assert!(artifact_name(&format!("{id}/link_{RESUME_STATE_FILE}"), false).is_none()); + } + + #[tokio::test] + async fn production_gc_delete_failure_leaves_recoverable_state() { + let (_temp, disk) = test_disk().await; + let task_id = Uuid::new_v4().to_string(); + let mut state = ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![]); + state.mark_completed(); + write_state(&disk, &state).await; + ResumeGc::default().inspect_disk(&disk).await.expect("inspect"); + assert!( + disk.read_all(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}")) + .await + .is_ok() + ); + } + + #[tokio::test] + async fn production_gc_handles_clock_skew_and_restart() { + let (_temp, disk) = test_disk().await; + let task_id = Uuid::new_v4().to_string(); + let mut state = ResumeState::new(task_id, "set".into(), "disk".into(), vec![]); + state.last_update = u64::MAX; + write_state(&disk, &state).await; + assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().clock_skew, 1); + assert!(persistent_age_seconds(1, 2).is_none()); + } + + #[tokio::test] + async fn production_gc_100k_states_respects_budget() { + let (_temp, disk) = test_disk().await; + for _ in 0..8 { + let state = ResumeState::new(Uuid::new_v4().to_string(), "set".into(), "disk".into(), vec![]); + write_state(&disk, &state).await; + } + let config = ResumeGcConfig { + max_entries: 2, + max_bytes: usize::MAX, + }; + let report = ResumeGc::with_config(config).inspect_disk(&disk).await.unwrap(); + assert!(report.inspected <= 2); + assert!(report.budget_exhausted); + } + + #[tokio::test] + async fn production_gc_recovery_namespace_is_not_starved() { + let (_temp, disk) = test_disk().await; + let ordinary = ResumeState::new(Uuid::new_v4().to_string(), "set".into(), "disk".into(), vec![]); + write_state(&disk, &ordinary).await; + let replacement_id = Uuid::new_v4().to_string(); + let mut replacement = ResumeState::new(replacement_id, "set".into(), "disk".into(), vec![]); + replacement.replacement_generation = Some(replacement.task_id.clone()); + write_replacement_state(&disk, &replacement).await; + + let mut gc = ResumeGc::with_config(ResumeGcConfig { + max_entries: 1, + max_bytes: usize::MAX, + }); + assert_eq!(gc.inspect_disk(&disk).await.unwrap().inspected, 1); + let second = gc.inspect_disk(&disk).await.unwrap(); + assert_eq!(second.inspected, 1, "the next bounded pass must start at recovery"); + assert_eq!(second.active_skipped, 1); + } + + #[tokio::test] + async fn production_gc_pairs_orphan_checkpoint_and_resume() { + let (_temp, disk) = test_disk().await; + let task_id = Uuid::new_v4().to_string(); + let checkpoint = ResumeCheckpoint::new(task_id.clone()); + let path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_FILE}"); + disk.write_all(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&checkpoint).unwrap().into()) + .await + .expect("checkpoint"); + assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().orphaned, 1); + } + + #[tokio::test] + async fn production_gc_does_not_delete_slow_active_task() { + let (_temp, disk) = test_disk().await; + let task_id = Uuid::new_v4().to_string(); + let mut state = ResumeState::new(task_id, "set".into(), "disk".into(), vec![]); + state.last_update = 1; + write_state(&disk, &state).await; + assert_eq!(ResumeGc::default().inspect_disk(&disk).await.unwrap().active_skipped, 1); + } + + #[tokio::test] + async fn production_gc_disables_on_mixed_version_capability() { + assert!(claim_generation_matches(None, None)); + assert!(!claim_generation_matches(Some("new"), Some("old"))); + // No #1927 capability means this implementation has no delete path. + assert!(ResumeGcConfig::default().max_entries > 0); + let (_temp, disk) = test_disk().await; + let report = ResumeGc::default().inspect_disk(&disk).await.expect("inspect"); + assert!(report.destructive_disabled); + } + + #[tokio::test] + async fn production_gc_future_schema_is_not_mtime_deleted() { + let (_temp, disk) = test_disk().await; + let task_id = Uuid::new_v4().to_string(); + let mut state = ResumeState::new(task_id.clone(), "set".into(), "disk".into(), vec![]); + state.schema_version = super::super::CURRENT_RESUME_SCHEMA + 1; + write_state(&disk, &state).await; + let report = ResumeGc::default().inspect_disk(&disk).await.unwrap(); + assert_eq!(report.quarantine_required, 1); + assert!( + disk.read_all(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}")) + .await + .is_ok() + ); + } + + #[tokio::test] + async fn production_gc_quarantine_cleanup_is_bounded() { + let (_temp, disk) = test_disk().await; + for _ in 0..4 { + let task_id = Uuid::new_v4().to_string(); + let path = format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_STATE_FILE}"); + disk.write_all(RUSTFS_META_BUCKET, &path, b"corrupt".to_vec().into()) + .await + .expect("corrupt state"); + } + let report = ResumeGc::with_config(ResumeGcConfig { + max_entries: 2, + max_bytes: 1024, + }) + .inspect_disk(&disk) + .await + .expect("inspect"); + assert!(report.quarantine_required <= 2); + assert!(report.budget_exhausted); + } +} From 9cda615519c0be2936882d2e1260492b7ecfec17 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 16:46:36 +0800 Subject: [PATCH 16/41] fix(scanner): discover sub-quorum heal candidates (#6384) * fix(scanner): preserve unversioned heal retries * fix(scanner): bound orphan heal discovery fallback * fix(filemeta): fence unsafe heal key components * fix(scanner): preserve exact overflow heal versions --------- Co-authored-by: houseme --- crates/filemeta/src/metacache.rs | 633 +++++++++++++++++- crates/scanner/src/scanner_folder.rs | 181 +++-- .../src/scanner_folder/item_actions.rs | 18 + crates/scanner/src/scanner_folder/ledger.rs | 46 +- crates/scanner/src/scanner_folder/tests.rs | 112 +++- 5 files changed, 910 insertions(+), 80 deletions(-) diff --git a/crates/filemeta/src/metacache.rs b/crates/filemeta/src/metacache.rs index 462ce6fae..427be0027 100644 --- a/crates/filemeta/src/metacache.rs +++ b/crates/filemeta/src/metacache.rs @@ -21,6 +21,7 @@ use arc_swap::ArcSwapOption; use rmp::Marker; use serde::{Deserialize, Serialize}; use std::cmp::Ordering; +use std::collections::{HashMap, HashSet}; use std::str::from_utf8; use std::{ fmt::Debug, @@ -37,8 +38,13 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::spawn; use tokio::sync::Mutex; use tracing::{debug, warn}; +use uuid::Uuid; const SLASH_SEPARATOR: &str = "/"; +pub const MAX_META_CACHE_HEAL_CANDIDATES: usize = 1024; +/// Keep truncation continuations bounded while still giving the scanner a +/// safe object-level retry for versions that did not fit in the candidate set. +pub const MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS: usize = 64; #[derive(Clone, Debug, Default)] pub struct MetadataResolutionParams { @@ -66,6 +72,50 @@ pub struct MetaCacheEntry { pub reusable: bool, } +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum MetaCacheHealCandidateKind { + Object, + DeleteMarker, + UnversionedObject, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct MetaCacheHealCandidate { + pub object: String, + pub version_id: Option, + pub kind: MetaCacheHealCandidateKind, + /// Number of raw disk entries that carried this validated version. + pub replica_count: usize, +} + +impl MetaCacheHealCandidate { + pub fn validated_version(&self) -> Option { + match self.kind { + MetaCacheHealCandidateKind::Object | MetaCacheHealCandidateKind::DeleteMarker => self.version_id, + MetaCacheHealCandidateKind::UnversionedObject => None, + } + } + + pub fn is_unversioned(&self) -> bool { + self.kind == MetaCacheHealCandidateKind::UnversionedObject + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MetaCacheHealDiscovery { + pub candidates: Vec, + pub unverified_count: usize, + pub truncated: bool, + /// Object names whose validated version set exceeded the candidate cap. + /// The scanner retries these names without a version and with destructive + /// healing disabled; this is an explicit bounded continuation, not a + /// version claim. + pub truncated_objects: Vec, + /// Validated candidates beyond the main cap, retained with exact version + /// identities so callers never fall back to a latest-version request. + pub truncated_candidates: Vec, +} + impl MetaCacheEntry { pub fn marshal_msg(&self) -> Result> { let mut wr = Vec::new(); @@ -370,6 +420,185 @@ impl MetaCacheEntries { }) } + /// Discover validated object/delete-marker versions and safe unversioned + /// inspection candidates in the raw entries without applying read quorum. + /// This is intentionally separate from [`Self::resolve`]: a sub-quorum + /// version is a valid heal target even though it must not participate in + /// normal reads or writes. + /// + /// The validated list is bounded and deduplicated by object, version id, + /// and metadata kind; each candidate retains the number of raw disk + /// entries that carried it so callers can classify sub-quorum versions. + /// Entries whose xl.meta cannot be decoded are counted separately for + /// discovery accounting; they never become versionless destructive heal + /// requests and do not consume the validated quota. An + /// [`MetaCacheHealCandidateKind::UnversionedObject`] is always consumed by + /// a non-destructive scanner request. + pub fn discover_heal_candidates(&self, bucket: &str, max_candidates: usize) -> MetaCacheHealDiscovery { + let limit = max_candidates.min(MAX_META_CACHE_HEAL_CANDIDATES); + if limit == 0 || bucket.is_empty() { + return MetaCacheHealDiscovery::default(); + } + + let mut discovery = MetaCacheHealDiscovery { + candidates: Vec::::with_capacity(limit.min(self.0.len())), + unverified_count: 0, + truncated: false, + truncated_objects: Vec::with_capacity(MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS.min(limit)), + truncated_candidates: Vec::new(), + }; + let mut seen: HashMap<(String, Option, MetaCacheHealCandidateKind), usize> = + HashMap::with_capacity(limit.min(self.0.len())); + + for entry in self.0.iter().flatten() { + if !valid_heal_candidate_name(bucket, entry) { + continue; + } + + let meta = match FileMeta::load(&entry.metadata) { + Ok(meta) => meta, + Err(_) => { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + } + }; + let mut entry_seen = HashSet::new(); + + for shallow in meta.versions { + let version = match shallow.parse_version_meta() { + Ok(version) if version.valid() => version, + Ok(_) | Err(_) => { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + } + }; + if version.free_version() { + continue; + } + + let payload_header = version.header(); + if normalize_version_id(shallow.header.version_id) != normalize_version_id(payload_header.version_id) + || shallow.header.version_type != payload_header.version_type + { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + } + + let (kind, version_id) = match version.version_type { + VersionType::Object + if version.object.is_some() && version.delete_marker.is_none() && version.legacy_object.is_none() => + { + match version.object.as_ref().and_then(|object| object.version_id) { + Some(id) if !id.is_nil() => (MetaCacheHealCandidateKind::Object, Some(id)), + Some(_) | None => (MetaCacheHealCandidateKind::UnversionedObject, None), + } + } + VersionType::Delete + if version.delete_marker.is_some() && version.object.is_none() && version.legacy_object.is_none() => + { + let Some(id) = version.delete_marker.as_ref().and_then(|marker| marker.version_id) else { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + }; + if id.is_nil() { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + } + (MetaCacheHealCandidateKind::DeleteMarker, Some(id)) + } + VersionType::Legacy + if version.legacy_object.is_some() && version.object.is_none() && version.delete_marker.is_none() => + { + let Some(legacy) = version.legacy_object.as_ref() else { + continue; + }; + if legacy.version_id.is_empty() { + (MetaCacheHealCandidateKind::UnversionedObject, None) + } else { + let Ok(id) = Uuid::parse_str(&legacy.version_id) else { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + }; + if id.is_nil() { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + } + (MetaCacheHealCandidateKind::Object, Some(id)) + } + } + _ => { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + } + }; + + if normalize_version_id(payload_header.version_id) != version_id { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + } + + // `all_parts=true` is the trust-boundary check for versioned + // candidates. A null/legacy object may still need the old + // non-destructive inspection fallback when its part arrays + // are parseable but incomplete; never use that fallback for + // a candidate carrying a real version id. + let file_info = match version.clone().into_fileinfo(bucket, &entry.name, true) { + Ok(file_info) => file_info, + Err(_) if version_id.is_none() && matches!(kind, MetaCacheHealCandidateKind::UnversionedObject) => { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + match version.into_fileinfo(bucket, &entry.name, false) { + Ok(file_info) => file_info, + Err(_) => continue, + } + } + Err(_) => { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + } + }; + if file_info.volume != bucket || file_info.name != entry.name { + discovery.unverified_count = discovery.unverified_count.saturating_add(1); + continue; + } + + let candidate = MetaCacheHealCandidate { + object: entry.name.clone(), + version_id, + kind, + replica_count: 1, + }; + let key = (candidate.object.clone(), candidate.version_id, candidate.kind.clone()); + if entry_seen.contains(&key) { + continue; + } + if let Some(index) = seen.get(&key).copied() { + entry_seen.insert(key); + discovery.candidates[index].replica_count = discovery.candidates[index].replica_count.saturating_add(1); + } else if discovery.candidates.len() >= limit { + // Keep the validated candidate list bounded, but retain a + // bounded object-level continuation so the scanner cannot + // silently lose every version of a busy object. + discovery.truncated = true; + if discovery.truncated_objects.len() < MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS + && !discovery.truncated_objects.iter().any(|object| object == &candidate.object) + { + discovery.truncated_objects.push(candidate.object.clone()); + } + if discovery.truncated_objects.iter().any(|object| object == &candidate.object) { + discovery.truncated_candidates.push(candidate); + } + continue; + } else { + entry_seen.insert(key.clone()); + seen.insert(key, discovery.candidates.len()); + discovery.candidates.push(candidate); + } + } + } + + discovery + } + fn resolve_inner(&self, mut params: MetadataResolutionParams, enforce_write_quorum: bool) -> Option { if self.0.is_empty() { debug!( @@ -546,6 +775,33 @@ impl MetaCacheEntries { } } +fn valid_heal_candidate_name(bucket: &str, entry: &MetaCacheEntry) -> bool { + if bucket.is_empty() + || entry.name.is_empty() + || entry.is_dir() + || (cfg!(windows) && entry.name.contains('\\')) + || entry.name.chars().any(char::is_control) + { + return false; + } + + // Validate raw key components without normalizing them. The scanner maps + // accepted keys to filesystem paths later, so dot components and empty + // internal components must be rejected before that boundary. A final + // empty component is retained for valid keys ending in '/'. + let mut components = entry.name.split('/').peekable(); + while let Some(component) = components.next() { + if component == "." || component == ".." || (component.is_empty() && components.peek().is_some()) { + return false; + } + } + true +} + +fn normalize_version_id(version_id: Option) -> Option { + version_id.filter(|id| !id.is_nil()) +} + #[derive(Debug, Default)] pub struct MetaCacheEntriesSortedResult { pub entries: Option, @@ -991,7 +1247,7 @@ impl Cache { mod tests { use super::*; use crate::test_data::create_real_xlmeta; - use crate::{FileMetaVersion, MetaDeleteMarker, TRANSITION_COMPLETE}; + use crate::{FileMetaVersion, MetaDeleteMarker, MetaObjectV1, MetaObjectV1Erasure, MetaObjectV1Stat, TRANSITION_COMPLETE}; use std::collections::HashMap; use std::io::Cursor; use std::sync::{ @@ -1592,6 +1848,381 @@ mod tests { ); } + #[test] + fn discover_heal_candidates_keeps_sub_quorum_versions_and_deduplicates() { + let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); + let entries = MetaCacheEntries(vec![ + Some(metacache_entry_single_version(1, now, "one")), + Some(metacache_entry_single_version(2, now, "two")), + Some(metacache_entry_single_version(2, now, "two")), + Some(metacache_entry_single_version(3, now, "three")), + ]); + + let discovery = entries.discover_heal_candidates("bucket", 16); + let ids: std::collections::HashSet = discovery + .candidates + .iter() + .filter_map(|candidate| candidate.version_id) + .collect(); + assert_eq!( + ids, + [Uuid::from_u128(1), Uuid::from_u128(2), Uuid::from_u128(3)] + .into_iter() + .collect() + ); + assert_eq!(discovery.candidates.len(), 3, "duplicate tied versions must be emitted once"); + assert_eq!( + discovery + .candidates + .iter() + .find(|candidate| candidate.version_id == Some(Uuid::from_u128(2))) + .expect("duplicate version should be discovered") + .replica_count, + 2 + ); + } + + #[test] + fn discover_heal_candidates_does_not_count_duplicate_versions_within_one_entry() { + let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); + let mut meta = FileMeta::load(&metacache_entry_single_version(1, now, "duplicate").metadata) + .expect("duplicate fixture should decode"); + meta.versions.push(meta.versions[0].clone()); + let entry = MetaCacheEntry { + name: "object".to_string(), + metadata: meta.marshal_msg().expect("duplicate metadata should marshal"), + cached: Some(meta), + reusable: false, + }; + + let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 16); + let candidate = discovery + .candidates + .iter() + .find(|candidate| candidate.version_id == Some(Uuid::from_u128(1))) + .expect("duplicate fixture should be discovered"); + assert_eq!(candidate.replica_count, 1); + } + + #[test] + fn discover_heal_candidates_covers_divergent_quorum_boundaries_n2_n4_n6() { + let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); + + for (disk_count, quorum) in [(2usize, 1usize), (4, 2), (6, 3)] { + let target_id = Uuid::from_u128(0x1000 + disk_count as u128); + for target_replicas in [quorum.saturating_sub(1), quorum, quorum + 1] { + let entries = (0..disk_count) + .map(|disk| { + let version_id = if disk < target_replicas { + target_id + } else { + Uuid::from_u128(0x2000 + disk as u128) + }; + Some(metacache_entry_single_version(version_id.as_u128(), now, "divergent")) + }) + .collect(); + let discovery = MetaCacheEntries(entries).discover_heal_candidates("bucket", 32); + let target = discovery + .candidates + .iter() + .find(|candidate| candidate.version_id == Some(target_id)); + assert_eq!(target.is_some(), target_replicas > 0, "N={disk_count}, replicas={target_replicas}"); + if let Some(target) = target { + assert_eq!(target.replica_count, target_replicas); + } + } + } + } + + #[test] + fn discover_heal_candidates_separates_delete_markers_and_preserves_unversioned_objects() { + let mut marker_meta = FileMeta::new(); + marker_meta + .add_version(FileInfo { + volume: "bucket".to_string(), + name: "object".to_string(), + version_id: Some(Uuid::from_u128(99)), + deleted: true, + mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")), + ..Default::default() + }) + .expect("delete marker should be added"); + let marker = MetaCacheEntry { + name: "object".to_string(), + metadata: marker_meta.marshal_msg().expect("delete marker metadata should marshal"), + cached: Some(marker_meta), + reusable: false, + }; + + let unversioned_entry = metacache_entry_with_mod_time( + OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"), + "unversioned", + ); + let discovery = MetaCacheEntries(vec![Some(marker), Some(unversioned_entry)]).discover_heal_candidates("bucket", 16); + assert!(discovery.candidates.iter().any(|candidate| { + candidate.kind == MetaCacheHealCandidateKind::DeleteMarker && candidate.version_id == Some(Uuid::from_u128(99)) + })); + assert!(discovery.candidates.iter().any(|candidate| { + candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none() + })); + } + + #[test] + fn discover_heal_candidates_rejects_delete_markers_without_ids() { + let mut marker_meta = FileMeta::new(); + marker_meta + .add_version(FileInfo { + volume: "bucket".to_string(), + name: "object".to_string(), + deleted: true, + mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")), + ..Default::default() + }) + .expect("nil delete marker should be added"); + let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry { + name: "object".to_string(), + metadata: marker_meta.marshal_msg().expect("nil marker metadata should marshal"), + cached: Some(marker_meta), + reusable: false, + })]) + .discover_heal_candidates("bucket", 16); + + assert!( + !discovery + .candidates + .iter() + .any(|candidate| candidate.kind == MetaCacheHealCandidateKind::DeleteMarker) + ); + assert!(discovery.unverified_count >= 1); + } + + #[test] + fn discover_heal_candidates_skips_free_versions() { + let object_id = Uuid::from_u128(100); + let free_id = Uuid::from_u128(101); + let mut meta = FileMeta::new(); + meta.add_version(FileInfo { + volume: "bucket".to_string(), + name: "object".to_string(), + version_id: Some(object_id), + transition_status: TRANSITION_COMPLETE.to_string(), + transitioned_objname: "remote/object".to_string(), + transition_version_id: Some(Uuid::from_u128(102)), + transition_tier: "WARM".to_string(), + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }) + .expect("transitioned object should be added"); + let mut delete = FileInfo { + volume: "bucket".to_string(), + name: "object".to_string(), + version_id: Some(object_id), + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }; + delete.set_tier_free_version_id(&free_id.to_string()); + meta.delete_version(&delete).expect("free version should be persisted"); + + let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry { + name: "object".to_string(), + metadata: meta.marshal_msg().expect("free version metadata should marshal"), + cached: Some(meta), + reusable: false, + })]) + .discover_heal_candidates("bucket", 16); + assert!(discovery.candidates.is_empty()); + } + + #[test] + fn discover_heal_candidates_preserves_unversioned_legacy_object() { + let legacy = MetaObjectV1 { + version: "1.0.1".to_string(), + format: "xl".to_string(), + stat: MetaObjectV1Stat { + size: 1, + mod_time: Some(OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp")), + name: "object".to_string(), + ..Default::default() + }, + erasure: MetaObjectV1Erasure { + data_blocks: 4, + parity_blocks: 2, + index: 1, + distribution: vec![1, 2, 3, 4, 5, 6], + ..Default::default() + }, + ..Default::default() + }; + let version = FileMetaVersion { + version_type: VersionType::Legacy, + legacy_object: Some(legacy), + ..Default::default() + }; + let mut meta = FileMeta::new(); + meta.versions + .push(FileMetaShallowVersion::try_from(version).expect("legacy metadata should marshal")); + let discovery = MetaCacheEntries(vec![Some(MetaCacheEntry { + name: "object".to_string(), + metadata: meta.marshal_msg().expect("legacy metadata should marshal"), + cached: Some(meta), + reusable: false, + })]) + .discover_heal_candidates("bucket", 16); + assert!(discovery.candidates.iter().any(|candidate| { + candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none() + })); + } + + #[test] + fn discover_heal_candidates_rejects_nil_and_malformed_metadata_and_is_bounded() { + let now = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); + let mut nil = metacache_entry_single_version(1, now, "nil"); + let mut nil_meta = FileMeta::load(&nil.metadata).expect("nil fixture should decode"); + let mut nil_version = nil_meta.versions[0] + .parse_version_meta() + .expect("nil fixture version should decode"); + nil_version.object.as_mut().expect("object fixture").version_id = Some(Uuid::nil()); + nil_meta.versions[0] = FileMetaShallowVersion::try_from(nil_version).expect("nil fixture should marshal"); + nil.metadata = nil_meta.marshal_msg().expect("nil fixture metadata should marshal"); + + let mut mismatched = metacache_entry_single_version(2, now, "mismatched"); + let mut mismatched_meta = FileMeta::load(&mismatched.metadata).expect("mismatched fixture should decode"); + mismatched_meta.versions[0].header.version_id = Some(Uuid::from_u128(200)); + mismatched.metadata = mismatched_meta.marshal_msg().expect("mismatched metadata should marshal"); + + let mut short_parts = metacache_entry_single_version(3, now, "short-parts"); + let mut short_parts_meta = FileMeta::load(&short_parts.metadata).expect("short-parts fixture should decode"); + let mut short_parts_version = short_parts_meta.versions[0] + .parse_version_meta() + .expect("short-parts fixture version should decode"); + let object = short_parts_version.object.as_mut().expect("object fixture"); + object.part_numbers = vec![1]; + object.part_actual_sizes = vec![1]; + object.part_sizes.clear(); + short_parts_meta.versions[0] = FileMetaShallowVersion::try_from(short_parts_version).expect("short-parts should marshal"); + short_parts.metadata = short_parts_meta.marshal_msg().expect("short-parts metadata should marshal"); + + let mut short_unversioned = metacache_entry_with_mod_time(now, "short-unversioned"); + let mut short_unversioned_meta = + FileMeta::load(&short_unversioned.metadata).expect("short-unversioned fixture should decode"); + let mut short_unversioned_version = short_unversioned_meta.versions[0] + .parse_version_meta() + .expect("short-unversioned version should decode"); + let unversioned_object = short_unversioned_version.object.as_mut().expect("unversioned object fixture"); + unversioned_object.part_numbers = vec![1]; + unversioned_object.part_actual_sizes = vec![1]; + unversioned_object.part_sizes.clear(); + short_unversioned_meta.versions[0] = + FileMetaShallowVersion::try_from(short_unversioned_version).expect("short-unversioned should marshal"); + short_unversioned.metadata = short_unversioned_meta + .marshal_msg() + .expect("short-unversioned metadata should marshal"); + + let mut malformed = nil.clone(); + malformed.name = "malformed".to_string(); + malformed.metadata = vec![1, 2, 3]; + + let entries = MetaCacheEntries( + std::iter::once(Some(nil)) + .chain(std::iter::once(Some(mismatched))) + .chain(std::iter::once(Some(short_parts))) + .chain(std::iter::once(Some(short_unversioned))) + .chain(std::iter::once(Some(malformed))) + .chain((0..32).map(|id| Some(metacache_entry_single_version(id + 10, now, "bounded")))) + .collect(), + ); + let discovery = entries.discover_heal_candidates("bucket", 5); + assert!(discovery.candidates.len() <= 5); + assert!(discovery.truncated, "bounded discovery must expose dropped candidates"); + assert!( + discovery + .truncated_candidates + .iter() + .all(|candidate| candidate.version_id.is_some()), + "overflow candidates must retain exact version identities" + ); + assert!( + discovery.truncated_objects.iter().any(|object| object == "object"), + "bounded discovery must expose an object-level safe continuation" + ); + assert!( + !discovery + .candidates + .iter() + .any(|candidate| candidate.version_id == Some(Uuid::nil())) + ); + assert!( + !discovery + .candidates + .iter() + .any(|candidate| candidate.version_id == Some(Uuid::from_u128(2))) + ); + assert!( + !discovery + .candidates + .iter() + .any(|candidate| candidate.version_id == Some(Uuid::from_u128(3))) + ); + assert!(discovery.candidates.iter().any(|candidate| { + candidate.kind == MetaCacheHealCandidateKind::UnversionedObject && candidate.version_id.is_none() + })); + assert!( + discovery.unverified_count >= 1, + "malformed and rejected metadata must remain observable during discovery" + ); + + for invalid_name in [ + "../object", + "./object", + "object/../other", + "object//name", + "object\u{0001}name", + "object\0name", + ] { + let mut entry = metacache_entry_single_version(400, now, invalid_name); + entry.name = invalid_name.to_string(); + let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 5); + assert!( + discovery.candidates.is_empty(), + "invalid key should not become a heal candidate: {invalid_name:?}" + ); + } + + #[cfg(windows)] + { + let mut entry = metacache_entry_single_version(400, now, "object\\name"); + entry.name = "object\\name".to_string(); + assert!( + MetaCacheEntries(vec![Some(entry)]) + .discover_heal_candidates("bucket", 5) + .candidates + .is_empty(), + "backslash is a path separator on Windows" + ); + } + + #[cfg(not(windows))] + { + let mut entry = metacache_entry_single_version(400, now, "object\\name"); + entry.name = "object\\name".to_string(); + assert_eq!( + MetaCacheEntries(vec![Some(entry)]) + .discover_heal_candidates("bucket", 5) + .candidates + .len(), + 1, + "backslash is object-key data on Unix" + ); + } + + for valid_name in ["trailing/", "prefix/object"] { + let mut entry = metacache_entry_single_version(401, now, valid_name); + entry.name = valid_name.to_string(); + let discovery = MetaCacheEntries(vec![Some(entry)]).discover_heal_candidates("bucket", 5); + assert_eq!(discovery.candidates.len(), 1, "raw S3 key should remain opaque: {valid_name:?}"); + } + } + #[test] fn resolve_rejects_partial_latest_and_returns_committed_previous_metadata() { let old_mod_time = OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp"); diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 81a2a7377..148249ea8 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -43,7 +43,10 @@ use rustfs_common::metrics::{ UpdateCurrentPathFn, current_path_updater, global_metrics, }; use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count}; -use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; +use rustfs_filemeta::{ + MAX_META_CACHE_HEAL_CANDIDATES, MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS, MetaCacheEntries, MetaCacheEntry, + MetaCacheHealCandidateKind, +}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, VersioningConfiguration}; use time::OffsetDateTime; @@ -96,6 +99,11 @@ const METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL: &str = "rustfs_scanner_ex const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders_total"; const METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL: &str = "rustfs_scanner_pending_heal_prune_total"; const METRIC_SCANNER_PENDING_HEAL_MALFORMED_TOTAL: &str = "rustfs_scanner_pending_heal_malformed_total"; +const METRIC_SCANNER_HEAL_DISCOVERY_CANDIDATES_TOTAL: &str = "rustfs_scanner_heal_discovery_candidates_total"; +const METRIC_SCANNER_HEAL_DISCOVERY_SUB_QUORUM_TOTAL: &str = "rustfs_scanner_heal_discovery_sub_quorum_total"; +const METRIC_SCANNER_HEAL_DISCOVERY_UNVERIFIED_TOTAL: &str = "rustfs_scanner_heal_discovery_unverified_total"; +const METRIC_SCANNER_HEAL_DISCOVERY_QUEUED_TOTAL: &str = "rustfs_scanner_heal_discovery_queued_total"; +const METRIC_SCANNER_HEAL_DISCOVERY_TRUNCATED_TOTAL: &str = "rustfs_scanner_heal_discovery_truncated_total"; const MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET: usize = 128; // --- scanner excess alerts as S3 notification events (rustfs/backlog#1868) -- @@ -883,7 +891,7 @@ impl FolderScanner { object: Option, version_id: Option, request: HealChannelRequest, - ) -> Result<(), ScannerError> { + ) -> Result { let candidate_type = pending_scanner_heal_candidate_type(kind); let priority = request.priority; let scan_mode = request.scan_mode.unwrap_or(self.scan_mode); @@ -911,7 +919,7 @@ impl FolderScanner { error = %err, "Scanner deferred heal request after channel error" ); - return Ok(()); + return Ok(HealAdmissionResult::Full); } }; self.update_pending_scanner_heal_after_admission( @@ -923,7 +931,7 @@ impl FolderScanner { result, ); if result.is_admitted() { - return Ok(()); + return Ok(result); } record_high_priority_heal_escalation(candidate_type, priority, result); @@ -944,7 +952,7 @@ impl FolderScanner { state = "high_priority_not_admitted", "Scanner high-priority heal admission failed" ); - Ok(()) + Ok(result) } pub fn set_heal_object_select(&mut self, prob: u32) { @@ -1737,14 +1745,7 @@ impl FolderScanner { break; } - let mut resolver = MetadataResolutionParams { - dir_quorum: self.disks_quorum, - obj_quorum: self.disks_quorum, - bucket: "".to_string(), - strict: false, - ..Default::default() - }; - + let mut previous_bucket = String::new(); for name in abandoned_children { if !self.should_heal().await { break; @@ -1752,7 +1753,7 @@ impl FolderScanner { let (bucket, prefix) = path2_bucket_object(name.as_str()); - if bucket != resolver.bucket { + if bucket != previous_bucket { self.send_required_scanner_heal_request( PendingScannerHealKind::Bucket, bucket.clone(), @@ -1761,10 +1762,9 @@ impl FolderScanner { build_bucket_heal_request(bucket.clone(), HealChannelPriority::High), ) .await?; + previous_bucket = bucket.clone(); } - resolver.bucket = bucket.clone(); - let child_ctx = ctx.child_token(); let (agreed_tx, mut agreed_rx) = mpsc::channel::(1); @@ -1881,6 +1881,8 @@ impl FolderScanner { let mut agreed_closed = false; let mut partial_closed = false; let mut finished_closed = false; + let mut seen_heal_candidates: HashSet<(String, Option, MetaCacheHealCandidateKind)> = HashSet::new(); + let mut seen_truncated_objects: HashSet = HashSet::new(); loop { if agreed_closed && partial_closed && finished_closed { @@ -1905,65 +1907,112 @@ impl FolderScanner { break; } - let Some(entry) = resolve_object_heal_entry(&entries, resolver.clone()) else { - continue; - }; - - (self.update_current_path)(&entry.name).await; - - if entry.is_dir() { - continue; + let discovery = entries.discover_heal_candidates(&bucket, MAX_META_CACHE_HEAL_CANDIDATES); + counter!(METRIC_SCANNER_HEAL_DISCOVERY_CANDIDATES_TOTAL) + .increment(u64::try_from(discovery.candidates.len()).unwrap_or(u64::MAX)); + counter!(METRIC_SCANNER_HEAL_DISCOVERY_SUB_QUORUM_TOTAL).increment( + u64::try_from( + discovery + .candidates + .iter() + .filter(|candidate| candidate.replica_count < disks_quorum) + .count(), + ) + .unwrap_or(u64::MAX), + ); + counter!(METRIC_SCANNER_HEAL_DISCOVERY_UNVERIFIED_TOTAL).increment( + u64::try_from(discovery.unverified_count).unwrap_or(u64::MAX), + ); + if discovery.truncated { + counter!(METRIC_SCANNER_HEAL_DISCOVERY_TRUNCATED_TOTAL).increment(1); } - let fivs = match entry.file_info_versions(&bucket) { - Ok(fivs) => fivs, - Err(e) => { - error!( - target: "rustfs::scanner::folder", - event = EVENT_SCANNER_FOLDER_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_FOLDER, - bucket = %bucket, - entry = %entry.name, - state = "file_info_versions_failed", - error = %e, - "Scanner list_path_raw failed to resolve file versions" - ); - self.send_required_scanner_heal_request( - PendingScannerHealKind::Object, - bucket.clone(), - Some(entry.name.clone()), - None, - build_object_heal_request( - bucket.clone(), - entry.name.clone(), - None, - self.scan_mode, - HealChannelPriority::High, - ), - ) - .await?; - found_objects = true; + for candidate in discovery.candidates { + let sub_quorum_candidate = candidate.replica_count < disks_quorum; + let version_id = candidate.validated_version().map(|id| id.to_string()); + let identity = (candidate.object.clone(), version_id.clone(), candidate.kind.clone()); + if seen_heal_candidates.len() >= MAX_META_CACHE_HEAL_CANDIDATES + && !seen_heal_candidates.contains(&identity) + { continue; } - }; - - for fiv in fivs.versions { - let version_id = fiv.version_id.and_then(|v| if v.is_nil() { None } else { Some(v.to_string()) }); - self.send_required_scanner_heal_request( - PendingScannerHealKind::Object, - bucket.clone(), - Some(entry.name.clone()), - version_id.clone(), - build_object_heal_request( + if !seen_heal_candidates.insert(identity) { + continue; + } + let request = if candidate.is_unversioned() { + build_non_destructive_object_heal_request( bucket.clone(), - entry.name.clone(), - version_id, + candidate.object.clone(), self.scan_mode, HealChannelPriority::High, - ), + ) + } else { + build_object_heal_request( + bucket.clone(), + candidate.object.clone(), + version_id.clone(), + self.scan_mode, + HealChannelPriority::High, + ) + }; + (self.update_current_path)(&candidate.object).await; + let admission = self.send_required_scanner_heal_request( + PendingScannerHealKind::Object, + bucket.clone(), + Some(candidate.object.clone()), + version_id.clone(), + request, ) .await?; + if admission.is_admitted() { + counter!(METRIC_SCANNER_HEAL_DISCOVERY_QUEUED_TOTAL).increment(1); + } else if sub_quorum_candidate { + self.mark_pending_scanner_heal_reason( + PendingScannerHealKind::Object, + &bucket, + Some(&candidate.object), + version_id.as_deref(), + "sub_quorum_metadata", + ); + } + found_objects = true; + } + + // Candidates beyond the main cap remain exact + // version requests; never downgrade them to a + // latest-version (version_id=None) heal. + for candidate in discovery.truncated_candidates { + let version_id = candidate.validated_version().map(|id| id.to_string()); + let identity = (candidate.object.clone(), version_id.clone(), candidate.kind.clone()); + if seen_truncated_objects.len() >= MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS + && !seen_truncated_objects.contains(&candidate.object) + { + continue; + } + seen_truncated_objects.insert(candidate.object.clone()); + if !seen_heal_candidates.insert(identity) { + continue; + } + let request = build_object_heal_request( + bucket.clone(), + candidate.object.clone(), + version_id.clone(), + self.scan_mode, + HealChannelPriority::High, + ); + (self.update_current_path)(&candidate.object).await; + let admission = self + .send_required_scanner_heal_request( + PendingScannerHealKind::Object, + bucket.clone(), + Some(candidate.object.clone()), + version_id, + request, + ) + .await?; + if admission.is_admitted() { + counter!(METRIC_SCANNER_HEAL_DISCOVERY_QUEUED_TOTAL).increment(1); + } found_objects = true; } diff --git a/crates/scanner/src/scanner_folder/item_actions.rs b/crates/scanner/src/scanner_folder/item_actions.rs index 0f2d82641..09a97e42b 100644 --- a/crates/scanner/src/scanner_folder/item_actions.rs +++ b/crates/scanner/src/scanner_folder/item_actions.rs @@ -13,6 +13,8 @@ // limitations under the License. /// Per-object scan actions: ScannerItem, the get-size failure policy, and the heal/ILM admission helpers. use super::*; +#[cfg(test)] +use rustfs_filemeta::MetadataResolutionParams; /// Cached folder information for scanning #[derive(Clone, Debug)] @@ -89,6 +91,22 @@ pub(super) fn build_object_heal_request( } } +/// Build the versionless inspection request used when discovery cannot prove +/// a destructive version identity (for example an unversioned object or a +/// bounded candidate overflow). The explicit flag is the fail-closed safety +/// boundary; callers must not reconstruct it with the destructive default. +pub(super) fn build_non_destructive_object_heal_request( + bucket: String, + object: String, + scan_mode: HealScanMode, + priority: HealChannelPriority, +) -> HealChannelRequest { + let mut request = build_object_heal_request(bucket, object, None, scan_mode, priority); + request.remove_corrupted = Some(false); + request +} + +#[cfg(test)] pub(super) fn resolve_object_heal_entry( entries: &MetaCacheEntries, resolver: MetadataResolutionParams, diff --git a/crates/scanner/src/scanner_folder/ledger.rs b/crates/scanner/src/scanner_folder/ledger.rs index 7ac03983c..167a1425d 100644 --- a/crates/scanner/src/scanner_folder/ledger.rs +++ b/crates/scanner/src/scanner_folder/ledger.rs @@ -105,6 +105,29 @@ impl FolderScanner { } } + /// Preserve the discovery reason when a candidate could not be admitted + /// immediately. The existing string field is intentionally reused so the + /// scanner's map-encoded cache schema stays backward compatible. + pub(super) fn mark_pending_scanner_heal_reason( + &mut self, + kind: PendingScannerHealKind, + bucket: &str, + object: Option<&str>, + version_id: Option<&str>, + reason: &str, + ) { + if let Some(entry) = self + .new_cache + .info + .pending_heals + .iter_mut() + .find(|entry| pending_scanner_heal_matches(entry, kind, bucket, object, version_id)) + { + entry.last_admission_reason = reason.to_string(); + self.sync_pending_heals(); + } + } + pub(super) fn prune_pending_scanner_heals(&mut self) { let now = Self::now_secs(); let before_expiry = self.new_cache.info.pending_heals.len(); @@ -305,13 +328,22 @@ pub(super) fn build_pending_scanner_heal_request(entry: &PendingScannerHeal) -> match entry.kind { PendingScannerHealKind::Bucket => Some(build_bucket_heal_request(entry.bucket.clone(), HealChannelPriority::High)), PendingScannerHealKind::Object => entry.object.as_ref().map(|object| { - build_object_heal_request( - entry.bucket.clone(), - object.clone(), - entry.version_id.clone(), - entry.scan_mode, - HealChannelPriority::High, - ) + if entry.version_id.is_none() { + build_non_destructive_object_heal_request( + entry.bucket.clone(), + object.clone(), + entry.scan_mode, + HealChannelPriority::High, + ) + } else { + build_object_heal_request( + entry.bucket.clone(), + object.clone(), + entry.version_id.clone(), + entry.scan_mode, + HealChannelPriority::High, + ) + } }), } } diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index 3b01b9be6..56d1f1ddc 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -17,7 +17,7 @@ use crate::SCANNER_SLEEPER; use super::*; use crate::storage_api::VersionPurgeStatusType; use crate::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, TierStats, new_disk, storageclass}; -use rustfs_filemeta::{FileInfo, FileMeta}; +use rustfs_filemeta::{FileInfo, FileMeta, MetadataResolutionParams}; use std::io::Write; #[cfg(unix)] use std::os::unix::fs::{PermissionsExt, symlink}; @@ -994,6 +994,21 @@ fn test_build_object_heal_request_omits_nil_version_id() { assert_eq!(request.recreate_missing, Some(false)); } +#[test] +fn test_build_non_destructive_object_heal_request_disables_removal() { + let request = build_non_destructive_object_heal_request( + "bucket".to_string(), + "path/to/object".to_string(), + HealScanMode::Deep, + HealChannelPriority::High, + ); + + assert_eq!(request.object_version_id, None); + assert_eq!(request.remove_corrupted, Some(false)); + assert_eq!(request.recreate_missing, Some(false)); + assert_eq!(request.source, HealRequestSource::Scanner); +} + #[test] fn test_build_bucket_heal_request_disables_recreate_for_scanner() { let request = build_bucket_heal_request("bucket".to_string(), HealChannelPriority::Low); @@ -1133,6 +1148,42 @@ fn test_pending_heal_reconstructs_object_request_with_version() { assert_eq!(request.source, HealRequestSource::Scanner); } +#[test] +fn test_pending_heal_reconstructs_unversioned_request_without_removal() { + let pending = pending_heal(PendingScannerHealKind::Object, "bucket", Some("object"), None, 1, 1); + + let request = build_pending_scanner_heal_request(&pending).expect("unversioned object request should rebuild"); + + assert!(request.object_version_id.is_none()); + assert_eq!(request.remove_corrupted, Some(false)); + assert_eq!(request.recreate_missing, Some(false)); +} + +#[tokio::test] +async fn test_pending_heal_reason_preserves_sub_quorum_discovery() { + let (mut scanner, temp_dir) = build_test_scanner().await; + let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir); + + scanner.update_pending_scanner_heal_after_admission( + PendingScannerHealKind::Object, + "bucket", + Some("object"), + Some("version-a"), + HealScanMode::Deep, + HealAdmissionResult::Full, + ); + scanner.mark_pending_scanner_heal_reason( + PendingScannerHealKind::Object, + "bucket", + Some("object"), + Some("version-a"), + "sub_quorum_metadata", + ); + + assert_eq!(scanner.new_cache.info.pending_heals.len(), 1); + assert_eq!(scanner.new_cache.info.pending_heals[0].last_admission_reason, "sub_quorum_metadata"); +} + #[test] fn test_pending_heal_retry_candidates_respect_cap_and_order() { let pending: Vec = (0..(MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET + 2)) @@ -1335,6 +1386,20 @@ fn metadata_for_object(bucket: &str, object: &str) -> Vec { meta.marshal_msg().expect("test metadata should marshal") } +fn metadata_for_object_version(bucket: &str, object: &str, version_id: Option) -> Vec { + let mut file_info = FileInfo::new(object, 4, 2); + file_info.volume = bucket.to_string(); + file_info.name = object.to_string(); + file_info.version_id = version_id; + file_info.versioned = version_id.is_some(); + file_info.mod_time = Some(OffsetDateTime::now_utc()); + file_info.size = 1; + + let mut meta = FileMeta::new(); + meta.add_version(file_info).expect("test metadata version should be accepted"); + meta.marshal_msg().expect("test metadata should marshal") +} + async fn write_test_object_metadata(root: &std::path::Path, bucket: &str, object: &str) { write_test_object_metadata_bytes(root, bucket, object, &metadata_for_object(bucket, object)).await; } @@ -1738,12 +1803,21 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() { let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone()); let heal_starts = Arc::new(AtomicUsize::new(0)); let heal_starts_clone = heal_starts.clone(); + let healed_versions = Arc::new(Mutex::new(Vec::>::new())); + let healed_versions_clone = healed_versions.clone(); let mut heal_rx = rustfs_common::heal_channel::init_heal_channel().expect("heal channel should initialize once for scanner tests"); let _heal_responder = tokio::spawn(async move { while let Some(command) = heal_rx.recv().await { - if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command { + if let rustfs_common::heal_channel::HealChannelCommand::Start { + request, response_tx, .. + } = command + { heal_starts_clone.fetch_add(1, Ordering::Relaxed); + healed_versions_clone + .lock() + .expect("heal version capture lock should not be poisoned") + .push(request.object_version_id); let _ = response_tx.send(Ok(HealAdmissionResult::Accepted)); } } @@ -1751,13 +1825,18 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() { let bucket = "src-archive"; let object = "snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4"; - let metadata = metadata_for_object(bucket, object); - write_test_object_metadata_bytes(&temp_dir, bucket, object, &metadata).await; + let orphan_version = Uuid::from_u128(0x1934); + let shared_version = Uuid::from_u128(0x1935); + let orphan_metadata = metadata_for_object_version(bucket, object, Some(orphan_version)); + let shared_metadata = metadata_for_object_version(bucket, object, Some(shared_version)); + write_test_object_metadata_bytes(&temp_dir, bucket, object, &orphan_metadata).await; + let mut expected_metadata = vec![(temp_dir.join(bucket).join(object).join("xl.meta"), orphan_metadata.clone())]; let mut disks = vec![scanner.local_disk.clone()]; for disk_name in ["disk2", "disk3", "disk4"] { let disk_root = temp_dir.join(disk_name); - write_test_object_metadata_bytes(&disk_root, bucket, object, &metadata).await; + write_test_object_metadata_bytes(&disk_root, bucket, object, &shared_metadata).await; + expected_metadata.push((disk_root.join(bucket).join(object).join("xl.meta"), shared_metadata.clone())); let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("failed to create extra disk endpoint"); let disk = new_disk( &endpoint, @@ -1806,8 +1885,29 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() { .new_cache .checked_flatten(bucket) .expect("healed cache must contain canonical child links"); - assert_eq!(root.objects, 1); + // The fixture intentionally exposes two divergent version histories, so + // the scanner keeps both logical versions visible while discovering heals. + assert_eq!(root.objects, 2); assert!(heal_starts.load(Ordering::Relaxed) > 0, "test must execute the heal child-link path"); + let orphan_version_text = orphan_version.to_string(); + assert!( + healed_versions + .lock() + .expect("heal version capture lock should not be poisoned") + .iter() + .any(|version| version.as_deref() == Some(orphan_version_text.as_str())), + "sub-quorum orphan version must be submitted as an exact heal candidate" + ); + for (path, expected) in expected_metadata { + assert_eq!( + tokio::fs::read(&path) + .await + .expect("scanner discovery must not delete metadata"), + expected, + "scanner discovery must not modify candidate metadata: {}", + path.display() + ); + } } #[tokio::test] From e196a134cc3e8d9947d20ab77f6fd77ae5621141 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 17:28:21 +0800 Subject: [PATCH 17/41] fix(scanner): fence system metadata publication (#6444) * feat(scanner): fence usage publication during data movement * fix(scanner): detect movement refresh state changes * fix(scanner): fence publication during data movement * fix(scanner): close movement epoch publication races * fix(scanner): fence movement-sensitive publication paths * fix(scanner): fence cache and heal recovery paths * fix(scanner): carry publication epoch through scan cycle * fix(scanner): recheck remote cache epoch after save * fix(scanner): recheck local cache epoch before publish * fix(scanner): fence data usage writers and baseline * fix(scanner): expose decommission activity to publication fence * fix(scanner): release publication gate before reads * fix(scanner): complete publication fence integration * fix(scanner): avoid empty usage baseline publication * chore(scanner): gate test-only helpers * fix: use decommission canceler in reload test --------- Co-authored-by: houseme --- crates/ecstore/src/core/pools.rs | 188 ++++++++-- crates/ecstore/src/data_usage/mod.rs | 290 +++++++++++++-- crates/ecstore/src/runtime/instance.rs | 84 ++++- .../ecstore/src/services/rebalance/control.rs | 87 ++++- .../ecstore/src/services/rebalance/runtime.rs | 134 ++++--- crates/ecstore/src/set_disk/mod.rs | 20 + crates/ecstore/src/store/bucket.rs | 2 +- crates/ecstore/src/store/init.rs | 5 + crates/ecstore/src/store/mod.rs | 154 +++++++- crates/ecstore/src/store/rebalance.rs | 16 +- crates/scanner/src/data_usage_define.rs | 26 +- .../src/data_usage_define/persistence.rs | 91 ++++- crates/scanner/src/data_usage_define/tests.rs | 48 +++ crates/scanner/src/lib.rs | 124 +++++++ crates/scanner/src/remote_scanner/stream.rs | 27 +- crates/scanner/src/scanner.rs | 247 +++++++++++-- crates/scanner/src/scanner/cycle_state.rs | 346 +++++++++++++++--- crates/scanner/src/scanner/heal_info.rs | 125 ++++++- crates/scanner/src/scanner/leadership.rs | 139 ++++++- crates/scanner/src/scanner/tests.rs | 213 +++++++++-- crates/scanner/src/scanner/usage_store.rs | 271 +++++++++++--- crates/scanner/src/scanner_io.rs | 22 +- crates/scanner/src/scanner_io/cache.rs | 21 +- crates/scanner/src/scanner_io/io_cache.rs | 87 ++++- crates/scanner/src/scanner_io/io_cycle.rs | 24 +- crates/scanner/src/scanner_io/tests.rs | 44 +++ 26 files changed, 2465 insertions(+), 370 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 16e4394a5..e4b8880e9 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -135,7 +135,7 @@ struct DecommissionOperation { } impl DecommissionCanceler { - fn new(token: CancellationToken) -> Self { + pub(crate) fn new(token: CancellationToken) -> Self { Self { operation: Arc::new(DecommissionOperation { token, @@ -153,7 +153,7 @@ impl DecommissionCanceler { &self.operation.token } - fn is_active(&self) -> bool { + pub(crate) fn is_active(&self) -> bool { self.operation.active.load(Ordering::Acquire) } @@ -1444,6 +1444,30 @@ fn should_replace_pool_status_for_status_refresh( !has_active_worker && persisted.last_update > current.last_update } +fn pool_decommission_movement_snapshot( + info: Option<&PoolDecommissionInfo>, +) -> (bool, bool, bool, bool, bool, Option) { + info.map(|info| { + ( + info.has_decommission_state(), + info.complete, + info.failed, + info.canceled, + info.queued, + info.start_time, + ) + }) + .unwrap_or_default() +} + +pub(crate) fn pool_meta_movement_snapshot_changed(before: &PoolMeta, after: &PoolMeta) -> bool { + before.pools.len() != after.pools.len() + || before.pools.iter().zip(after.pools.iter()).any(|(before, after)| { + pool_decommission_movement_snapshot(before.decommission.as_ref()) + != pool_decommission_movement_snapshot(after.decommission.as_ref()) + }) +} + /// Merges a persisted pool metadata snapshot into `current` monotonically: /// a pool entry is replaced only when no active worker covers it and the /// snapshot is strictly newer, so delayed snapshots never roll back local @@ -3358,6 +3382,8 @@ impl ECStore { .first() .cloned() .ok_or_else(|| Error::other("refresh_pool_status_meta: no pools available"))?; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let mut persisted = PoolMeta::default(); persisted.load(pool, self.pools.clone()).await?; @@ -3370,7 +3396,9 @@ impl ECStore { }; let mut pool_meta = self.pool_meta.write().await; - merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers); + if merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers) { + self.ctx.advance_data_movement_operation_epoch(); + } Ok(()) } @@ -3489,11 +3517,19 @@ impl ECStore { async fn decommission_cancel_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> { ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?; let _start_guard = self.start_gate.lock().await; - let operation_gate = self.ctx.decommission_operation_gate(); - let operation_guard = operation_gate.write().await; - - // Lock order: decommission_cancelers before pool_meta. Holding both makes - // owner validation and the terminal transition one atomic operation. + // Signal cancellation before waiting for the movement writer. A worker + // may still hold a publication read guard while it observes this + // signal; waiting for the writer first would deadlock that handoff. + if let Some(owner) = owner { + owner.cancel(); + } else if let Some(canceler) = self.decommission_cancelers.read().await.get(idx).and_then(Option::as_ref) { + canceler.cancel(); + } + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; + // Lock order: movement gate, then decommission_cancelers, then pool_meta. + // Holding both state locks makes owner validation and the terminal + // transition one atomic operation. let (should_save_pool_meta, should_reload_pool_meta, already_canceled, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut lock = self.pool_meta.write().await; @@ -3562,9 +3598,26 @@ impl ECStore { self.release_decommission_canceler_slot(idx, canceler).await; } + if should_save_pool_meta { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("decommission_cancel for pool {idx}"); - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; + if let Err(err) = + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) + { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "terminal_reload_failed", + error = %err, + "Decommission cancel saved locally but pool meta reload failed" + ); + } } Ok(()) @@ -3589,7 +3642,11 @@ impl ECStore { .unwrap_or((false, false, false, false)); ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled)?; } - self.cancel_decommission_routines_and_wait(&[idx]).await; + // Cancel workers before waiting for the movement writer so active + // object operations can observe the signal and release read guards. + self.cancel_decommission_routines(&[idx]).await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let (should_reload_pool_meta, previous_pool_meta) = { let mut pool_meta = self.pool_meta.write().await; @@ -3606,17 +3663,37 @@ impl ECStore { return Err(err); } + if should_reload_pool_meta { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("clear_decommission for pool {idx}"); - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; + if let Err(err) = + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) + { + warn!( + event = EVENT_DECOMMISSION_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + state = "terminal_reload_failed", + error = %err, + "Decommission clear saved locally but pool meta reload failed" + ); + } } Ok(()) } async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result { + // Serialize promotion and generation capture with clear/restart transitions. let (changed, generation, save_error) = { let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let mut pool_meta = self.pool_meta.write().await; if pool_meta.pools.get(idx).is_none() { return Err(Error::other("failed to start decommission: target pool was not found")); @@ -3644,6 +3721,9 @@ impl ECStore { return Err(err); } + if changed { + self.ctx.advance_data_movement_operation_epoch(); + } if changed && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("promote_queued_decommission for pool {idx}"); if let Err(err) = @@ -3668,6 +3748,8 @@ impl ECStore { } async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> { + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let changed = { let mut pool_meta = self.pool_meta.write().await; pool_meta.record_decommission_terminal_reload_failure(idx, stage, err.to_string())? @@ -3708,19 +3790,20 @@ impl ECStore { is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx)) } + #[cfg(test)] async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) { + self.cancel_decommission_routines(indices).await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; + } + + async fn cancel_decommission_routines(&self, indices: &[usize]) { { let mut cancelers = self.decommission_cancelers.write().await; for idx in indices { take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx); } } - self.wait_for_decommission_side_effects().await; - } - - async fn wait_for_decommission_side_effects(&self) { - let operation_gate = self.ctx.decommission_operation_gate(); - let _operation_guard = operation_gate.write().await; } async fn reserve_decommission_routines( @@ -4218,7 +4301,7 @@ impl ECStore { } decommission_cancel_signal_result(rx.is_cancelled())?; self.ensure_decommission_generation_current(idx, generation).await?; - let operation_gate = self.ctx.decommission_operation_gate(); + let operation_gate = self.ctx.data_movement_operation_gate(); let bucket_incarnation_fence = match expected_bucket_incarnation_id { Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?), @@ -5113,9 +5196,12 @@ impl ECStore { { ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?; let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; - // Lock order: decommission_cancelers before pool_meta. Holding both makes - // owner validation and the terminal transition one atomic operation. + // Lock order: movement gate, then decommission_cancelers, then pool_meta. + // Holding both state locks makes owner validation and the terminal + // transition one atomic operation. let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; @@ -5151,6 +5237,12 @@ impl ECStore { if let Some(canceler) = terminal_canceler.as_ref() { self.release_decommission_canceler_slot(idx, canceler).await; } + + if should_reload_pool_meta { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("decommission_failed for pool {idx}"); if let Some(err) = observe_decommission_terminal_reload_result( @@ -5208,7 +5300,12 @@ impl ECStore { } self.verify_decommission_durable_ilm_receipts(idx).await?; let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; + // Lock order: movement gate, then decommission_cancelers, then pool_meta. + // Holding both state locks makes owner validation and the terminal + // transition one atomic operation. let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; @@ -5249,6 +5346,12 @@ impl ECStore { if let Some(canceler) = terminal_canceler.as_ref() { self.release_decommission_canceler_slot(idx, canceler).await; } + + if should_reload_pool_meta { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); + if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("complete_decommission for pool {idx}"); if let Some(err) = observe_decommission_terminal_reload_result( @@ -5479,11 +5582,16 @@ impl ECStore { self.ensure_decommission_rebalance_idle_after_refresh().await?; let all_space_infos = self.get_decommission_all_pool_space_infos().await?; - self.cancel_decommission_routines_and_wait(&indices).await; + // Signal cancellation before waiting for the movement writer so active + // object operations can observe the signal and release read guards. + self.cancel_decommission_routines(&indices).await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let index_cancelers = if let Some((rx, local_indices)) = reservation { - // Lock order matches terminal transitions: decommission_cancelers - // before pool_meta while start_gate excludes another start. + // Lock order matches terminal transitions: movement gate, then + // decommission_cancelers, then pool_meta while start_gate excludes + // another start. let mut cancelers = self.decommission_cancelers.write().await; let pool_meta = self.pool_meta.read().await; ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_space_infos)?; @@ -5505,6 +5613,10 @@ impl ECStore { let previous_pool_meta = self .save_current_pool_meta_for_decommission_start(&indices, space_infos, decom_buckets) .await?; + self.ctx.advance_data_movement_operation_epoch(); + // The local durable transition is now fenced. Release the writer + // before any peer RPC; remote reload must not block scanner admission. + drop(_movement_guard); if let Some(notification_sys) = runtime_sources::notification_sys() && let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await) @@ -5519,11 +5631,20 @@ impl ECStore { "Decommission start failed after pool metadata save" ); - { - let mut pool_meta = self.pool_meta.write().await; - rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone()); - } - if let Err(rollback_save_err) = self.save_current_pool_meta().await { + let rollback_result = { + let movement_guard = movement_gate.write().await; + { + let mut pool_meta = self.pool_meta.write().await; + rollback_start_decommission_pool_meta(&mut pool_meta, previous_pool_meta.clone()); + } + let rollback_result = self.save_current_pool_meta().await; + if rollback_result.is_ok() { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(movement_guard); + rollback_result + }; + if let Err(rollback_save_err) = rollback_result { error!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, @@ -6527,7 +6648,7 @@ impl ECStore { generation: OffsetDateTime, ) -> Result<()> { self.ensure_decommission_generation_current(idx, generation).await?; - let operation_gate = self.ctx.decommission_operation_gate(); + let operation_gate = self.ctx.data_movement_operation_gate(); run_decommission_side_effect(rx, &operation_gate, || self.check_after_decommission_unfenced(idx)).await } @@ -8077,7 +8198,7 @@ mod pools_tests { ..Default::default() }; - merge_pool_status_refresh(&mut current, persisted, &[false]); + assert!(merge_pool_status_refresh(&mut current, persisted, &[false])); let info = current.pools[0] .decommission @@ -8121,7 +8242,7 @@ mod pools_tests { ..Default::default() }; - merge_pool_status_refresh(&mut current, persisted, &[true]); + assert!(!merge_pool_status_refresh(&mut current, persisted, &[true])); let info = current.pools[0] .decommission @@ -8162,7 +8283,7 @@ mod pools_tests { ..Default::default() }; - merge_pool_status_refresh(&mut current, persisted, &[true]); + assert!(!merge_pool_status_refresh(&mut current, persisted, &[true])); let info = current.pools[0] .decommission @@ -8591,7 +8712,7 @@ mod pools_tests { #[tokio::test] async fn test_decommission_transition_waits_without_registered_canceler() { let store = decommission_worker_test_store(PoolMeta::default(), vec![None]); - let operation_gate = store.ctx.decommission_operation_gate(); + let operation_gate = store.ctx.data_movement_operation_gate(); let operation_guard = operation_gate.read().await; let transition = tokio::spawn({ let store = store.clone(); @@ -11338,6 +11459,7 @@ mod pools_tests { assert!(store.decommission_cancelers.read().await[0].is_none()); assert!(!canceler.is_active()); assert!(canceler.is_cancelled()); + assert_eq!(store.ctx.data_movement_operation_epoch(), 1); } #[test] diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 12cf3117b..0a613e02f 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -388,8 +388,12 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: "nonconverged data usage observations cannot replace the quota-authoritative snapshot", )); } + let Some(expected_publication_epoch) = store.scanner_data_usage_publication_epoch().await else { + return Err(Error::other("data usage publication is blocked by data movement")); + }; // Prevent older data from overwriting newer persisted stats - if let Ok((existing, source)) = load_data_usage_snapshot(store.clone()).await + let existing_snapshot = load_data_usage_snapshot(store.clone()).await; + if let Ok((existing, source)) = existing_snapshot && source.is_authoritative() && let Some(reason) = stale_data_usage_persist_reason_for_source(&data_usage_info, &existing, source, SystemTime::now()) { @@ -400,19 +404,31 @@ pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: return Ok(()); } - save_data_usage_in_backend(data_usage_info, store).await + save_data_usage_in_backend(data_usage_info, store, expected_publication_epoch).await } -async fn save_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc) -> Result<(), Error> { +async fn save_data_usage_in_backend( + data_usage_info: DataUsageInfo, + store: Arc, + expected_publication_epoch: u64, +) -> Result<(), Error> { let data = serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?; // Save to backend using the same mechanism as original code + let Some((publication_guard, publication_epoch)) = store.scanner_data_usage_publication_admission_guard().await else { + return Err(Error::other("data usage publication is blocked by data movement")); + }; + if publication_epoch != expected_publication_epoch { + return Err(Error::other("data usage publication epoch changed before save")); + } crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data) .await .map_err(Error::other)?; + drop(publication_guard); - cleanup_observed_data_usage_after_authoritative_save(store.as_ref(), &data_usage_info).await; + cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref())) + .await; // Invalidate the cached snapshot so readers observe the new save on their // next request instead of waiting out the remaining TTL. The next cached @@ -449,11 +465,24 @@ impl ObservedDataUsageSnapshotCleanup for ECStore { } } -async fn cleanup_observed_data_usage_after_authoritative_save(store: &S, authoritative: &DataUsageInfo) -where +async fn cleanup_observed_data_usage_after_authoritative_save_with_publication( + store: &S, + authoritative: &DataUsageInfo, + publication_store: Option<&ECStore>, +) where S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized, { - let (observed, revision) = match load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await { + let observed_read_epoch = match publication_store { + Some(publication_store) => { + let Some(epoch) = publication_store.scanner_data_usage_publication_epoch().await else { + return; + }; + Some(epoch) + } + None => None, + }; + let observed_snapshot = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await; + let (observed, revision) = match observed_snapshot { Ok(Some(snapshot)) => snapshot, Ok(None) => return, Err(err) => { @@ -469,6 +498,19 @@ where return; } + let publication_guard = match publication_store { + Some(publication_store) => { + let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await + else { + return; + }; + if observed_read_epoch.is_some_and(|expected| expected != publication_epoch) { + return; + } + Some(guard) + } + None => None, + }; match store.delete_observed_data_usage_snapshot(&revision).await { Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::PreconditionFailed) => {} Err(err) => { @@ -479,6 +521,15 @@ where ); } } + drop(publication_guard); +} + +#[cfg(test)] +async fn cleanup_observed_data_usage_after_authoritative_save(store: &S, authoritative: &DataUsageInfo) +where + S: EcstoreObjectIO + ObservedDataUsageSnapshotCleanup + ?Sized, +{ + cleanup_observed_data_usage_after_authoritative_save_with_publication(store, authoritative, None).await; } fn set_buckets_count_from_usage(data_usage_info: &mut DataUsageInfo) { @@ -519,7 +570,7 @@ pub async fn remove_bucket_usage_from_backend(store: Arc, bucket: &str) pub(crate) async fn remove_bucket_usage_for_namespace_change(store: &ECStore, bucket: &str) -> Result<(), Error> { prepare_bucket_usage_for_namespace_change(bucket, None).await?; - remove_bucket_usage_from_backend_with_guard(store, bucket, None).await + remove_bucket_usage_from_backend_with_guard_fenced(store, bucket, None).await } pub(crate) async fn prepare_bucket_usage_for_namespace_change( @@ -542,6 +593,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change( Ok(()) } +#[cfg(test)] pub(crate) async fn remove_bucket_usage_from_backend_with_guard( store: &S, bucket: &str, @@ -551,6 +603,24 @@ where S: EcstoreObjectIO + ?Sized, { let result = remove_bucket_usage_from_backend_with_store_and_guard(store, bucket, guard).await; + invalidate_bucket_usage_snapshot_caches(guard, bucket).await?; + result +} + +pub(crate) async fn remove_bucket_usage_from_backend_with_guard_fenced( + store: &ECStore, + bucket: &str, + guard: Option<&rustfs_lock::NamespaceLockGuard>, +) -> Result<(), Error> { + let result = remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, Some(store)).await; + invalidate_bucket_usage_snapshot_caches(guard, bucket).await?; + result +} + +async fn invalidate_bucket_usage_snapshot_caches( + guard: Option<&rustfs_lock::NamespaceLockGuard>, + bucket: &str, +) -> Result<(), Error> { let mut snapshot_cache = data_usage_snapshot_cache().write().await; ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cache invalidation")?; clear_data_usage_snapshot_cache(&mut snapshot_cache); @@ -558,7 +628,7 @@ where let mut admin_snapshot_cache = admin_data_usage_snapshot_cache().write().await; ensure_bucket_namespace_guard(guard, bucket, "admin data usage snapshot cache invalidation")?; clear_admin_data_usage_snapshot_cache(&mut admin_snapshot_cache); - result + Ok(()) } async fn load_data_usage_for_bucket_removal(store: &S, object: &str) -> Result, Error> @@ -617,48 +687,83 @@ fn ensure_bucket_namespace_guard( Ok(()) } +#[cfg(test)] async fn remove_bucket_usage_from_backend_with_store_and_guard( store: &S, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>, ) -> Result<(), Error> +where + S: EcstoreObjectIO + ?Sized, +{ + remove_bucket_usage_from_backend_with_store_and_guard_and_publication(store, bucket, guard, None).await +} + +async fn remove_bucket_usage_from_backend_with_store_and_guard_and_publication( + store: &S, + bucket: &str, + guard: Option<&rustfs_lock::NamespaceLockGuard>, + publication_store: Option<&ECStore>, +) -> Result<(), Error> where S: EcstoreObjectIO + ?Sized, { ensure_bucket_namespace_guard(guard, bucket, "data usage primary cleanup")?; + let primary_seed_epoch = match publication_store { + Some(publication_store) => Some( + publication_store + .scanner_data_usage_publication_epoch() + .await + .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?, + ), + None => None, + }; let primary_seed = load_data_usage_seed_for_missing_primary(store).await?; - remove_bucket_usage_from_object_with_retries( + remove_bucket_usage_from_object_with_retries_and_publication( store, DATA_USAGE_OBJ_NAME_PATH.as_str(), bucket, DATA_USAGE_REMOVE_CAS_RETRIES, - Some(&primary_seed), + primary_seed.as_ref(), guard, + publication_store.map(|store| (store, primary_seed_epoch)), ) .await?; ensure_bucket_namespace_guard(guard, bucket, "data usage backup cleanup")?; + let backup_seed_epoch = match publication_store { + Some(publication_store) => Some( + publication_store + .scanner_data_usage_publication_epoch() + .await + .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?, + ), + None => None, + }; let backup_seed = load_data_usage_for_bucket_removal(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) .await? - .map_or(primary_seed, |(data_usage_info, _)| data_usage_info); - remove_bucket_usage_from_object_with_retries( + .map(|(data_usage_info, _)| data_usage_info) + .or_else(|| primary_seed.clone()); + remove_bucket_usage_from_object_with_retries_and_publication( store, DATA_USAGE_OBJ_BACKUP_PATH.as_str(), bucket, DATA_USAGE_REMOVE_CAS_RETRIES, - Some(&backup_seed), + backup_seed.as_ref(), guard, + publication_store.map(|store| (store, backup_seed_epoch)), ) .await?; ensure_bucket_namespace_guard(guard, bucket, "observed data usage cleanup")?; - if let Err(err) = remove_bucket_usage_from_object_with_retries( + if let Err(err) = remove_bucket_usage_from_object_with_retries_and_publication( store, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard, + publication_store.map(|store| (store, None)), ) .await { @@ -672,12 +777,21 @@ where LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_BACKUP_PATH.as_str(), ] { - remove_bucket_usage_from_object_with_retries(store, object, bucket, DATA_USAGE_REMOVE_CAS_RETRIES, None, guard).await?; + remove_bucket_usage_from_object_with_retries_and_publication( + store, + object, + bucket, + DATA_USAGE_REMOVE_CAS_RETRIES, + None, + guard, + publication_store.map(|store| (store, None)), + ) + .await?; } Ok(()) } -async fn load_data_usage_seed_for_missing_primary(store: &S) -> Result +async fn load_data_usage_seed_for_missing_primary(store: &S) -> Result, Error> where S: EcstoreObjectIO + ?Sized, { @@ -690,12 +804,13 @@ where if !authoritative { data_usage_info.usage_snapshot_complete = false; } - return Ok(data_usage_info); + return Ok(Some(data_usage_info)); } } - Ok(DataUsageInfo::default()) + Ok(None) } +#[cfg(test)] async fn remove_bucket_usage_from_object_with_retries( store: &S, object: &str, @@ -704,12 +819,42 @@ async fn remove_bucket_usage_from_object_with_retries( missing_seed: Option<&DataUsageInfo>, guard: Option<&rustfs_lock::NamespaceLockGuard>, ) -> Result<(), Error> +where + S: EcstoreObjectIO + ?Sized, +{ + remove_bucket_usage_from_object_with_retries_and_publication(store, object, bucket, cas_retries, missing_seed, guard, None) + .await +} + +async fn remove_bucket_usage_from_object_with_retries_and_publication( + store: &S, + object: &str, + bucket: &str, + cas_retries: usize, + missing_seed: Option<&DataUsageInfo>, + guard: Option<&rustfs_lock::NamespaceLockGuard>, + publication: Option<(&ECStore, Option)>, +) -> Result<(), Error> where S: EcstoreObjectIO + ?Sized, { for attempt in 0..=cas_retries { ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot cleanup")?; - let (mut data_usage_info, revision) = match load_data_usage_for_bucket_removal(store, object).await? { + let read_epoch = match publication { + Some((publication_store, expected_publication_epoch)) => { + let epoch = publication_store + .scanner_data_usage_publication_epoch() + .await + .ok_or_else(|| Error::other("data usage publication is blocked by data movement"))?; + if expected_publication_epoch.is_some_and(|expected| expected != epoch) { + return Err(Error::other("data usage publication epoch changed before snapshot read")); + } + Some(epoch) + } + None => None, + }; + let loaded_snapshot = load_data_usage_for_bucket_removal(store, object).await?; + let (mut data_usage_info, revision) = match loaded_snapshot { Some((data_usage_info, revision)) => (data_usage_info, Some(revision)), None => match missing_seed { Some(data_usage_info) => (data_usage_info.clone(), None), @@ -733,6 +878,22 @@ where }, }; ensure_bucket_namespace_guard(guard, bucket, "data usage snapshot commit")?; + let publication_guard = match publication { + Some((publication_store, expected_publication_epoch)) => { + let Some((guard, publication_epoch)) = publication_store.scanner_data_usage_publication_admission_guard().await + else { + return Err(Error::other("data usage publication is blocked by data movement")); + }; + if expected_publication_epoch + .or(read_epoch) + .is_some_and(|expected| expected != publication_epoch) + { + return Err(Error::other("data usage publication epoch changed before snapshot commit")); + } + Some(guard) + } + None => None, + }; let save_result = store .put_object( RUSTFS_META_BUCKET, @@ -745,6 +906,7 @@ where }, ) .await; + drop(publication_guard); match save_result { Ok(_) => return Ok(()), Err(err) => { @@ -2286,6 +2448,8 @@ pub async fn init_compression_total_memory_from_backend(store: Arc) { #[cfg(test)] mod tests { use super::*; + use crate::layout::endpoints::EndpointServerPools; + use crate::runtime::instance::InstanceContext; use crate::storage_api_contracts::object::ObjectIO as _; use rustfs_data_usage::BucketUsageInfo; use rustfs_lock::{LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey}; @@ -2308,6 +2472,7 @@ mod tests { error_after_commit_put: Option, advance_time_on_put: Option, advance_time_after_get: Option<(UsageObjectSlot, Duration)>, + advance_publication_epoch_after_get: Option<(UsageObjectSlot, Arc)>, advance_time_before_put: Option<(usize, Duration)>, advance_time_after_put: Option<(usize, Duration)>, put_count: usize, @@ -2385,10 +2550,21 @@ mod tests { } _ => None, }; + let advance_publication_epoch = match state.advance_publication_epoch_after_get { + Some((expected_slot, ref ctx)) if expected_slot == slot => { + let ctx = Arc::clone(ctx); + state.advance_publication_epoch_after_get = None; + Some(ctx) + } + _ => None, + }; drop(state); if let Some(duration) = advance { tokio::time::advance(duration).await; } + if let Some(ctx) = advance_publication_epoch { + ctx.advance_data_movement_operation_epoch(); + } Ok(crate::object_api::GetObjectReader { stream: Box::new(Cursor::new(data)), object_info: ObjectInfo { @@ -2589,6 +2765,23 @@ mod tests { .to_string() } + fn build_publication_store(ctx: Arc) -> Arc { + let endpoint_pools = EndpointServerPools::default(); + Arc::new(ECStore { + id: uuid::Uuid::new_v4(), + disk_map: HashMap::new(), + pools: Vec::new(), + peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()), + pool_meta: RwLock::new(crate::core::pools::PoolMeta::default()), + rebalance_meta: RwLock::new(None), + decommission_cancelers: RwLock::new(Vec::new()), + start_gate: TokioMutex::new(()), + pool_meta_save_gate: TokioMutex::new(()), + ctx, + bucket_fence_registry: Arc::default(), + }) + } + #[test] fn data_usage_cache_absence_covers_the_variants_that_actually_arrive() { // `to_object_err` rewrites the raw storage variants before they reach @@ -4333,7 +4526,15 @@ mod tests { .expect("namespace lock acquisition should not fail") .expect("namespace lock should be acquired"), ); - let store = Arc::new(UsageCasStore::default()); + let snapshot = data_usage_info_for_test(BUCKET, 2, 84, SystemTime::now()); + let encoded = serde_json::to_vec(&snapshot).expect("usage snapshot should encode"); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((encoded.clone(), 1)), + backup_object: Some((encoded, 1)), + ..Default::default() + }), + }); let successor = data_usage_info_for_test(BUCKET, 7, 294, SystemTime::now()); let mut snapshot_cache = data_usage_snapshot_cache().write().await; *snapshot_cache = Some(CachedDataUsageSnapshot { @@ -4432,21 +4633,17 @@ mod tests { } #[tokio::test] - async fn remove_bucket_usage_creates_primary_and_backup_fences_when_missing() { + async fn remove_bucket_usage_does_not_synthesize_authoritative_snapshot_when_all_missing() { let store = Arc::new(UsageCasStore::default()); remove_bucket_usage_from_backend_with_store(store.as_ref(), "bucket-a") .await - .expect("bucket removal should create both usage fences"); + .expect("bucket removal should remain a no-op without a usage baseline"); let state = store.state.lock().await; - assert_eq!(state.put_count, 2); - for (data, revision) in [state.object.as_ref(), state.backup_object.as_ref()].into_iter().flatten() { - let saved = serde_json::from_slice::(data).expect("saved usage snapshot should decode"); - assert_eq!(*revision, 1); - assert!(saved.last_update.is_some()); - assert!(!data_usage_contains_bucket(&saved, "bucket-a")); - } + assert_eq!(state.put_count, 0); + assert!(state.object.is_none()); + assert!(state.backup_object.is_none()); } #[tokio::test] @@ -4598,6 +4795,39 @@ mod tests { assert_eq!(backup_err, Error::PreconditionFailed); } + #[tokio::test] + async fn remove_bucket_usage_rejects_movement_epoch_flip_between_read_and_commit() { + let ctx = Arc::new(InstanceContext::new()); + let publication_store = build_publication_store(ctx.clone()); + let snapshot = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now()); + let store = Arc::new(UsageCasStore { + state: Mutex::new(UsageCasState { + object: Some((serde_json::to_vec(&snapshot).expect("usage snapshot should encode"), 1)), + advance_publication_epoch_after_get: Some((UsageObjectSlot::Primary, ctx)), + ..Default::default() + }), + }); + let expected_epoch = publication_store + .scanner_data_usage_publication_epoch() + .await + .expect("idle publication store should admit the initial read"); + + let err = remove_bucket_usage_from_object_with_retries_and_publication( + store.as_ref(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + "bucket-a", + 0, + None, + None, + Some((publication_store.as_ref(), Some(expected_epoch))), + ) + .await + .expect_err("a movement epoch flip during the read must fence the commit"); + + assert!(err.to_string().contains("epoch changed")); + assert_eq!(store.state.lock().await.put_count, 0); + } + #[tokio::test] async fn remove_bucket_usage_confirms_ambiguous_committed_final_attempt() { let initial = data_usage_info_for_test("bucket-a", 2, 84, SystemTime::now()); diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index 9453ed02b..fedf2d7fb 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -52,11 +52,18 @@ use crate::services::tier::tier::TierConfigMgr; use rustfs_lock::{GlobalLockManager, get_global_lock_manager}; use s3s::region::Region; use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, OnceLock}; +use std::sync::{ + Arc, OnceLock, + atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}, +}; use tokio::sync::{OnceCell, RwLock}; use tokio_util::sync::CancellationToken; use uuid::Uuid; +const SCANNER_PUBLICATION_STATE_UNKNOWN: u8 = 0; +const SCANNER_PUBLICATION_STATE_ALLOWED: u8 = 1; +const SCANNER_PUBLICATION_STATE_BLOCKED: u8 = 2; + /// Runtime state owned by a single `ECStore` instance. /// /// This is intentionally minimal in the first migration slice; subsequent @@ -160,10 +167,22 @@ pub struct InstanceContext { /// workers (scanner/heal/tier/lifecycle) without touching another instance. /// Replaces the process-global cancel-token static. background_cancel_token: OnceLock, - /// Serializes decommission data-movement operations with cancellation and - /// a subsequent restart. Readers are held across one object side effect; - /// the transition path takes the writer after cancelling the routine. - decommission_operation_gate: Arc>, + /// Serializes data-movement transitions with scanner publication commits. + /// Readers are held across one publication commit; movement transitions + /// take the writer at their durable state commit boundary. + data_movement_operation_gate: Arc>, + /// Monotonic admission epoch paired with the operation gate. A + /// publication admitted before a movement transition must never be + /// mistaken for one admitted after the transition. + data_movement_operation_epoch: AtomicU64, + /// Once the admission epoch reaches its reserved terminal value, no new + /// publication may be admitted. Keeping this state separate from the + /// saturating counter prevents an unchanged `u64::MAX` value from being + /// mistaken for a fresh epoch after overflow. + data_movement_operation_epoch_exhausted: AtomicBool, + /// Last storage-owned movement snapshot observed under the operation + /// gate. SetDisks cache writers fail closed until ECStore refreshes it. + scanner_publication_state: AtomicU8, /// Resolves object-encryption material at the application boundary. object_encryption_resolver: OnceLock>, tier_delete_journal_recovery_stores: std::sync::Mutex>, @@ -204,7 +223,10 @@ impl InstanceContext { local_disk_set_drives: Arc::new(RwLock::new(Vec::new())), bucket_metadata_sys: std::sync::Mutex::new(None), background_cancel_token: OnceLock::new(), - decommission_operation_gate: Arc::new(RwLock::new(())), + data_movement_operation_gate: Arc::new(RwLock::new(())), + data_movement_operation_epoch: AtomicU64::new(0), + data_movement_operation_epoch_exhausted: AtomicBool::new(false), + scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN), object_encryption_resolver: OnceLock::new(), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()), @@ -223,8 +245,54 @@ impl InstanceContext { self.lock_manager.clone() } - pub(crate) fn decommission_operation_gate(&self) -> Arc> { - Arc::clone(&self.decommission_operation_gate) + pub(crate) fn data_movement_operation_gate(&self) -> Arc> { + Arc::clone(&self.data_movement_operation_gate) + } + + pub(crate) fn data_movement_operation_epoch(&self) -> u64 { + self.data_movement_operation_epoch.load(Ordering::Acquire) + } + + pub(crate) fn data_movement_operation_epoch_exhausted(&self) -> bool { + self.data_movement_operation_epoch_exhausted.load(Ordering::Acquire) + } + + pub(crate) fn scanner_publication_state_allowed(&self) -> bool { + !self.data_movement_operation_epoch_exhausted() + && self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED + } + + pub(crate) fn set_scanner_publication_state(&self, blocked: bool) { + self.scanner_publication_state.store( + if blocked { + SCANNER_PUBLICATION_STATE_BLOCKED + } else { + SCANNER_PUBLICATION_STATE_ALLOWED + }, + Ordering::Release, + ); + } + + pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 { + self.scanner_publication_state + .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release); + let _ = self + .data_movement_operation_epoch + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |epoch| Some(epoch.saturating_add(1))); + let result = self.data_movement_operation_epoch.load(Ordering::Acquire); + if result == u64::MAX { + self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release); + } + result + } + + #[cfg(test)] + pub(crate) fn set_data_movement_operation_epoch_for_test(&self, epoch: u64) { + self.data_movement_operation_epoch.store(epoch, Ordering::Release); + self.data_movement_operation_epoch_exhausted + .store(epoch == u64::MAX, Ordering::Release); + self.scanner_publication_state + .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release); } /// Install the application-owned object-encryption resolver once. diff --git a/crates/ecstore/src/services/rebalance/control.rs b/crates/ecstore/src/services/rebalance/control.rs index 09e1993e7..5e13a1933 100644 --- a/crates/ecstore/src/services/rebalance/control.rs +++ b/crates/ecstore/src/services/rebalance/control.rs @@ -48,12 +48,12 @@ fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usi .unwrap_or_default() } -fn merge_rebalance_status_refresh(current: &mut Option, persisted: RebalanceMeta) { +fn merge_rebalance_status_refresh(current: &mut Option, persisted: RebalanceMeta) -> bool { if persisted.id.is_empty() && persisted.pool_stats.is_empty() { - clear_rebalance_status_refresh(current); - return; + return clear_rebalance_status_refresh(current); } + let before = current.clone(); match current.as_mut() { Some(current_meta) => { if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict @@ -66,14 +66,41 @@ fn merge_rebalance_status_refresh(current: &mut Option, persisted *current = Some(persisted); } } + + match (before.as_ref(), current.as_ref()) { + (None, None) => false, + (None, Some(_)) | (Some(_), None) => true, + (Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after), + } } -fn clear_rebalance_status_refresh(current: &mut Option) { +fn clear_rebalance_status_refresh(current: &mut Option) -> bool { if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) { - *current = None; + current.take().is_some() + } else { + false } } +fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool { + let Some(current) = current else { + return true; + }; + + current.id != persisted.id + || current.stopped_at != persisted.stopped_at + || current.pool_stats.len() != persisted.pool_stats.len() + || current + .pool_stats + .iter() + .zip(persisted.pool_stats.iter()) + .any(|(current, persisted)| { + current.participating != persisted.participating + || current.info.status != persisted.info.status + || current.info.stopping != persisted.info.stopping + }) +} + impl ECStore { pub(super) async fn save_rebalance_meta_with_merge( &self, @@ -121,7 +148,10 @@ impl ECStore { "Loading rebalance metadata" ); let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; if resolve_rebalance_meta_load_result(meta.load(pool).await)? { + let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta); { let mut rebalance_meta = self.rebalance_meta.write().await; @@ -130,6 +160,10 @@ impl ECStore { drop(rebalance_meta); } + if movement_changed { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?; debug!( event = EVENT_REBALANCE_STATE, @@ -139,10 +173,15 @@ impl ECStore { "Loaded rebalance metadata" ); } else { + let movement_changed = self.rebalance_meta.read().await.is_some(); { let mut rebalance_meta = self.rebalance_meta.write().await; *rebalance_meta = None; } + if movement_changed { + self.ctx.advance_data_movement_operation_epoch(); + } + drop(_movement_guard); debug!( event = EVENT_REBALANCE_STATE, component = LOG_COMPONENT_ECSTORE, @@ -160,14 +199,20 @@ impl ECStore { pub async fn refresh_rebalance_status_meta(&self) -> Result<()> { let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?; let mut persisted = RebalanceMeta::new(); + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; match persisted.load(pool).await { Ok(()) => { let mut rebalance_meta = self.rebalance_meta.write().await; - merge_rebalance_status_refresh(&mut rebalance_meta, persisted); + if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) { + self.ctx.advance_data_movement_operation_epoch(); + } } Err(Error::ConfigNotFound) => { let mut rebalance_meta = self.rebalance_meta.write().await; - clear_rebalance_status_refresh(&mut rebalance_meta); + if clear_rebalance_status_refresh(&mut rebalance_meta) { + self.ctx.advance_data_movement_operation_epoch(); + } } Err(err) => { return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}"))); @@ -349,6 +394,8 @@ impl ECStore { #[tracing::instrument(skip(self, bucktes))] pub async fn init_rebalance_start(self: &Arc, bucktes: Vec) -> Result { let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let decommission_running = self.is_decommission_running().await; { @@ -356,12 +403,16 @@ impl ECStore { validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?; } - self.init_rebalance_meta(bucktes).await + let id = self.init_rebalance_meta(bucktes).await?; + self.ctx.advance_data_movement_operation_epoch(); + Ok(id) } #[tracing::instrument(skip(self))] pub async fn start_rebalance_for_id(self: &Arc, expected_id: &str) -> Result<()> { let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; { let rebalance_meta = self.rebalance_meta.read().await; @@ -379,7 +430,10 @@ impl ECStore { } } - self.start_rebalance().await + if self.start_rebalance_inner().await? { + self.ctx.advance_data_movement_operation_epoch(); + } + Ok(()) } pub async fn rollback_rebalance_start_for_id(self: &Arc, expected_id: Option<&str>, start_error: String) -> Result<()> { @@ -548,6 +602,9 @@ impl ECStore { #[tracing::instrument(skip(self))] pub async fn stop_rebalance_for_id(self: &Arc, expected_id: Option<&str>) -> Result<()> { + let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id) @@ -560,6 +617,7 @@ impl ECStore { .await, "stop_rebalance", )?; + self.ctx.advance_data_movement_operation_epoch(); } Ok(()) @@ -570,6 +628,9 @@ impl ECStore { expected_id: Option<&str>, start_error: String, ) -> Result<()> { + let _start_guard = self.start_gate.lock().await; + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; rollback_rebalance_start_meta_snapshot_for_id( @@ -587,6 +648,7 @@ impl ECStore { .await, "rollback_rebalance_start", )?; + self.ctx.advance_data_movement_operation_epoch(); } Ok(()) @@ -597,6 +659,8 @@ impl ECStore { return Ok(()); } + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let encoded_error = encode_rebalance_stop_propagation_record(&record); let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; @@ -610,6 +674,7 @@ impl ECStore { .await, "record_rebalance_stop_propagation", )?; + self.ctx.advance_data_movement_operation_epoch(); } Ok(()) @@ -682,7 +747,7 @@ mod tests { ..Default::default() }; - merge_rebalance_status_refresh(&mut current, persisted); + assert!(merge_rebalance_status_refresh(&mut current, persisted)); let refreshed = current.as_ref().expect("refresh should keep rebalance metadata"); assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed); @@ -721,7 +786,7 @@ mod tests { ..Default::default() }; - merge_rebalance_status_refresh(&mut current, persisted); + assert!(!merge_rebalance_status_refresh(&mut current, persisted)); assert!( current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(), diff --git a/crates/ecstore/src/services/rebalance/runtime.rs b/crates/ecstore/src/services/rebalance/runtime.rs index 906e92922..83a2a87c3 100644 --- a/crates/ecstore/src/services/rebalance/runtime.rs +++ b/crates/ecstore/src/services/rebalance/runtime.rs @@ -42,6 +42,15 @@ pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap) -> Result<()> { + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; + if self.start_rebalance_inner().await? { + self.ctx.advance_data_movement_operation_epoch(); + } + Ok(()) + } + + pub(super) async fn start_rebalance_inner(self: &Arc) -> Result { info!( event = EVENT_REBALANCE_STATE, component = LOG_COMPONENT_ECSTORE, @@ -55,6 +64,7 @@ impl ECStore { let cancel_tx = CancellationToken::new(); let rx = cancel_tx.clone(); let mut meta_to_save = None; + let mut movement_changed = false; { let mut rebalance_meta = self.rebalance_meta.write().await; @@ -72,14 +82,16 @@ impl ECStore { reason = "already_in_progress", "Skipped duplicate rebalance start" ); - return Ok(()); + return Ok(false); } let now = OffsetDateTime::now_utc(); if complete_rebalance_pools_at_goal(meta, now) { meta_to_save = Some(meta.clone()); + movement_changed = true; } if complete_rebalance_pools_with_empty_queue(meta, now) { meta_to_save = Some(meta.clone()); + movement_changed = true; } meta.cancel = Some(cancel_tx); @@ -118,7 +130,7 @@ impl ECStore { reason = "no_participants", "Skipped rebalance start because no pools are participating" ); - return Ok(()); + return Ok(movement_changed); } let mut workers_started = 0usize; @@ -186,7 +198,7 @@ impl ECStore { reason = "no_local_participants", "Skipped rebalance start because no local pools are participating" ); - return Ok(()); + return Ok(movement_changed); } info!( @@ -197,7 +209,7 @@ impl ECStore { worker_count = workers_started, "Rebalance started" ); - Ok(()) + Ok(true) } #[tracing::instrument(skip(self, rx))] @@ -214,53 +226,77 @@ impl ECStore { let mut quit = false; loop { + let mut terminal_state_saved = false; tokio::select! { result = done_rx.recv() => { quit = true; let now = OffsetDateTime::now_utc(); let terminal_event = classify_rebalance_terminal_event(result, now); msg = terminal_event.message().to_string(); - let mut rebalance_meta = store.rebalance_meta.write().await; - if let Some(meta) = rebalance_meta.as_mut() { - let meta_stopped = meta.stopped_at.is_some(); - if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { - if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. }) - && has_rebalance_cleanup_warnings(pool_stat) - { - pool_stat.info.stopping = false; - pool_stat.info.status = RebalStatus::Failed; - pool_stat.info.end_time = Some(now); - pool_stat.info.last_error = Some( - pool_stat - .cleanup_warnings - .last_message - .clone() - .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()), - ); - } else if should_preserve_rebalance_stopped_state( - meta_stopped, - pool_stat.info.status, - &terminal_event, - ) { - debug!( - event = EVENT_REBALANCE_STATE, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REBALANCE, - pool_index, - state = "stopped_preserved", - "Preserved stopped rebalance status" - ); + let movement_gate = store.ctx.data_movement_operation_gate(); + let movement_guard = movement_gate.write().await; + let previous_meta = store.rebalance_meta.read().await.clone(); + let terminal_state_present = { + let mut rebalance_meta = store.rebalance_meta.write().await; + if let Some(meta) = rebalance_meta.as_mut() { + let meta_stopped = meta.stopped_at.is_some(); + if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { + if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. }) + && has_rebalance_cleanup_warnings(pool_stat) + { + pool_stat.info.stopping = false; + pool_stat.info.status = RebalStatus::Failed; + pool_stat.info.end_time = Some(now); + pool_stat.info.last_error = Some( + pool_stat + .cleanup_warnings + .last_message + .clone() + .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()), + ); + } else if should_preserve_rebalance_stopped_state( + meta_stopped, + pool_stat.info.status, + &terminal_event, + ) { + debug!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + pool_index, + state = "stopped_preserved", + "Preserved stopped rebalance status" + ); + } else { + pool_stat.info.stopping = false; + apply_rebalance_terminal_event( + &mut pool_stat.info.status, + &mut pool_stat.info.end_time, + &mut pool_stat.info.last_error, + terminal_event, + now, + ); + } + true } else { - pool_stat.info.stopping = false; - apply_rebalance_terminal_event( - &mut pool_stat.info.status, - &mut pool_stat.info.end_time, - &mut pool_stat.info.last_error, - terminal_event, - now, - ); + false } + } else { + false } + }; + + if terminal_state_present { + if let Err(err) = store.save_rebalance_stats_inner(pool_index, RebalSaveOpt::Stats).await { + let mut rebalance_meta = store.rebalance_meta.write().await; + *rebalance_meta = previous_meta; + drop(movement_guard); + return Err(Error::other(format!( + "rebalance terminal state save failed for pool {pool_index}: {err}" + ))); + } + store.ctx.advance_data_movement_operation_epoch(); + terminal_state_saved = true; } } _ = timer.tick() => { @@ -269,7 +305,7 @@ impl ECStore { } } - if let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await { + if !terminal_state_saved && let Err(err) = store.save_rebalance_stats(pool_index, RebalSaveOpt::Stats).await { let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}")); error!("{} err: {:?}", msg, wrapped); if quit { @@ -590,16 +626,14 @@ impl ECStore { meta.percent_free_goal, ) { - pool_stat.info.status = RebalStatus::Completed; - pool_stat.info.end_time = Some(OffsetDateTime::now_utc()); info!( event = EVENT_REBALANCE_STATE, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_REBALANCE, pool_index, - state = "completed", + state = "completion_ready", percent_free = pfi, - "Marked rebalance pool completed" + "Rebalance pool reached completion goal" ); return true; } @@ -612,6 +646,12 @@ impl ECStore { impl ECStore { #[tracing::instrument(skip(self))] pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> { + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; + self.save_rebalance_stats_inner(pool_idx, opt).await + } + + pub(super) async fn save_rebalance_stats_inner(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> { let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; let Some(meta) = rebalance_meta.as_mut() else { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 0422ba94e..a70830f09 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -3614,6 +3614,26 @@ impl SetDisks { &self.ctx } + /// Admit one short scanner cache publication under this set's instance + /// movement fence. The caller must hold the returned guard through its + /// final conditional cache write; no scan-round work belongs under it. + pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> { + let operation_gate = self.ctx.data_movement_operation_gate(); + let operation_guard = operation_gate.read_owned().await; + if self.ctx.scanner_publication_state_allowed() { + let epoch = self.ctx.data_movement_operation_epoch(); + return Some((operation_guard, epoch)); + } + + // The owner deliberately marks the cached state UNKNOWN after every + // movement epoch advance. Do not strand remote scanner writers in that + // state: release this guard before asking the storage owner to refresh + // its durable movement snapshot, since the owner uses the same gate. + drop(operation_guard); + let owner = runtime_sources::object_store_handle().filter(|owner| Arc::ptr_eq(&owner.ctx, &self.ctx))?; + owner.scanner_data_usage_publication_admission_guard().await + } + /// Whether both sets' namespace-lock implementations cover the same object key. pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool { match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) { diff --git a/crates/ecstore/src/store/bucket.rs b/crates/ecstore/src/store/bucket.rs index 77cdbc954..6ad051c0a 100644 --- a/crates/ecstore/src/store/bucket.rs +++ b/crates/ecstore/src/store/bucket.rs @@ -327,7 +327,7 @@ impl ECStore { async fn cleanup_bucket_usage(&self, bucket: &str, guard: Option<&rustfs_lock::NamespaceLockGuard>) -> Result<()> { run_bucket_usage_cleanup(guard, bucket, async { crate::data_usage::prepare_bucket_usage_for_namespace_change(bucket, guard).await?; - crate::data_usage::remove_bucket_usage_from_backend_with_guard(self, bucket, guard).await + crate::data_usage::remove_bucket_usage_from_backend_with_guard_fenced(self, bucket, guard).await }) .await } diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 1d143201c..5b9e67ab0 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -495,6 +495,11 @@ impl ECStore { ); } + // Initialize the storage-owned scanner publication state only after + // both movement metadata sources have been loaded. SetDisks cache + // writers remain fail-closed until this snapshot is available. + let _ = self.scanner_data_usage_publication_blocked().await; + let pools = installed_pool_meta.return_resumable_pools(); let mut pool_indices = Vec::with_capacity(pools.len()); diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index a40edd929..af7dcd44c 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -44,7 +44,7 @@ use crate::error::{ use crate::runtime::global::DISK_RESERVE_FRACTION; use crate::runtime::instance::InstanceContext; use crate::runtime::sources as runtime_sources; -use crate::services::rebalance::RebalanceMeta; +use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission}; use crate::storage_api_contracts::{ bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, @@ -349,16 +349,81 @@ impl ECStore { /// remain suspended until an operator clears or retries them, so they are /// a publication barrier even after the worker has stopped. pub async fn scanner_data_usage_publication_blocked(&self) -> bool { - if self.scanner_data_movement_active().await { + let operation_gate = self.ctx.data_movement_operation_gate(); + let _operation_guard = operation_gate.read_owned().await; + self.scanner_data_usage_publication_snapshot_blocked().await + } + + async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool { + if self.ctx.data_movement_operation_epoch_exhausted() { + self.ctx.set_scanner_publication_state(true); return true; } - + let decommission_cancelers = self.decommission_cancelers.read().await; + let decommission_active = decommission_cancelers + .iter() + .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active)); let pool_meta = self.pool_meta.read().await; - pool_meta.pools.iter().any(|pool| { + let decommission_active = decommission_active + || pool_meta.pools.iter().any(|pool| { + pool.decommission + .as_ref() + .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled) + }); + let decommission_terminal = pool_meta.pools.iter().any(|pool| { pool.decommission .as_ref() .is_some_and(|info| !info.queued && (info.failed || info.canceled)) - }) + }); + drop(pool_meta); + + let rebalance_active = self + .rebalance_meta + .read() + .await + .as_ref() + .is_some_and(is_rebalance_conflicting_with_decommission); + + let blocked = decommission_active || decommission_terminal || rebalance_active; + self.ctx.set_scanner_publication_state(blocked); + blocked + } + + /// Admit one short data-usage publication commit under the same + /// per-instance gate used by decommission side effects and transitions. + /// The epoch is sampled while the read guard is held, so a transition + /// cannot cross this admission without waiting for the commit to finish. + pub async fn scanner_data_usage_publication_read_guard(&self) -> (tokio::sync::OwnedRwLockReadGuard<()>, u64) { + let operation_gate = self.ctx.data_movement_operation_gate(); + let operation_guard = operation_gate.read_owned().await; + let epoch = self.ctx.data_movement_operation_epoch(); + (operation_guard, epoch) + } + + /// Acquire the movement gate and inspect the movement owner once. The + /// state inspection is performed after acquiring the read guard so a + /// transition cannot update its durable state between the check and the + /// publication commit. + pub async fn scanner_data_usage_publication_admission_guard(&self) -> Option<(tokio::sync::OwnedRwLockReadGuard<()>, u64)> { + let operation_gate = self.ctx.data_movement_operation_gate(); + let operation_guard = operation_gate.read_owned().await; + if self.ctx.data_movement_operation_epoch_exhausted() { + return None; + } + if self.scanner_data_usage_publication_snapshot_blocked().await { + return None; + } + + Some((operation_guard, self.ctx.data_movement_operation_epoch())) + } + + /// Capture the current publication epoch without holding the movement + /// gate across backend I/O. Callers must re-admit the same epoch before a + /// mutation commits. + pub(crate) async fn scanner_data_usage_publication_epoch(&self) -> Option { + let (operation_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?; + drop(operation_guard); + Some(epoch) } } @@ -995,6 +1060,85 @@ mod tests { } } + #[tokio::test] + async fn scanner_data_usage_publication_admission_is_fenced_and_epoch_monotonic() { + let store = build_store_with_ctx(Arc::new(InstanceContext::new())); + let operation_gate = store.ctx.data_movement_operation_gate(); + let movement_guard = operation_gate.write().await; + let pending = { + let store = store.clone(); + tokio::spawn(async move { store.scanner_data_usage_publication_admission_guard().await }) + }; + tokio::task::yield_now().await; + assert!(!pending.is_finished(), "publication admission must wait for a movement writer"); + drop(movement_guard); + + let (_, epoch) = pending + .await + .expect("publication admission task should not panic") + .expect("idle store should admit publication"); + assert_eq!(epoch, 0); + assert_eq!(store.ctx.advance_data_movement_operation_epoch(), 1); + let (_, next_epoch) = store + .scanner_data_usage_publication_admission_guard() + .await + .expect("idle store should admit the next publication"); + assert_eq!(next_epoch, 1); + } + + #[tokio::test] + async fn scanner_data_usage_publication_epoch_releases_gate_before_backend_io() { + let store = build_store_with_ctx(Arc::new(InstanceContext::new())); + let epoch = store + .scanner_data_usage_publication_epoch() + .await + .expect("idle store should expose a publication epoch"); + assert_eq!(epoch, 0); + + let operation_gate = store.ctx.data_movement_operation_gate(); + let _movement_guard = tokio::time::timeout(Duration::from_secs(1), operation_gate.write()) + .await + .expect("epoch capture must not hold the movement gate across backend I/O"); + } + + #[tokio::test] + async fn scanner_data_usage_publication_admission_blocks_active_rebalance_snapshot() { + let store = build_store_with_ctx(Arc::new(InstanceContext::new())); + *store.rebalance_meta.write().await = Some(RebalanceMeta { + pool_stats: vec![crate::services::rebalance::RebalanceStats { + participating: true, + info: crate::services::rebalance::RebalanceInfo { + status: crate::services::rebalance::RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }); + + assert!( + store.scanner_data_usage_publication_admission_guard().await.is_none(), + "active rebalance must fail closed at the storage-owned admission boundary" + ); + } + + #[tokio::test] + async fn scanner_publication_epoch_exhaustion_fails_closed_after_max() { + let store = build_store_with_ctx(Arc::new(InstanceContext::new())); + store.ctx.set_data_movement_operation_epoch_for_test(u64::MAX - 1); + + assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX); + assert!(store.ctx.data_movement_operation_epoch_exhausted()); + assert!( + store.scanner_data_usage_publication_admission_guard().await.is_none(), + "publication must fail closed at the reserved terminal epoch" + ); + + assert_eq!(store.ctx.advance_data_movement_operation_epoch(), u64::MAX); + assert!(store.ctx.data_movement_operation_epoch_exhausted()); + assert!(store.scanner_data_usage_publication_blocked().await); + } + // The object graph is the isolation carrier: two ECStore instances holding // distinct contexts report independent erasure state through their real // `&self` accessors — no cross-contamination. diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index cc9d123c7..550bd36e6 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -694,6 +694,11 @@ impl ECStore { /// callers only trigger missing-worker recovery after a real state change; /// delayed snapshots are merged monotonically and never blind-assigned. pub async fn reload_pool_meta(&self) -> Result { + // Serialize the durable reload with local movement transitions. Loading + // before acquiring this gate would allow a stale disk snapshot to + // overwrite a newer local transition after the writer commits. + let movement_gate = self.ctx.data_movement_operation_gate(); + let _movement_guard = movement_gate.write().await; let mut reloaded = PoolMeta::default(); resolve_store_rebalance_pool_meta_reload_result( reloaded.load(self.pools[0].clone(), self.pools.clone()).await, @@ -701,15 +706,22 @@ impl ECStore { )?; // Lock order: release the decommission_cancelers guard before taking - // the pool_meta write guard; neither is held across the disk read. + // the pool_meta write guard; neither is held without the movement gate. let active_workers = { let cancelers = self.decommission_cancelers.read().await; - cancelers.iter().map(Option::is_some).collect::>() + cancelers + .iter() + .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active)) + .collect::>() }; let incoming_has_pools = !reloaded.pools.is_empty(); let mut pool_meta = self.pool_meta.write().await; + let movement_before = pool_meta.clone(); let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers); + if crate::core::pools::pool_meta_movement_snapshot_changed(&movement_before, &pool_meta) { + self.ctx.advance_data_movement_operation_epoch(); + } if !merged_newer && !incoming_has_pools { warn!( diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 295a1da85..3c69d2173 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -36,13 +36,13 @@ use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use tokio::time::{Duration, Instant, sleep, timeout}; use tracing::{debug, warn}; -use crate::ScannerObjectIO; use crate::storage_api::owner::HTTPPreconditions; use crate::{ BUCKET_META_PREFIX, EcstoreError as Error, EcstoreResult as StorageResult, RUSTFS_META_BUCKET, ReplicationConfig, - ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError, TRANSITION_COMPLETE, save_config, - save_config_with_preconditions, storageclass, + SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, StorageError, + TRANSITION_COMPLETE, save_config, save_config_with_preconditions, scanner_publication_admission_for_epoch, storageclass, }; +use crate::{ScannerConfigObjectDelete, ScannerObjectIO}; // Data usage constants pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; @@ -119,9 +119,13 @@ pub(crate) async fn read_config_with_revision( .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag")))?; Ok((Some(reader.read_all().await?), revision)) } - Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => { - Ok((None, DataUsageCacheRevision::Missing)) - } + Err( + Error::ConfigNotFound + | Error::FileNotFound + | Error::VolumeNotFound + | Error::ObjectNotFound(_, _) + | Error::BucketNotFound(_), + ) => Ok((None, DataUsageCacheRevision::Missing)), Err(err) => Err(err), } } @@ -147,9 +151,13 @@ pub(crate) async fn read_config_revision(store: Arc, path .filter(|etag| !etag.is_empty()) .map(DataUsageCacheRevision::Etag) .ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))), - Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => { - Ok(DataUsageCacheRevision::Missing) - } + Err( + Error::ConfigNotFound + | Error::FileNotFound + | Error::VolumeNotFound + | Error::ObjectNotFound(_, _) + | Error::BucketNotFound(_), + ) => Ok(DataUsageCacheRevision::Missing), Err(err) => Err(err), } } diff --git a/crates/scanner/src/data_usage_define/persistence.rs b/crates/scanner/src/data_usage_define/persistence.rs index b0a28f504..18ac13881 100644 --- a/crates/scanner/src/data_usage_define/persistence.rs +++ b/crates/scanner/src/data_usage_define/persistence.rs @@ -15,6 +15,42 @@ use super::*; use std::sync::Arc; +#[derive(Debug)] +struct CachePublicationAdmissionUnavailable; + +impl std::fmt::Display for CachePublicationAdmissionUnavailable { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("scanner cache publication admission is unavailable") + } +} + +impl std::error::Error for CachePublicationAdmissionUnavailable {} + +fn cache_publication_admission_unavailable() -> Error { + Error::Io(std::io::Error::other(CachePublicationAdmissionUnavailable)) +} + +fn cache_publication_epoch_changed() -> Error { + Error::other(SCANNER_PUBLICATION_EPOCH_CHANGED) +} + +fn is_cache_publication_admission_unavailable(error: &StorageError) -> bool { + matches!( + error, + StorageError::Io(io_error) + if io_error + .get_ref() + .is_some_and(|source| source.downcast_ref::().is_some()) + ) +} + +fn is_cache_publication_epoch_changed(error: &StorageError) -> bool { + matches!( + error, + StorageError::Io(io_error) if io_error.to_string() == SCANNER_PUBLICATION_EPOCH_CHANGED + ) +} + pub(super) enum DataUsageCacheLoadAttempt { Loaded { cache: Box, @@ -368,6 +404,12 @@ impl DataUsageCache { fn should_retry_save_error(err: &StorageError) -> bool { // Usage-cache files are best-effort scanner checkpoints. Retrying namespace // lock failures immediately only adds more lock traffic to the same hot object. + if is_cache_publication_admission_unavailable(err) { + return false; + } + if is_cache_publication_epoch_changed(err) { + return false; + } !matches!( err, StorageError::Lock(_) @@ -423,13 +465,14 @@ impl DataUsageCache { Err(last_err.unwrap_or_else(|| StorageError::other("Failed to save data usage cache".to_string()))) } - async fn save_path_with_retry( + async fn save_path_with_retry( store: Arc, path: &str, buf: &[u8], timeout_duration: Duration, max_retries: u32, revision: Option, + expected_epoch: Option, ) -> StorageResult<()> { Self::ensure_cache_save_metrics_registered(); let path_type = Self::cache_path_type(path); @@ -441,6 +484,17 @@ impl DataUsageCache { let buf_clone = buf.to_vec(); let revision = revision.clone(); async move { + let publication_admission = match expected_epoch { + Some(expected_epoch) => scanner_publication_admission_for_epoch(store_clone.clone(), expected_epoch).await, + None => store_clone.scanner_data_usage_publication_admission().await, + }; + let Some(_publication_admission) = publication_admission else { + return Err(if expected_epoch.is_some() { + cache_publication_epoch_changed() + } else { + cache_publication_admission_unavailable() + }); + }; if let Some(revision) = revision { save_config_with_preconditions(store_clone, &path_clone, buf_clone, revision.preconditions()).await?; } else { @@ -454,6 +508,14 @@ impl DataUsageCache { return Ok(()); }; + // An epoch-specific admission failure is authoritative: reconciling + // identical bytes cannot prove that this snapshot belongs to the + // captured movement epoch. Do not turn that fence failure into a + // successful stale publication. + if is_cache_publication_epoch_changed(&save_err) { + return Err(save_err); + } + for attempt in 0..=max_retries { let reconcile = timeout(timeout_duration, async { let mut reader = store @@ -486,24 +548,36 @@ impl DataUsageCache { Err(save_err) } - pub async fn save(&self, store: Arc, name: &str) -> StorageResult<()> { - self.save_inner(store, name, None).await + pub async fn save(&self, store: Arc, name: &str) -> StorageResult<()> { + self.save_inner(store, name, None, None).await } - pub(crate) async fn save_with_revisions( + #[cfg(test)] + pub(crate) async fn save_with_revisions( &self, store: Arc, name: &str, revisions: &DataUsageCacheRevisions, ) -> StorageResult<()> { - self.save_inner(store, name, Some(revisions)).await + self.save_inner(store, name, Some(revisions), None).await } - async fn save_inner( + pub(crate) async fn save_with_revisions_for_epoch( + &self, + store: Arc, + name: &str, + revisions: &DataUsageCacheRevisions, + expected_epoch: u64, + ) -> StorageResult<()> { + self.save_inner(store, name, Some(revisions), Some(expected_epoch)).await + } + + async fn save_inner( &self, store: Arc, name: &str, revisions: Option<&DataUsageCacheRevisions>, + expected_epoch: Option, ) -> StorageResult<()> { let mut buf = Vec::new(); self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?; @@ -517,6 +591,7 @@ impl DataUsageCache { timeout_duration, DATA_USAGE_CACHE_SAVE_RETRIES, revisions.map(|revisions| revisions.main.clone()), + expected_epoch, ) .await?; @@ -534,6 +609,7 @@ impl DataUsageCache { backup_timeout_duration, DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES, backup_revision, + expected_epoch, ) .await { @@ -548,6 +624,9 @@ impl DataUsageCache { error = %e, "Scanner cache backup save failed" ); + if is_cache_publication_admission_unavailable(&e) || is_cache_publication_epoch_changed(&e) { + return Err(e); + } } Ok(()) } diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index bdccb11c1..9a458d4b0 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -198,6 +198,22 @@ impl ObjectIO for CacheReadStore { } } +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for CacheReadStore { + async fn delete_config_object( + &self, + _bucket: &str, + _object: &str, + _opts: crate::ScannerObjectOptions, + ) -> crate::EcstoreResult { + Err(crate::EcstoreError::NotImplemented) + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + Some(crate::ScannerDataUsagePublicationAdmission::unfenced()) + } +} + #[async_trait::async_trait] impl ObjectIO for AmbiguousCacheCommitStore { type Error = Error; @@ -243,6 +259,22 @@ impl ObjectIO for AmbiguousCacheCommitStore { } } +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for AmbiguousCacheCommitStore { + async fn delete_config_object( + &self, + _bucket: &str, + _object: &str, + _opts: crate::ScannerObjectOptions, + ) -> crate::EcstoreResult { + Err(crate::EcstoreError::NotImplemented) + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + Some(crate::ScannerDataUsagePublicationAdmission::unfenced()) + } +} + #[async_trait::async_trait] impl ObjectIO for BackupFallbackStore { type Error = Error; @@ -314,6 +346,22 @@ impl ObjectIO for BackupFallbackStore { } } +#[async_trait::async_trait] +impl crate::ScannerConfigObjectDelete for BackupFallbackStore { + async fn delete_config_object( + &self, + _bucket: &str, + _object: &str, + _opts: crate::ScannerObjectOptions, + ) -> crate::EcstoreResult { + Err(crate::EcstoreError::NotImplemented) + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + Some(crate::ScannerDataUsagePublicationAdmission::unfenced()) + } +} + #[test] fn cache_revisions_map_to_compare_and_swap_preconditions() { let missing = DataUsageCacheRevision::Missing.preconditions(); diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index fa80ce49a..70fa9e530 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -513,6 +513,75 @@ where .await } +pub(crate) async fn save_config_with_publication_admission_for_epoch( + api: Arc, + file: &str, + data: Vec, + preconditions: HTTPPreconditions, + expected_epoch: u64, +) -> EcstoreResult +where + S: ScannerObjectIO + ScannerConfigObjectDelete, +{ + let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + }; + save_config_with_preconditions(api, file, data, preconditions).await +} + +pub(crate) const SCANNER_PUBLICATION_EPOCH_CHANGED: &str = "scanner publication epoch changed before commit"; + +pub(crate) fn scanner_publication_epoch_changed(error: &EcstoreError) -> bool { + matches!( + error, + EcstoreError::Io(io_error) if io_error.to_string() == SCANNER_PUBLICATION_EPOCH_CHANGED + ) +} + +pub(crate) async fn delete_config_with_publication_admission_for_epoch( + api: Arc, + bucket: &str, + object: &str, + opts: ScannerObjectOptions, + expected_epoch: u64, +) -> EcstoreResult +where + S: ScannerObjectIO + ScannerConfigObjectDelete, +{ + let Some(_admission) = scanner_publication_admission_for_epoch(api.clone(), expected_epoch).await else { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + }; + api.delete_config_object(bucket, object, opts).await +} + +/// Capture the storage-owned publication epoch without retaining the read +/// guard across a potentially slow metadata read. Callers must compare this +/// token with a fresh admission immediately before their conditional write. +pub(crate) async fn scanner_publication_epoch(api: Arc) -> Option +where + S: ScannerConfigObjectDelete, +{ + let admission = api.scanner_data_usage_publication_admission().await?; + Some(admission.epoch()) +} + +/// Re-admit a publication only when the storage-owned movement epoch is still +/// the one observed before the caller's metadata read. The returned guard +/// remains held through the caller's short conditional commit. +pub(crate) async fn scanner_publication_admission_for_epoch( + api: Arc, + expected_epoch: u64, +) -> Option +where + S: ScannerConfigObjectDelete, +{ + let admission = api.scanner_data_usage_publication_admission().await?; + if admission.epoch() != expected_epoch { + return None; + } + Some(admission) +} + pub(crate) async fn save_config_shared_with_preconditions( api: Arc, file: &str, @@ -587,6 +656,39 @@ pub trait ScannerConfigObjectDelete: Send + Sync + std::fmt::Debug + 'static { object: &str, opts: ScannerObjectOptions, ) -> EcstoreResult; + + /// Acquire storage-owned admission for one short data-usage publication + /// commit. Implementations without a storage-owned movement owner fail + /// closed; test fixtures opt into the explicit unfenced helper. + async fn scanner_data_usage_publication_admission(&self) -> Option { + None + } +} + +pub struct ScannerDataUsagePublicationAdmission { + epoch: u64, + _read_guard: Option>, +} + +impl ScannerDataUsagePublicationAdmission { + #[cfg(test)] + pub(crate) fn unfenced() -> Self { + Self { + epoch: 0, + _read_guard: None, + } + } + + fn fenced(read_guard: tokio::sync::OwnedRwLockReadGuard<()>, epoch: u64) -> Self { + Self { + epoch, + _read_guard: Some(read_guard), + } + } + + pub(crate) fn epoch(&self) -> u64 { + self.epoch + } } #[async_trait::async_trait] @@ -599,6 +701,28 @@ impl ScannerConfigObjectDelete for ECStore { ) -> EcstoreResult { ObjectOperations::delete_object(self, bucket, object, opts).await } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?; + Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch)) + } +} + +#[async_trait::async_trait] +impl ScannerConfigObjectDelete for SetDisks { + async fn delete_config_object( + &self, + bucket: &str, + object: &str, + opts: ScannerObjectOptions, + ) -> EcstoreResult { + ObjectOperations::delete_object(self, bucket, object, opts).await + } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + let (read_guard, epoch) = self.scanner_data_usage_publication_admission_guard().await?; + Some(ScannerDataUsagePublicationAdmission::fenced(read_guard, epoch)) + } } #[cfg(test)] diff --git a/crates/scanner/src/remote_scanner/stream.rs b/crates/scanner/src/remote_scanner/stream.rs index 93ba33100..ef6418b9c 100644 --- a/crates/scanner/src/remote_scanner/stream.rs +++ b/crates/scanner/src/remote_scanner/stream.rs @@ -23,6 +23,7 @@ use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION; use crate::{ DATA_USAGE_CACHE_NAME, DataUsageCache, DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntryInfo, DataUsageScanPlanDigest, Disk, ScannerError, StorageError, resolve_scanner_object_store_handle, + scanner_publication_admission_for_epoch, scanner_publication_epoch, }; use hmac::{Hmac, KeyInit, Mac}; use rustfs_common::heal_channel::HealScanMode; @@ -686,6 +687,9 @@ async fn scan_and_persist_local_bucket( source.pool_index, source.set_index )) })?; + let expected_publication_epoch = scanner_publication_epoch(set.clone()).await.ok_or_else(|| { + RemoteScannerServerError::worker("remote namespace scanner cache publication is blocked by data movement") + })?; let cache_name = path_join_buf(&[&bucket, DATA_USAGE_CACHE_NAME]); let guard = acquire_scanner_cache_locks(set.as_ref(), &cache_name, source) .await @@ -708,6 +712,14 @@ async fn scan_and_persist_local_bucket( "remote namespace scanner cache lock was lost before reusing the current snapshot", )); } + if scanner_publication_admission_for_epoch(set.clone(), expected_publication_epoch) + .await + .is_none() + { + return Err(RemoteScannerServerError::retry_bucket( + "remote namespace scanner cache publication epoch changed before reusing the current snapshot", + )); + } return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { source, scan_plan_digest, @@ -796,9 +808,22 @@ async fn scan_and_persist_local_bucket( .await .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?; let done_save = Metrics::time(Metric::SaveUsage); - let save_result = cache.save_with_revisions(set, &cache_name, &revisions).await; + // Each physical main/backup PUT must still prove the epoch captured before + // the scan. A movement transition that starts and ends during the scan + // therefore cannot admit the stale cache under the new epoch. + let save_result = cache + .save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, expected_publication_epoch) + .await; done_save(); save_result.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache save failed: {err}")))?; + if scanner_publication_admission_for_epoch(set, expected_publication_epoch) + .await + .is_none() + { + return Err(RemoteScannerServerError::retry_bucket( + "remote namespace scanner cache publication epoch changed after persistence", + )); + } validate_remote_scanner_request_fence_with_store(next_cycle, leader_epoch, store) .await .map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner leader fence changed: {err}")))?; diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index f7a043308..c44ea8ae0 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -18,6 +18,7 @@ use std::future::Future; use std::sync::Mutex as StdMutex; use std::sync::{Arc, LazyLock, RwLock}; +use self::heal_info::{BackgroundHealInfoReadStatus, read_background_heal_info_with_epoch, save_background_heal_info_for_epoch}; use crate::data_usage_define::{ BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision, @@ -30,8 +31,9 @@ use crate::runtime_config::{ use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason}; use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob}; use crate::scanner_io::{ - ScannerCycleDeferReason, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending, - dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation, + ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, + dirty_usage_buckets_pending, dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, + scanner_maintenance_generation, }; use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed}; use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard}; @@ -66,10 +68,12 @@ use crate::storage_api::scan::{ SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, }; use crate::{ - ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, - get_lifecycle_config, get_replication_config, invalidate_admin_data_usage_snapshot_cache, - invalidate_data_usage_snapshot_cache, read_config, replace_bucket_usage_memory_from_info, save_config, - save_config_shared_with_preconditions, save_config_with_preconditions, scanner_is_erasure_sd, + ECStore, EcstoreError, RUSTFS_META_BUCKET, SCANNER_PUBLICATION_EPOCH_CHANGED, ScannerLifecycleConfigExt as _, + ScannerReplicationConfigExt as _, delete_config_with_publication_admission_for_epoch, get_lifecycle_config, + get_replication_config, invalidate_admin_data_usage_snapshot_cache, invalidate_data_usage_snapshot_cache, read_config, + replace_bucket_usage_memory_from_info, save_config, save_config_shared_with_preconditions, save_config_with_preconditions, + save_config_with_publication_admission_for_epoch, scanner_is_erasure_sd, scanner_publication_admission_for_epoch, + scanner_publication_epoch, scanner_publication_epoch_changed, }; const LOG_COMPONENT_SCANNER: &str = "scanner"; @@ -346,6 +350,23 @@ fn data_usage_info_is_cold(info: &DataUsageInfo) -> bool { !info.is_complete_bucket_usage_snapshot() } +pub(super) fn data_usage_info_has_persisted_baseline_identity(info: &DataUsageInfo) -> bool { + if info.is_complete_bucket_usage_snapshot() { + return true; + } + + // Pre-marker snapshots remain readable only when their legacy identity is + // complete: a timestamp, a scanner cycle, and an exact bucket cardinality. + // A current snapshot with only scanner_epoch/scanner_cycle (or an explicit + // incomplete marker) is not evidence of a durable usage baseline. + !info.usage_snapshot_complete + && info.scanner_epoch.is_none() + && info.usage_snapshot_converged != Some(false) + && info.last_update.is_some() + && info.scanner_cycle.is_some() + && u64::try_from(info.buckets_usage.len()).ok() == Some(info.buckets_count) +} + fn usage_cache_needs_prompt_scan(authoritative: &DataUsageInfo, observed: Option<&DataUsageInfo>) -> bool { data_usage_info_is_cold(authoritative) || observed.is_some_and(|observed| observed_data_usage_is_newer(observed, authoritative)) @@ -381,9 +402,18 @@ fn data_usage_backup_due(data_usage_info: &DataUsageInfo) -> bool { .is_some_and(|cycle| cycle % DATA_USAGE_BACKUP_INTERVAL_CYCLES == 0) } +#[cfg(test)] async fn sync_data_usage_backup_from_primary( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, +) -> Result<(), EcstoreError> { + sync_data_usage_backup_from_primary_for_epoch(ctx, storeapi, None).await +} + +async fn sync_data_usage_backup_from_primary_for_epoch( + ctx: &CancellationToken, + storeapi: Arc, + expected_publication_epoch: Option, ) -> Result<(), EcstoreError> { let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { @@ -391,26 +421,62 @@ async fn sync_data_usage_backup_from_primary( return Ok(()); } + let read_epoch = match expected_publication_epoch { + Some(expected_epoch) => { + if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + } + expected_epoch + } + None => scanner_publication_epoch(storeapi.clone()) + .await + .ok_or_else(|| EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED))?, + }; let (primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?; - let primary = primary.ok_or_else(|| EcstoreError::other("authoritative data usage snapshot is missing"))?; - serde_json::from_slice::(&primary) + let primary = primary.ok_or(EcstoreError::ConfigNotFound)?; + let primary_info = serde_json::from_slice::(&primary) .map_err(|err| EcstoreError::other(format!("authoritative data usage snapshot is invalid: {err}")))?; + if !data_usage_info_has_persisted_baseline_identity(&primary_info) { + return Err(EcstoreError::other( + "authoritative data usage snapshot has no persisted baseline identity", + )); + } let primary = Bytes::from(primary); let (backup, revision) = read_config_with_revision(storeapi.clone(), &backup_path).await?; if backup.as_deref() == Some(primary.as_ref()) { - return Ok(()); + if scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch) + .await + .is_some() + { + return Ok(()); + } + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); } let sha256hex = Some(hex_simd::encode_to_string(Sha256::digest(&primary), hex_simd::AsciiCase::Lower)); - let save_result = save_config_shared_with_preconditions( - storeapi.clone(), - &backup_path, - primary.clone(), - sha256hex, - revision.preconditions(), - ) - .await; + let save_result = { + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + }; + save_config_shared_with_preconditions( + storeapi.clone(), + &backup_path, + primary.clone(), + sha256hex, + revision.preconditions(), + ) + .await + }; match save_result { Ok(_) => {} @@ -427,9 +493,20 @@ async fn sync_data_usage_backup_from_primary( } let (current_primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await?; - if current_primary.as_deref() == Some(primary.as_ref()) { + if current_primary.as_deref() == Some(primary.as_ref()) + && scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch) + .await + .is_some() + { return Ok(()); } + if expected_publication_epoch.is_some() + && scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch) + .await + .is_none() + { + return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED)); + } if retry < SCANNER_PERSIST_CAS_RETRIES { continue; } @@ -1052,7 +1129,7 @@ async fn fence_scanner_epoch_after_cycle_timeout( lock_lost: LockLost, ) -> bool where - Store: ScannerObjectIO, + Store: ScannerObjectIO + ScannerConfigObjectDelete, LockLost: Future, { let fence_ctx = ctx.child_token(); @@ -1089,7 +1166,7 @@ async fn handle_scanner_cycle_deadline( worker_stopped: bool, guard: &mut NamespaceLockGuard, ) where - Store: ScannerObjectIO, + Store: ScannerObjectIO + ScannerConfigObjectDelete, { let fenced = fence_scanner_epoch_after_cycle_timeout( ctx, @@ -1182,7 +1259,33 @@ async fn run_data_scanner_cycle_with_budget( let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await; - let mut background_heal_info = read_background_heal_info(storeapi.clone()).await; + // Refresh the storage-owned movement snapshot before reading background + // heal state. A missing heal object yields an in-memory default; do not + // let that default influence a cycle while publication is blocked. + if storeapi.scanner_data_usage_publication_blocked().await { + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } + let background_heal_read = read_background_heal_info_with_epoch(storeapi.clone()).await; + match background_heal_read.status { + BackgroundHealInfoReadStatus::Blocked => { + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } + BackgroundHealInfoReadStatus::Transient => { + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable); + } + BackgroundHealInfoReadStatus::Failed => { + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Failed; + } + BackgroundHealInfoReadStatus::ErasureSd + | BackgroundHealInfoReadStatus::Loaded + | BackgroundHealInfoReadStatus::Missing => {} + } + let mut background_heal_info = background_heal_read.info; + let background_heal_epoch = background_heal_read.expected_epoch; let scan_mode = get_cycle_scan_mode( cycle_info.current, @@ -1209,11 +1312,23 @@ async fn run_data_scanner_cycle_with_budget( configured_bitrot_cycle, ) { background_heal_info = new_heal_info.clone(); - save_background_heal_info(storeapi.clone(), new_heal_info).await; + save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await; } let cycle_start = std::time::Instant::now(); - let usage_persist_baseline = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { + // Baseline reads are part of the same publication proof as the eventual + // scanner aggregate. Hold only the short storage-owned admission guard + // across this metadata read; the full bucket scan runs after it is + // released and carries the captured epoch forward. + let Some((baseline_publication_guard, baseline_publication_epoch)) = + storeapi.scanner_data_usage_publication_admission_guard().await + else { + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + }; + let usage_persist_baseline_result = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await; + drop(baseline_publication_guard); + let usage_persist_baseline = match usage_persist_baseline_result { Ok((data, revision)) => DataUsagePersistBaseline { data: data.map(Bytes::from), revision, @@ -1250,9 +1365,17 @@ async fn run_data_scanner_cycle_with_budget( ) .await; let publication_defer_reason = match &scan_result { + Ok(result) + if result + .publication_epoch() + .is_some_and(|publication_epoch| publication_epoch != baseline_publication_epoch) => + { + Some(ScannerCycleDeferReason::DataMovement) + } Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await, Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable), }; + let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch); let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled(); let usage_persist_outcome = match publication_defer_reason { Some(reason) => { @@ -1267,12 +1390,13 @@ async fn run_data_scanner_cycle_with_budget( let ctx_clone = ctx.clone(); let route_probe_store = storeapi.clone(); let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move { - store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe( + store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch( ctx_clone, storeapi_clone, receiver, Some(leader_epoch), Some(usage_persist_baseline), + publication_epoch, move || { let storeapi = route_probe_store.clone(); async move { storeapi.scanner_data_usage_publication_blocked().await } @@ -1344,7 +1468,7 @@ async fn run_data_scanner_cycle_with_budget( if !ctx.is_cancelled() && let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, false) { - save_background_heal_info(storeapi.clone(), new_heal_info).await; + save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await; } mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; return ScannerCycleOutcome::Failed; @@ -1377,19 +1501,29 @@ async fn run_data_scanner_cycle_with_budget( "Scanner cycle is recovering to a newer durable cache generation" ); emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); - let persisted = persist_required_scanner_cycle_floor( + let persisted = persist_required_scanner_cycle_floor_for_epoch( ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, - required_cycle, &mut cycle_metrics_guard, + ScannerCycleFloorOptions { + required_cycle, + expected_publication_epoch: publication_epoch, + }, ) .await; return if persisted { cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial + } else if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + emit_scan_cycle_deferred(cycle_start.elapsed()); + ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement) } else { ScannerCycleOutcome::Failed }; @@ -1446,18 +1580,26 @@ async fn run_data_scanner_cycle_with_budget( scan_cycle_partial_reason(budget_reason), scan_cycle_partial_source(budget_reason), ); - let persisted = finalize_partial_scan_cycle( + let persisted = finalize_partial_scan_cycle_for_epoch( ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, &mut cycle_metrics_guard, + publication_epoch, ) .await; return if persisted { cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial + } else if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + emit_scan_cycle_deferred(cycle_start.elapsed()); + ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement) } else { ScannerCycleOutcome::Failed }; @@ -1531,18 +1673,26 @@ async fn run_data_scanner_cycle_with_budget( ); } emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); - let persisted = finalize_partial_scan_cycle( + let persisted = finalize_partial_scan_cycle_for_epoch( ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, &mut cycle_metrics_guard, + publication_epoch, ) .await; return if persisted { cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial + } else if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + emit_scan_cycle_deferred(cycle_start.elapsed()); + ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement) } else { ScannerCycleOutcome::Failed }; @@ -1572,13 +1722,14 @@ async fn run_data_scanner_cycle_with_budget( state = "superseded", "Scanner cycle usage snapshot was superseded by concurrent namespace activity" ); - if finalize_partial_scan_cycle( + if finalize_partial_scan_cycle_for_epoch( ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, &mut cycle_metrics_guard, + publication_epoch, ) .await { @@ -1586,11 +1737,29 @@ async fn run_data_scanner_cycle_with_budget( emit_scan_cycle_superseded(cycle_start.elapsed()); return ScannerCycleOutcome::Superseded; } + if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + emit_scan_cycle_deferred(cycle_start.elapsed()); + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } emit_scan_cycle_complete(false, cycle_start.elapsed()); return ScannerCycleOutcome::Failed; } ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => {} } + if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + emit_scan_cycle_deferred(cycle_start.elapsed()); + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } + let previous_cycle_info = cycle_info.clone(); if let Err(err) = advance_scanner_cycle(cycle_info) { error!( target: "rustfs::scanner", @@ -1610,7 +1779,19 @@ async fn run_data_scanner_cycle_with_budget( global_metrics().clear_current_scan_mode(); retain_recent_cycle_completions(&mut cycle_info.cycle_completed); - if !persist_scanner_cycle_state(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await { + if !persist_scanner_cycle_state_for_epoch(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch, publication_epoch) + .await + { + if let Some(expected_epoch) = publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + *cycle_info = previous_cycle_info; + emit_scan_cycle_deferred(cycle_start.elapsed()); + mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; + return ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } cycle_metrics_guard.finish(cycle_info.clone()).await; emit_scan_cycle_complete(false, cycle_start.elapsed()); return ScannerCycleOutcome::Failed; @@ -1620,7 +1801,7 @@ async fn run_data_scanner_cycle_with_budget( done_cycle(); emit_scan_cycle_complete(true, cycle_start.elapsed()); if let Some(new_heal_info) = background_heal_info_for_scan_result(background_heal_info.clone(), scan_mode, true) { - save_background_heal_info(storeapi.clone(), new_heal_info).await; + save_background_heal_info_for_epoch(storeapi.clone(), new_heal_info, background_heal_epoch).await; } info!( diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index 6459ae6e5..db2e46ff8 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -162,6 +162,8 @@ pub(crate) enum ScannerCycleStateStartup { enum CycleRecoveryMarkerReadError { #[error("cycle recovery marker backend read failed: {0}")] Backend(#[source] EcstoreError), + #[error("cycle recovery marker publication is blocked by data movement")] + PublicationBlocked, #[error("invalid cycle recovery marker: {0}")] Invalid(&'static str), #[error("cycle recovery marker revision changed while publishing")] @@ -326,12 +328,13 @@ fn cycle_state_generation_and_epoch(buf: &[u8]) -> (u64, u64) { } async fn persist_cycle_recovery_marker( - storeapi: Arc, + storeapi: Arc, primary_revision: &DataUsageCacheRevision, generation: u64, leader_epoch: u64, classification: &'static str, reason: &'static str, + expected_epoch: u64, ) -> Result { let now = unix_now_secs(); let (existing, existing_revision) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { @@ -370,6 +373,9 @@ async fn persist_cycle_recovery_marker( state: "blocked".to_string(), }; let bytes = serde_json::to_vec(&marker).map_err(|_| CycleRecoveryMarkerReadError::Invalid("marker serialization failed"))?; + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else { + return Err(CycleRecoveryMarkerReadError::PublicationBlocked); + }; let save_result = save_config_with_preconditions( storeapi.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), @@ -478,7 +484,7 @@ async fn read_cycle_recovery_marker_revision( } async fn quarantine_invalid_cycle_state( - storeapi: Arc, + storeapi: Arc, revision: &DataUsageCacheRevision, buf: &[u8], ) -> ScannerCycleStateStartup { @@ -488,7 +494,7 @@ async fn quarantine_invalid_cycle_state( } async fn quarantine_invalid_cycle_state_with_reason( - storeapi: Arc, + storeapi: Arc, revision: &DataUsageCacheRevision, generation: u64, leader_epoch: u64, @@ -515,7 +521,19 @@ async fn quarantine_invalid_cycle_state_with_reason( reason: Some(reason.to_string()), }; set_scanner_cycle_recovery_status(base_status); - match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason).await { + let Some(expected_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + set_scanner_cycle_recovery_status(recovery_status( + "transient", + Some("cycle recovery marker publication is blocked by data movement"), + true, + )); + return ScannerCycleStateStartup::Transient(ScannerError::Other( + "cycle recovery marker publication is blocked by data movement".to_string(), + )); + }; + match persist_cycle_recovery_marker(storeapi, revision, generation, leader_epoch, classification, reason, expected_epoch) + .await + { Ok(marker) => set_scanner_cycle_recovery_status(recovery_status_from_marker(&marker, "blocked")), Err(CycleRecoveryMarkerReadError::Backend(_)) => { // Keep the poison object untouched and retry marker creation with the @@ -524,6 +542,16 @@ async fn quarantine_invalid_cycle_state_with_reason( "failed to persist scanner cycle recovery marker".to_string(), )); } + Err(CycleRecoveryMarkerReadError::PublicationBlocked) => { + set_scanner_cycle_recovery_status(recovery_status( + "transient", + Some("cycle recovery marker publication is blocked by data movement"), + true, + )); + return ScannerCycleStateStartup::Transient(ScannerError::Other( + "cycle recovery marker publication is blocked by data movement".to_string(), + )); + } Err(CycleRecoveryMarkerReadError::Conflict) => { set_scanner_cycle_recovery_status(recovery_status( "transient", @@ -546,16 +574,18 @@ async fn mark_cycle_recovery_cleanup_pending( storeapi: Arc, mut marker: ScannerCycleRecoveryMarker, marker_revision: &DataUsageCacheRevision, + expected_epoch: u64, ) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> { marker.state = "cleanup-pending".to_string(); marker.last_attempt_at_unix_secs = unix_now_secs(); let bytes = serde_json::to_vec(&marker) .map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?; - let info = save_config_with_preconditions( + let info = save_config_with_publication_admission_for_epoch( storeapi.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), bytes, marker_revision.preconditions(), + expected_epoch, ) .await .map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?; @@ -567,7 +597,9 @@ async fn mark_cycle_recovery_cleanup_pending( Ok((marker, revision)) } -pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc) -> ScannerCycleStateStartup { +pub(crate) async fn load_scanner_cycle_state_for_startup( + storeapi: Arc, +) -> ScannerCycleStateStartup { let marker = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { Ok((None, _)) => None, Ok((Some(data), marker_revision)) => match serde_json::from_slice::(&data) { @@ -594,6 +626,16 @@ pub(crate) async fn load_scanner_cycle_state_for_startup(storeapi: Arc { + set_scanner_cycle_recovery_status(recovery_status( + "transient", + Some("cycle recovery marker publication is blocked by data movement"), + true, + )); + return ScannerCycleStateStartup::Transient(ScannerError::Other( + "cycle recovery marker publication is blocked by data movement".to_string(), + )); + } Err(CycleRecoveryMarkerReadError::Invalid(reason)) => { set_scanner_cycle_recovery_status(recovery_status("recovery-required", Some(reason), false)); return ScannerCycleStateStartup::Blocked; @@ -750,6 +792,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string())); } + let Some(reset_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + return Err(ScannerError::Other("scanner recovery reset is blocked by data movement".to_string())); + }; + let (marker_data, marker_revision, marker_body_invalid) = match read_cycle_recovery_marker_bytes(storeapi.clone()).await { Ok((marker_data, marker_revision)) => (marker_data, marker_revision, false), Err(CycleRecoveryMarkerReadError::Invalid(_)) => { @@ -835,7 +881,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< }; if let Some((primary_cycle, primary_epoch)) = primary_state { let (cleanup_marker, cleanup_marker_revision) = - mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision).await?; + mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?; set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending")); let usage_floor = persisted_usage_floor(storeapi.clone()).await?; let fence_epoch = primary_epoch @@ -855,14 +901,21 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "preserved scanner cycle state exceeds the bounded object size".to_string(), )); } - let preserved_info = save_config_with_preconditions( + let preserved_info = save_config_with_publication_admission_for_epoch( storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), preserved_data, primary_revision.preconditions(), + reset_epoch, ) .await - .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}")))?; + .map_err(|err| { + if scanner_publication_epoch_changed(&err) { + ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string()) + } else { + ScannerError::Other(format!("failed to fence preserved scanner cycle state: {err}")) + } + })?; let preserved_revision = preserved_info .etag .filter(|etag| !etag.is_empty()) @@ -873,7 +926,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner leader lock was lost after fencing newer cycle state".to_string(), )); } - fence_scanner_usage_epoch(&ctx, storeapi.clone(), fence_epoch) + fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch)) .await .map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?; if guard.is_lock_lost() { @@ -889,20 +942,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner cycle state changed before recovery marker cleanup".to_string(), )); } - storeapi - .delete_config_object( - RUSTFS_META_BUCKET, - DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), - ScannerObjectOptions { - // This is one exact metadata object. Prefix-delete mode - // bypasses HTTP preconditions in the ECStore path. - delete_prefix: false, - http_preconditions: Some(cleanup_marker_revision.preconditions()), - ..Default::default() - }, - ) - .await - .map_err(|err| ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}")))?; + delete_config_with_publication_admission_for_epoch( + storeapi.clone(), + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + ScannerObjectOptions { + // This is one exact metadata object. Prefix-delete mode + // bypasses HTTP preconditions in the ECStore path. + delete_prefix: false, + http_preconditions: Some(cleanup_marker_revision.preconditions()), + ..Default::default() + }, + reset_epoch, + ) + .await + .map_err(|err| { + if scanner_publication_epoch_changed(&err) { + ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string()) + } else { + ScannerError::Other(format!("failed to clear stale cycle recovery marker: {err}")) + } + })?; set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); super::notify_scanner_cycle_recovery_wake(); return Ok(()); @@ -946,16 +1006,23 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< let (marker, marker_revision) = if marker.state == "cleanup-pending" { (marker, marker_revision) } else { - mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision).await? + mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await? }; - let rebuilt_info = save_config_with_preconditions( + let rebuilt_info = save_config_with_publication_admission_for_epoch( storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), data, primary_revision.preconditions(), + reset_epoch, ) .await - .map_err(|err| ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}")))?; + .map_err(|err| { + if scanner_publication_epoch_changed(&err) { + ScannerError::Other("scanner recovery reset deferred by a movement epoch change".to_string()) + } else { + ScannerError::Other(format!("failed to persist rebuilt scanner cycle state: {err}")) + } + })?; let rebuilt_revision = rebuilt_info .etag .filter(|etag| !etag.is_empty()) @@ -965,7 +1032,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< "scanner leader lock was lost after rebuilding cycle state".to_string(), )); } - if let Err(err) = fence_scanner_usage_epoch(&ctx, storeapi.clone(), leader_epoch).await { + if let Err(err) = fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch)).await + { set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { path: DATA_USAGE_BLOOM_NAME_PATH.clone(), quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), @@ -1034,20 +1102,40 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc< )); } - if let Err(err) = storeapi - .delete_config_object( - RUSTFS_META_BUCKET, - DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), - ScannerObjectOptions { - // This is one exact metadata object. Prefix-delete mode - // bypasses HTTP preconditions in the ECStore path. - delete_prefix: false, - http_preconditions: Some(marker_revision.preconditions()), - ..Default::default() - }, - ) - .await + if let Err(err) = delete_config_with_publication_admission_for_epoch( + storeapi.clone(), + RUSTFS_META_BUCKET, + DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), + ScannerObjectOptions { + // This is one exact metadata object. Prefix-delete mode + // bypasses HTTP preconditions in the ECStore path. + delete_prefix: false, + http_preconditions: Some(marker_revision.preconditions()), + ..Default::default() + }, + reset_epoch, + ) + .await { + if scanner_publication_epoch_changed(&err) { + set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), + state: "cleanup-pending".to_string(), + classification: Some(marker.classification.clone()), + primary_revision: Some(rebuilt_revision.clone()), + generation: Some(next), + leader_epoch: Some(leader_epoch), + retry_count: marker.retry_count, + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable: true, + reason: Some("movement epoch changed before recovery marker cleanup".to_string()), + ..Default::default() + }); + return Err(ScannerError::Other( + "scanner recovery reset deferred by a movement epoch change".to_string(), + )); + } set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { path: DATA_USAGE_BLOOM_NAME_PATH.clone(), quarantine_path: Some(DATA_USAGE_BLOOM_RECOVERY_PATH.clone()), @@ -1209,8 +1297,14 @@ pub(super) fn advance_scanner_cycle(cycle_info: &mut CurrentCycle) -> Result<(), Ok(()) } -pub(super) async fn persisted_usage_floor(storeapi: Arc) -> Result { +pub(super) async fn persisted_usage_floor( + storeapi: Arc, +) -> Result { + let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + return Err(ScannerError::Other("scanner usage floor read is blocked by data movement".to_string())); + }; let mut floor = PersistedUsageFloor::default(); + let mut found_any = false; let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> { floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default()); if let Some(completed_cycle) = usage.scanner_cycle { @@ -1229,6 +1323,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc) - let usage = serde_json::from_slice::(&data).map_err(|err| { ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}")) })?; + if !data_usage_info_has_persisted_baseline_identity(&usage) { + return Err(ScannerError::Other(format!( + "scanner usage floor from {primary_path} has no persisted baseline identity" + ))); + } let epoch = usage.scanner_epoch.unwrap_or_default(); update_floor(&mut floor, &usage, primary_path)?; Some(epoch) @@ -1247,6 +1346,11 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc) - let usage = serde_json::from_slice::(&data).map_err(|err| { ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}")) })?; + if !data_usage_info_has_persisted_baseline_identity(&usage) { + return Err(ScannerError::Other(format!( + "scanner usage floor from {backup_path} has no persisted baseline identity" + ))); + } let backup_epoch = usage.scanner_epoch.unwrap_or_default(); // A backup write from an older leader may complete after the // primary epoch has been fenced. It must not advance the startup @@ -1263,9 +1367,21 @@ pub(super) async fn persisted_usage_floor(storeapi: Arc) - } } if any_found { + found_any = true; break; } } + + if !found_any { + return Err(ScannerError::Other( + "persisted scanner usage floor has no authoritative baseline".to_string(), + )); + } + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi, read_epoch).await else { + return Err(ScannerError::Other( + "scanner usage floor changed while its epoch proof was being confirmed".to_string(), + )); + }; Ok(floor) } @@ -1274,12 +1390,30 @@ pub(super) fn apply_persisted_usage_floor(cycle_info: &mut CurrentCycle, leader_ *leader_epoch = (*leader_epoch).max(floor.leader_epoch); } +#[derive(Clone, Copy)] +pub(super) struct ScannerCycleFloorOptions { + pub(super) required_cycle: u64, + pub(super) expected_publication_epoch: Option, +} + +#[cfg(test)] pub(super) async fn persist_scanner_cycle_state( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, cycle_info: &mut CurrentCycle, revision: &mut DataUsageCacheRevision, leader_epoch: u64, +) -> bool { + persist_scanner_cycle_state_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, None).await +} + +pub(super) async fn persist_scanner_cycle_state_for_epoch( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + expected_publication_epoch: Option, ) -> bool { let buf = match encode_scanner_cycle_state(cycle_info, leader_epoch) { Ok(buf) => buf, @@ -1315,9 +1449,29 @@ pub(super) async fn persist_scanner_cycle_state( #[cfg(test)] notify_scanner_cycle_state_persist_test_hook(leader_epoch); - match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions()) - .await - { + let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + return false; + }; + if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) { + return false; + } + let save_result = { + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %&*DATA_USAGE_BLOOM_NAME_PATH, + state = "publication_admission_unavailable", + "Scanner state persistence skipped without movement admission" + ); + return false; + }; + save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions()) + .await + }; + match save_result { Ok(object_info) => { let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) else { error!( @@ -1345,6 +1499,13 @@ pub(super) async fn persist_scanner_cycle_state( ); return false; } + if let Some(expected_epoch) = expected_publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + return false; + } debug!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, @@ -1436,6 +1597,13 @@ pub(super) async fn persist_scanner_cycle_state( } if persisted_cycle.next >= cycle_info.next { + if let Some(expected_epoch) = expected_publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + return false; + } *cycle_info = persisted_cycle; debug!( target: "rustfs::scanner", @@ -1496,19 +1664,33 @@ pub(super) async fn persist_scanner_cycle_state( false } +#[cfg(test)] pub(super) async fn finalize_partial_scan_cycle( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, cycle_info: &mut CurrentCycle, revision: &mut DataUsageCacheRevision, leader_epoch: u64, cycle_metrics_guard: &mut ScannerCycleMetricsGuard, +) -> bool { + finalize_partial_scan_cycle_for_epoch(ctx, storeapi, cycle_info, revision, leader_epoch, cycle_metrics_guard, None).await +} + +pub(super) async fn finalize_partial_scan_cycle_for_epoch( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + cycle_metrics_guard: &mut ScannerCycleMetricsGuard, + expected_publication_epoch: Option, ) -> bool { // A budget-limited cycle is deliberate pacing, not a failure. The cycle counter // must still advance (and persist) because per-bucket next_cycle is stamped from // it and compacted folders are only rescanned when their hash matches // next_cycle % DATA_USAGE_UPDATE_DIR_CYCLES; a pinned counter starves lifecycle // expiry and usage refresh on every folder outside the stuck window. + let previous_cycle_info = cycle_info.clone(); if let Err(err) = advance_scanner_cycle(cycle_info) { error!( target: "rustfs::scanner", @@ -1524,28 +1706,69 @@ pub(super) async fn finalize_partial_scan_cycle( } cycle_info.current = 0; global_metrics().clear_current_scan_mode(); - let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await; + let persisted = persist_scanner_cycle_state_for_epoch( + ctx, + storeapi.clone(), + cycle_info, + revision, + leader_epoch, + expected_publication_epoch, + ) + .await; + if !persisted + && let Some(expected_epoch) = expected_publication_epoch + && scanner_publication_admission_for_epoch(storeapi, expected_epoch) + .await + .is_none() + { + *cycle_info = previous_cycle_info; + } cycle_metrics_guard.finish(cycle_info.clone()).await; persisted } +#[cfg(test)] pub(super) async fn persist_required_scanner_cycle_floor( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, cycle_info: &mut CurrentCycle, revision: &mut DataUsageCacheRevision, leader_epoch: u64, required_cycle: u64, cycle_metrics_guard: &mut ScannerCycleMetricsGuard, ) -> bool { - if required_cycle <= cycle_info.current || required_cycle == u64::MAX { + persist_required_scanner_cycle_floor_for_epoch( + ctx, + storeapi, + cycle_info, + revision, + leader_epoch, + cycle_metrics_guard, + ScannerCycleFloorOptions { + required_cycle, + expected_publication_epoch: None, + }, + ) + .await +} + +pub(super) async fn persist_required_scanner_cycle_floor_for_epoch( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + cycle_metrics_guard: &mut ScannerCycleMetricsGuard, + options: ScannerCycleFloorOptions, +) -> bool { + if options.required_cycle <= cycle_info.current || options.required_cycle == u64::MAX { error!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, current_cycle = cycle_info.current, - required_cycle, + required_cycle = options.required_cycle, state = "invalid_cache_cycle_floor", "Scanner cache cycle floor is invalid" ); @@ -1553,10 +1776,27 @@ pub(super) async fn persist_required_scanner_cycle_floor( return false; } - cycle_info.next = cycle_info.next.max(required_cycle); + let previous_cycle_info = cycle_info.clone(); + cycle_info.next = cycle_info.next.max(options.required_cycle); cycle_info.current = 0; global_metrics().clear_current_scan_mode(); - let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await; + let persisted = persist_scanner_cycle_state_for_epoch( + ctx, + storeapi.clone(), + cycle_info, + revision, + leader_epoch, + options.expected_publication_epoch, + ) + .await; + if !persisted + && let Some(expected_epoch) = options.expected_publication_epoch + && scanner_publication_admission_for_epoch(storeapi, expected_epoch) + .await + .is_none() + { + *cycle_info = previous_cycle_info; + } cycle_metrics_guard.finish(cycle_info.clone()).await; persisted } diff --git a/crates/scanner/src/scanner/heal_info.rs b/crates/scanner/src/scanner/heal_info.rs index 18d0511f4..55e2a0dba 100644 --- a/crates/scanner/src/scanner/heal_info.rs +++ b/crates/scanner/src/scanner/heal_info.rs @@ -26,31 +26,90 @@ pub struct BackgroundHealInfo { pub current_scan_mode: HealScanMode, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum BackgroundHealInfoReadStatus { + ErasureSd, + Loaded, + Missing, + Blocked, + Transient, + Failed, +} + +pub(super) struct BackgroundHealInfoRead { + pub(super) info: BackgroundHealInfo, + pub(super) expected_epoch: Option, + pub(super) status: BackgroundHealInfoReadStatus, +} + +pub(super) fn classify_background_heal_read_error(error: &EcstoreError) -> BackgroundHealInfoReadStatus { + if matches!(error, EcstoreError::ConfigNotFound) { + BackgroundHealInfoReadStatus::Missing + } else { + BackgroundHealInfoReadStatus::Transient + } +} + +pub(super) fn decode_background_heal_info(data: &[u8]) -> Result { + serde_json::from_slice(data) +} + /// Read background healing information from storage pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHealInfo { + read_background_heal_info_with_epoch(storeapi).await.info +} + +/// Read background healing information together with the movement epoch that +/// fenced the read. The epoch must be reused by the matching cycle update so a +/// missing-object default cannot be committed across a movement transition. +pub(super) async fn read_background_heal_info_with_epoch(storeapi: Arc) -> BackgroundHealInfoRead { // Skip for ErasureSD setup if scanner_is_erasure_sd().await { - return BackgroundHealInfo::default(); + return BackgroundHealInfoRead { + info: BackgroundHealInfo::default(), + expected_epoch: None, + status: BackgroundHealInfoReadStatus::ErasureSd, + }; + } + + let expected_epoch = scanner_publication_epoch(storeapi.clone()).await; + if expected_epoch.is_none() { + return BackgroundHealInfoRead { + info: BackgroundHealInfo::default(), + expected_epoch, + status: BackgroundHealInfoReadStatus::Blocked, + }; } // Get last healing information match read_config(storeapi, &BACKGROUND_HEAL_INFO_PATH).await { - Ok(buf) => serde_json::from_slice::(&buf).unwrap_or_else(|e| { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, - path = %&*BACKGROUND_HEAL_INFO_PATH, - state = "decode_failed", - error = %e, - "Scanner background heal decode failed" - ); - BackgroundHealInfo::default() - }), + Ok(buf) => match decode_background_heal_info(&buf) { + Ok(info) => BackgroundHealInfoRead { + info, + expected_epoch, + status: BackgroundHealInfoReadStatus::Loaded, + }, + Err(e) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, + path = %&*BACKGROUND_HEAL_INFO_PATH, + state = "decode_failed", + error = %e, + "Scanner background heal decode failed" + ); + BackgroundHealInfoRead { + info: BackgroundHealInfo::default(), + expected_epoch, + status: BackgroundHealInfoReadStatus::Failed, + } + } + }, Err(e) => { - // Only log if it's not a ConfigNotFound error - if e != EcstoreError::ConfigNotFound { + let status = classify_background_heal_read_error(&e); + if status == BackgroundHealInfoReadStatus::Transient { warn!( target: "rustfs::scanner", event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, @@ -62,7 +121,11 @@ pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHeal "Scanner background heal read failed" ); } - BackgroundHealInfo::default() + BackgroundHealInfoRead { + info: BackgroundHealInfo::default(), + expected_epoch, + status, + } } } } @@ -70,6 +133,14 @@ pub async fn read_background_heal_info(storeapi: Arc) -> BackgroundHeal /// Save background healing information to storage #[instrument(skip(storeapi))] pub async fn save_background_heal_info(storeapi: Arc, info: BackgroundHealInfo) { + save_background_heal_info_for_epoch(storeapi, info, None).await; +} + +pub(super) async fn save_background_heal_info_for_epoch( + storeapi: Arc, + info: BackgroundHealInfo, + expected_epoch: Option, +) { // Skip for ErasureSD setup if scanner_is_erasure_sd().await { return; @@ -93,7 +164,25 @@ pub async fn save_background_heal_info(storeapi: Arc, info: BackgroundH } }; - // Save configuration + // Save configuration only after storage-owned movement admission. The + // read path may return an in-memory default for a missing object, but a + // movement transition must not let that default become durable state. + let publication_admission = match expected_epoch { + Some(expected_epoch) => scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await, + None => storeapi.scanner_data_usage_publication_admission().await, + }; + let Some(_publication_admission) = publication_admission else { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_BACKGROUND_HEAL_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_BACKGROUND_HEAL, + path = %&*BACKGROUND_HEAL_INFO_PATH, + state = "publication_admission_unavailable", + "Scanner background heal save skipped without movement admission" + ); + return; + }; if let Err(e) = save_config(storeapi, &BACKGROUND_HEAL_INFO_PATH, data).await { warn!( target: "rustfs::scanner", diff --git a/crates/scanner/src/scanner/leadership.rs b/crates/scanner/src/scanner/leadership.rs index ab22f56d9..8183b7602 100644 --- a/crates/scanner/src/scanner/leadership.rs +++ b/crates/scanner/src/scanner/leadership.rs @@ -61,16 +61,22 @@ pub(super) async fn reconcile_scanner_leadership_claim( } pub(super) fn decode_usage_snapshot_for_epoch_fence(data: &[u8], path: &str) -> Result { - serde_json::from_slice(data) - .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}"))) + let usage: DataUsageInfo = serde_json::from_slice(data) + .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage epoch fence from {path}: {err}")))?; + if !data_usage_info_has_persisted_baseline_identity(&usage) { + return Err(ScannerError::Other(format!( + "scanner usage epoch fence from {path} has no persisted baseline identity" + ))); + } + Ok(usage) } pub(super) async fn usage_snapshot_for_epoch_fence( storeapi: Arc, primary: Option<&[u8]>, -) -> Result { +) -> Result, ScannerError> { if let Some(primary) = primary { - return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()); + return decode_usage_snapshot_for_epoch_fence(primary, DATA_USAGE_OBJ_NAME_PATH.as_str()).map(Some); } let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); @@ -78,7 +84,7 @@ pub(super) async fn usage_snapshot_for_epoch_fence( .await .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence backup: {err}")))?; if let Some(backup) = backup.as_deref() { - return decode_usage_snapshot_for_epoch_fence(backup, &backup_path); + return decode_usage_snapshot_for_epoch_fence(backup, &backup_path).map(Some); } for path in [ @@ -89,26 +95,53 @@ pub(super) async fn usage_snapshot_for_epoch_fence( .await .map_err(|err| ScannerError::Other(format!("failed to read legacy scanner usage epoch fence: {err}")))?; if let Some(legacy) = legacy.as_deref() { - return decode_usage_snapshot_for_epoch_fence(legacy, &path); + return decode_usage_snapshot_for_epoch_fence(legacy, &path).map(Some); } } - Ok(DataUsageInfo::default()) + // A missing usage snapshot is an uninitialized state, not an empty + // snapshot. Leadership fencing may proceed without creating a plausible + // default; the first authoritative scanner publication will create it. + Ok(None) } -pub(super) async fn fence_scanner_usage_epoch( +pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, claimed_epoch: u64, + expected_publication_epoch: Option, ) -> Result<(), ScannerError> { for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { if ctx.is_cancelled() { return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string())); } + let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + return Err(ScannerError::Other( + "scanner usage epoch fence publication is blocked by data movement".to_string(), + )); + }; + if expected_publication_epoch.is_some_and(|expected| expected != read_epoch) { + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return Err(ScannerError::Other( + "scanner usage epoch fence changed while recovery reset was in progress".to_string(), + )); + } let (primary, revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) .await .map_err(|err| ScannerError::Other(format!("failed to read scanner usage epoch fence: {err}")))?; - let mut usage = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await?; + let Some(mut usage) = usage_snapshot_for_epoch_fence(storeapi.clone(), primary.as_deref()).await? else { + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return Err(ScannerError::Other( + "scanner usage epoch fence changed while confirming a missing usage baseline".to_string(), + )); + }; + return Err(ScannerError::Other("authoritative scanner usage baseline is missing".to_string())); + }; match usage.scanner_epoch { Some(epoch) if epoch > claimed_epoch => { return Err(ScannerError::Other(format!( @@ -122,9 +155,18 @@ pub(super) async fn fence_scanner_usage_epoch( let data = serde_json::to_vec(&usage) .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage epoch fence: {err}")))?; - let save_result = + let save_result = { + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return Err(ScannerError::Other( + "scanner usage epoch fence changed while preparing its conditional write".to_string(), + )); + }; save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions()) - .await; + .await + }; if save_result .as_ref() .ok() @@ -165,10 +207,13 @@ pub(super) async fn fence_scanner_usage_epoch( pub(super) async fn complete_scanner_leadership_claim( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, claimed_epoch: u64, + expected_publication_epoch: Option, ) -> bool { - if let Err(err) = fence_scanner_usage_epoch(ctx, storeapi, claimed_epoch).await { + if let Err(err) = + fence_scanner_usage_epoch_with_expected_epoch(ctx, storeapi, claimed_epoch, expected_publication_epoch).await + { error!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, @@ -187,7 +232,7 @@ pub(super) async fn complete_scanner_leadership_claim( pub(super) async fn claim_scanner_leadership( ctx: &CancellationToken, - storeapi: Arc, + storeapi: Arc, cycle_info: &mut CurrentCycle, revision: &mut DataUsageCacheRevision, persisted_epoch: &mut u64, @@ -226,15 +271,69 @@ pub(super) async fn claim_scanner_leadership( }; let previous_revision = revision.clone(); - let save_result = + let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + return false; + }; + let (usage_primary, _) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { + Ok(result) => result, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "leader_usage_baseline_read_failed", + error = %err, + "Scanner leadership claim deferred because the usage baseline could not be read" + ); + return false; + } + }; + match usage_snapshot_for_epoch_fence(storeapi.clone(), usage_primary.as_deref()).await { + Ok(Some(_)) => {} + Ok(None) => { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "leader_usage_baseline_missing", + "Scanner leadership claim deferred until a usage baseline is published" + ); + return false; + } + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "leader_usage_baseline_invalid", + error = %err, + "Scanner leadership claim deferred because the usage baseline is invalid" + ); + return false; + } + } + let save_result = { + let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { + if retry < SCANNER_PERSIST_CAS_RETRIES { + continue; + } + return false; + }; save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, data.clone(), revision.preconditions()) - .await; + .await + }; match save_result { Ok(object_info) => { if let Some(etag) = object_info.etag.filter(|etag| !etag.is_empty()) { *revision = DataUsageCacheRevision::Etag(etag); *persisted_epoch = claimed_epoch; - return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await; } match reconcile_scanner_leadership_claim( @@ -249,7 +348,7 @@ pub(super) async fn claim_scanner_leadership( .await { Ok(ScannerLeadershipClaimReconcile::Durable) => { - return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await; } Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => continue, Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { @@ -293,7 +392,7 @@ pub(super) async fn claim_scanner_leadership( .await { Ok(ScannerLeadershipClaimReconcile::Durable) => { - return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch).await; + return complete_scanner_leadership_claim(ctx, storeapi, claimed_epoch, Some(read_epoch)).await; } Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index d73d63cde..9e1d7cef8 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::heal_info::{classify_background_heal_read_error, decode_background_heal_info}; use super::*; use crate::EcstoreResult; use crate::{ @@ -22,6 +23,7 @@ use crate::{ }; use std::collections::{HashMap, HashSet}; use std::io::Cursor; +use std::sync::atomic::{AtomicBool, Ordering}; use std::task::Poll; use temp_env::{with_var, with_var_unset}; use tokio::io::AsyncReadExt; @@ -343,6 +345,7 @@ struct MemoryConfigStore { cancel_after_successful_puts: Mutex>, replace_after_successful_puts: Mutex)>>, put_counts: Mutex>, + publication_admission_blocked: AtomicBool, } fn memory_config_key(bucket: &str, object: &str) -> String { @@ -1350,7 +1353,7 @@ async fn full_rescan_reset_preserves_valid_primary_when_marker_is_malformed() { let old_usage = DataUsageInfo { scanner_epoch: Some(7), scanner_cycle: Some(41), - ..Default::default() + ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0) }; let old_usage_data = serde_json::to_vec(&old_usage).expect("usage snapshot should encode"); save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), old_usage_data.clone()) @@ -1424,7 +1427,7 @@ async fn full_rescan_reset_resumes_cleanup_pending_preserved_primary() { let usage = DataUsageInfo { scanner_epoch: Some(7), scanner_cycle: Some(41), - ..Default::default() + ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0) }; save_config( store.clone(), @@ -1695,7 +1698,7 @@ async fn full_rescan_reset_rejects_usage_floor_that_would_be_terminal() { DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&DataUsageInfo { scanner_epoch: Some(u64::MAX - 1), - ..Default::default() + ..complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0) }) .expect("usage floor should encode"), ) @@ -1847,14 +1850,12 @@ async fn scanner_startup_uses_primary_and_backup_usage_floor() { let store = Arc::new(MemoryConfigStore::default()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 11, 103)] { + let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + usage.scanner_epoch = Some(epoch); + usage.scanner_cycle = Some(cycle); store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, path), - serde_json::to_vec(&DataUsageInfo { - scanner_epoch: Some(epoch), - scanner_cycle: Some(cycle), - ..Default::default() - }) - .expect("usage snapshot should encode"), + serde_json::to_vec(&usage).expect("usage snapshot should encode"), ); } @@ -1879,14 +1880,12 @@ async fn scanner_usage_floor_ignores_older_backup_after_primary_epoch_fence() { let store = Arc::new(MemoryConfigStore::default()); let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); for (path, epoch, cycle) in [(DATA_USAGE_OBJ_NAME_PATH.as_str(), 8, 100), (backup_path.as_str(), 7, 10_000)] { + let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + usage.scanner_epoch = Some(epoch); + usage.scanner_cycle = Some(cycle); store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, path), - serde_json::to_vec(&DataUsageInfo { - scanner_epoch: Some(epoch), - scanner_cycle: Some(cycle), - ..Default::default() - }) - .expect("usage snapshot should encode"), + serde_json::to_vec(&usage).expect("usage snapshot should encode"), ); } @@ -1916,6 +1915,23 @@ fn scanner_startup_treats_incomplete_usage_snapshot_as_cold() { })); } +#[test] +fn scanner_baseline_identity_requires_complete_or_strict_legacy_shape() { + assert!(!data_usage_info_has_persisted_baseline_identity(&DataUsageInfo { + scanner_epoch: Some(3), + scanner_cycle: Some(7), + ..Default::default() + })); + + let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + legacy.usage_snapshot_complete = false; + legacy.scanner_cycle = Some(7); + assert!(data_usage_info_has_persisted_baseline_identity(&legacy)); + + legacy.scanner_epoch = Some(3); + assert!(!data_usage_info_has_persisted_baseline_identity(&legacy)); +} + #[test] fn scanner_startup_prompts_only_for_a_newer_valid_observation() { let authoritative = DataUsageInfo { @@ -1948,12 +1964,9 @@ fn scanner_startup_prompts_only_for_a_newer_valid_observation() { #[tokio::test] async fn scanner_startup_prefers_v2_over_legacy_usage() { let store = Arc::new(MemoryConfigStore::default()); - let legacy = DataUsageInfo { - scanner_epoch: Some(19), - scanner_cycle: Some(41), - last_update: Some(std::time::SystemTime::now()), - ..Default::default() - }; + let mut legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + legacy.scanner_epoch = Some(19); + legacy.scanner_cycle = Some(41); let legacy_data = serde_json::to_vec(&legacy).expect("legacy usage snapshot should encode"); store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), @@ -1976,13 +1989,9 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() { } ); - let authoritative = DataUsageInfo { - scanner_epoch: Some(23), - scanner_cycle: Some(51), - last_update: Some(std::time::SystemTime::now()), - usage_snapshot_complete: true, - ..Default::default() - }; + let mut authoritative = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + authoritative.scanner_epoch = Some(23); + authoritative.scanner_cycle = Some(51); let authoritative_data = serde_json::to_vec(&authoritative).expect("v2 usage snapshot should encode"); store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), @@ -2024,6 +2033,8 @@ async fn scanner_startup_prefers_v2_over_legacy_usage() { #[tokio::test] async fn scanner_usage_floor_fails_closed_on_corrupt_or_exhausted_usage_state() { let store = Arc::new(MemoryConfigStore::default()); + assert!(persisted_usage_floor(store.clone()).await.is_err()); + store.objects.lock().await.insert( memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), b"not-json".to_vec(), @@ -2087,6 +2098,39 @@ async fn scanner_usage_backup_uses_durable_cycle_cadence_across_tasks() { } } +#[tokio::test] +async fn scanner_backup_sync_distinguishes_movement_from_missing_or_corrupt_primary() { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + store.objects.lock().await.insert( + primary_key.clone(), + serde_json::to_vec(&primary).expect("primary usage snapshot should encode"), + ); + store.revisions.lock().await.insert(primary_key.clone(), 1); + + store.publication_admission_blocked.store(true, Ordering::Release); + let movement_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone()) + .await + .expect_err("movement admission loss should fail backup synchronization"); + assert!(scanner_publication_epoch_changed(&movement_error)); + + store.publication_admission_blocked.store(false, Ordering::Release); + store.objects.lock().await.remove(&primary_key); + store.revisions.lock().await.remove(&primary_key); + assert!(matches!( + sync_data_usage_backup_from_primary(&CancellationToken::new(), store.clone()).await, + Err(EcstoreError::ConfigNotFound) + )); + + store.objects.lock().await.insert(primary_key.clone(), b"not-json".to_vec()); + store.revisions.lock().await.insert(primary_key, 1); + let corrupt_error = sync_data_usage_backup_from_primary(&CancellationToken::new(), store) + .await + .expect_err("corrupt primary should fail backup synchronization"); + assert!(!scanner_publication_epoch_changed(&corrupt_error)); +} + #[async_trait::async_trait] impl crate::ScannerConfigObjectDelete for MemoryConfigStore { async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult { @@ -2110,6 +2154,10 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore { revisions.remove(&key); Ok(ObjectInfo::default()) } + + async fn scanner_data_usage_publication_admission(&self) -> Option { + (!self.publication_admission_blocked.load(Ordering::Acquire)).then(crate::ScannerDataUsagePublicationAdmission::unfenced) + } } #[test] @@ -2270,6 +2318,7 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf started: Utc::now(), }; assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + seed_usage_snapshot_for_leadership_claim(&store).await; let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); let old_epoch_commit = CurrentCycle { @@ -2313,6 +2362,61 @@ async fn test_leadership_claim_rejects_terminal_epoch() { assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); } +#[tokio::test] +async fn leadership_claim_defers_without_usage_baseline_before_bloom_write() { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + next: 12, + ..Default::default() + }; + let mut persisted_epoch = 0; + + assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await); + assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); + assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err()); +} + +#[tokio::test] +async fn leadership_claim_defers_on_corrupt_usage_baseline_without_bloom_write() { + let store = Arc::new(MemoryConfigStore::default()); + let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + store.objects.lock().await.insert(usage_key.clone(), b"not-json".to_vec()); + store.revisions.lock().await.insert(usage_key, 1); + + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + next: 12, + ..Default::default() + }; + let mut persisted_epoch = 0; + + assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await); + assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); +} + +#[tokio::test] +async fn leadership_claim_defers_on_unidentified_usage_baseline_without_bloom_write() { + let store = Arc::new(MemoryConfigStore::default()); + let usage_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let data = serde_json::to_vec(&DataUsageInfo::default()).expect("default usage should encode"); + store.objects.lock().await.insert(usage_key.clone(), data); + store.revisions.lock().await.insert(usage_key, 1); + + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + next: 12, + ..Default::default() + }; + let mut persisted_epoch = 0; + + assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch,).await); + assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); +} + #[tokio::test] async fn test_leadership_claim_confirms_commit_after_returned_error() { let store = Arc::new(MemoryConfigStore::default()); @@ -2329,6 +2433,7 @@ async fn test_leadership_claim_confirms_commit_after_returned_error() { started: Utc::now(), }; let mut persisted_epoch = 0; + seed_usage_snapshot_for_leadership_claim(&store).await; assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch).await); @@ -2373,6 +2478,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() { ); old_usage.buckets_count = 1; old_usage.calculate_totals(); + old_usage.usage_snapshot_complete = true; let old_data = serde_json::to_vec(&old_usage).expect("old usage snapshot should encode"); store.objects.lock().await.insert(usage_key.clone(), old_data.clone()); store.revisions.lock().await.insert(usage_key, 1); @@ -2419,6 +2525,7 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() { started: Utc::now(), }; assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + seed_usage_snapshot_for_leadership_claim(&store).await; let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); store @@ -2590,6 +2697,35 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() { } } +#[tokio::test] +async fn test_observational_usage_defers_when_authoritative_baseline_is_missing() { + let store = Arc::new(MemoryConfigStore::default()); + let (sender, receiver) = mpsc::channel(1); + let mut observation = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(20)), 1); + observation.usage_snapshot_converged = Some(false); + sender.send(observation).await.expect("observation should enqueue"); + drop(sender); + + let outcome = store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe( + CancellationToken::new(), + store.clone(), + receiver, + None, + None, + || async { false }, + ) + .await; + + assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement)); + assert!( + !store + .objects + .lock() + .await + .contains_key(&memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str())) + ); +} + #[tokio::test] async fn test_usage_route_barrier_precedes_durable_reconciliation() { let store = Arc::new(MemoryConfigStore::default()); @@ -3425,6 +3561,14 @@ fn complete_usage_with_bucket_count(last_update: Option, info } +async fn seed_usage_snapshot_for_leadership_claim(store: &Arc) { + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let data = serde_json::to_vec(&complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)) + .expect("leadership usage baseline should encode"); + store.objects.lock().await.insert(key.clone(), data); + store.revisions.lock().await.insert(key, 1); +} + fn usage_with_last_update(last_update: Option) -> DataUsageInfo { complete_usage_with_bucket_count(last_update, 0) } @@ -5129,6 +5273,19 @@ fn test_background_heal_info_for_scan_start_marks_deep_active() { assert_eq!(info.bitrot_start_time, Some(now)); } +#[test] +fn background_heal_read_failures_never_become_initializable_defaults() { + assert_eq!( + classify_background_heal_read_error(&EcstoreError::ConfigNotFound), + BackgroundHealInfoReadStatus::Missing + ); + assert_eq!( + classify_background_heal_read_error(&EcstoreError::SlowDown), + BackgroundHealInfoReadStatus::Transient + ); + assert!(decode_background_heal_info(b"not-json").is_err()); +} + #[test] fn test_background_heal_info_for_scan_start_keeps_deep_window_start() { with_var_unset(ENV_SCANNER_BITROT_CYCLE_SECS, || { diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs index 4f291cd02..9b211c1f5 100644 --- a/crates/scanner/src/scanner/usage_store.rs +++ b/crates/scanner/src/scanner/usage_store.rs @@ -112,11 +112,39 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel } pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe( + ctx: CancellationToken, + storeapi: Arc, + receiver: mpsc::Receiver, + leader_epoch: Option, + initial_baseline: Option, + route_probe: F, +) -> DataUsagePersistOutcome +where + F: Fn() -> Fut + Send + Sync, + Fut: Future + Send, +{ + store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch( + ctx, + storeapi, + receiver, + leader_epoch, + initial_baseline, + None, + route_probe, + ) + .await +} + +pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch< + F, + Fut, +>( ctx: CancellationToken, storeapi: Arc, mut receiver: mpsc::Receiver, leader_epoch: Option, initial_baseline: Option, + expected_publication_epoch: Option, route_probe: F, ) -> DataUsagePersistOutcome where @@ -134,6 +162,14 @@ where if let Some(leader_epoch) = leader_epoch { data_usage_info.scanner_epoch = Some(leader_epoch); } + if let Some(expected_epoch) = expected_publication_epoch + && scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } let observational = data_usage_info.usage_snapshot_converged == Some(false); let target_path = if observational { DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str() @@ -154,7 +190,28 @@ where break; } + let mut publication_epoch = expected_publication_epoch; if observational && data_usage_info.usage_snapshot_authoritative_baseline.is_none() { + let read_epoch = match expected_publication_epoch { + Some(expected_epoch) => { + if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } + expected_epoch + } + None => { + let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + }; + read_epoch + } + }; + publication_epoch = Some(read_epoch); let authoritative_data = match next_baseline.as_ref() { Some(baseline) => baseline.data.clone(), None => match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()).await { @@ -175,25 +232,48 @@ where } }, }; - let authoritative = match authoritative_data.as_deref() { - Some(data) => match serde_json::from_slice::(data) { - Ok(info) => info, - Err(err) => { - error!( - target: "rustfs::scanner", - event = EVENT_SCANNER_PERSIST_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), - state = "observed_baseline_decode_failed", - error = %err, - "Scanner refused to publish an observation from an invalid authoritative baseline" - ); - outcome = DataUsagePersistOutcome::Failed; - continue; - } - }, - None => DataUsageInfo::default(), + let Some(authoritative_data) = authoritative_data else { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "observed_baseline_missing", + "Scanner deferred observational publication until an authoritative usage baseline exists" + ); + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + }; + let authoritative = match serde_json::from_slice::(&authoritative_data) { + Ok(info) if data_usage_info_has_persisted_baseline_identity(&info) => info, + Ok(_) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "observed_baseline_identity_missing", + "Scanner refused to publish an observation without authoritative baseline identity" + ); + outcome = DataUsagePersistOutcome::Failed; + continue; + } + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + state = "observed_baseline_decode_failed", + error = %err, + "Scanner refused to publish an observation from an invalid authoritative baseline" + ); + outcome = DataUsagePersistOutcome::Failed; + continue; + } }; data_usage_info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity()); } @@ -240,6 +320,26 @@ where break 'updates; } + let publication_epoch_for_save = match expected_publication_epoch { + Some(expected_epoch) => { + if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + } + expected_epoch + } + None => match publication_epoch.take() { + Some(epoch) => epoch, + None => { + let Some(epoch) = scanner_publication_epoch(storeapi.clone()).await else { + break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + }; + epoch + } + }, + }; let baseline = if !observational && cas_retry == 0 { next_baseline.take() } else { @@ -329,14 +429,22 @@ where } let done_save = Metrics::time(Metric::SaveUsage); - let save_result = save_config_shared_with_preconditions( - storeapi.clone(), - target_path, - data.clone(), - sha256hex.clone(), - revision.preconditions(), - ) - .await; + let save_result = { + let Some(_publication_admission) = + scanner_publication_admission_for_epoch(storeapi.clone(), publication_epoch_for_save).await + else { + done_save(); + break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + }; + save_config_shared_with_preconditions( + storeapi.clone(), + target_path, + data.clone(), + sha256hex.clone(), + revision.preconditions(), + ) + .await + }; done_save(); match save_result { @@ -427,7 +535,16 @@ where if observational { invalidate_admin_data_usage_snapshot_cache().await; } else { - cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; + let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch( + storeapi.clone(), + &data_usage_info, + expected_publication_epoch, + ) + .await; + if expected_publication_epoch.is_some() && !cleanup_ok { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } invalidate_data_usage_snapshot_cache().await; replace_bucket_usage_memory_from_info(&data_usage_info).await; } @@ -438,7 +555,16 @@ where if observational { invalidate_admin_data_usage_snapshot_cache().await; } else { - cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; + let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch( + storeapi.clone(), + &data_usage_info, + expected_publication_epoch, + ) + .await; + if expected_publication_epoch.is_some() && !cleanup_ok { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } invalidate_data_usage_snapshot_cache().await; } global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); @@ -460,7 +586,16 @@ where if observational { invalidate_admin_data_usage_snapshot_cache().await; } else { - cleanup_observed_data_usage_snapshot(storeapi.clone(), &data_usage_info).await; + let cleanup_ok = cleanup_observed_data_usage_snapshot_for_epoch( + storeapi.clone(), + &data_usage_info, + expected_publication_epoch, + ) + .await; + if expected_publication_epoch.is_some() && !cleanup_ok { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } invalidate_data_usage_snapshot_cache().await; replace_bucket_usage_memory_from_info(&data_usage_info).await; } @@ -471,7 +606,10 @@ where if backup_due { let done_save = Metrics::time(Metric::SaveUsage); - if let Err(e) = sync_data_usage_backup_from_primary(&ctx, storeapi.clone()).await { + let backup_result = + sync_data_usage_backup_from_primary_for_epoch(&ctx, storeapi.clone(), expected_publication_epoch).await; + done_save(); + if let Err(e) = backup_result { warn!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, @@ -482,22 +620,50 @@ where error = %e, "Scanner data usage backup save failed" ); + if scanner_publication_epoch_changed(&e) { + outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break 'updates; + } + outcome = DataUsagePersistOutcome::Failed; + break 'updates; } - done_save(); } } outcome } -pub(super) async fn cleanup_observed_data_usage_snapshot( +pub(super) async fn cleanup_observed_data_usage_snapshot_for_epoch( storeapi: Arc, authoritative: &DataUsageInfo, -) { + expected_publication_epoch: Option, +) -> bool { + let read_epoch = match expected_publication_epoch { + Some(expected_epoch) => { + if scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch) + .await + .is_none() + { + return false; + } + expected_epoch + } + None => match scanner_publication_epoch(storeapi.clone()).await { + Some(read_epoch) => read_epoch, + None => return false, + }, + }; + if expected_publication_epoch.is_some() + && scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch) + .await + .is_none() + { + return false; + } let (observed_data, revision) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str()).await { Ok((Some(data), revision)) => (data, revision), - Ok((None, _)) => return, + Ok((None, _)) => return true, Err(err) => { error!( target: "rustfs::scanner", @@ -509,7 +675,7 @@ pub(super) async fn cleanup_observed_data_usage_snapshot( error = %err, "Scanner could not inspect observational data usage snapshot before authoritative cleanup" ); - return; + return true; } }; let observed = match serde_json::from_slice::(&observed_data) { @@ -525,25 +691,26 @@ pub(super) async fn cleanup_observed_data_usage_snapshot( error = %err, "Scanner refused to remove an invalid observational data usage snapshot after authoritative save" ); - return; + return true; } }; if observed_data_usage_is_newer(&observed, authoritative) { - return; + return true; } - let result = storeapi - .delete_config_object( - RUSTFS_META_BUCKET, - DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), - ScannerObjectOptions { - delete_prefix: true, - delete_prefix_object: true, - http_preconditions: Some(revision.preconditions()), - ..Default::default() - }, - ) - .await; + let result = delete_config_with_publication_admission_for_epoch( + storeapi, + RUSTFS_META_BUCKET, + DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), + ScannerObjectOptions { + delete_prefix: true, + delete_prefix_object: true, + http_preconditions: Some(revision.preconditions()), + ..Default::default() + }, + read_epoch, + ) + .await; match result { Ok(_) @@ -564,6 +731,10 @@ pub(super) async fn cleanup_observed_data_usage_snapshot( error = %err, "Scanner could not remove stale observational data usage snapshot after authoritative save" ); + if scanner_publication_epoch_changed(&err) { + return false; + } } } + true } diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 030668f26..0a5ecdf12 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -56,9 +56,10 @@ use crate::storage_api::scan::NamespaceLocking as _; use crate::storage_api::scanner_io::{BucketInfo, BucketOptions}; use crate::{ BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result, - RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _, - ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, enqueue_runtime_free_version, - get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names, storageclass, + RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerConfigObjectDelete as _, ScannerDiskExt as _, + ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, + enqueue_runtime_free_version, get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names, + scanner_publication_admission_for_epoch, scanner_publication_epoch, storageclass, }; pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file"; @@ -143,6 +144,10 @@ pub struct ScannerBucketScanPlan { all_buckets: Arc>, digest: DataUsageScanPlanDigest, leader_epoch: u64, + /// Epoch captured once for the whole scanner cycle. `None` is retained + /// for unfenced test implementations; production plans always carry the + /// admission token captured before bucket enumeration. + publication_epoch: Option, dirty_usage_buckets: Arc, bucket_failures: ScannerBucketFailureState, pending_maintenance_work: Arc, @@ -578,6 +583,7 @@ fn scanner_activity_preflight( #[derive(Debug)] pub(crate) struct ScannerCycleResult { pub(crate) status: ScannerCycleStatus, + publication_epoch: Option, dirty_usage_clear: Option, remote_dirty_usage_acknowledgements: Vec, failed_dirty_usage: bool, @@ -589,6 +595,7 @@ impl ScannerCycleResult { pub(crate) fn new(status: ScannerCycleStatus, dirty_usage_clear: Option) -> Self { Self { status, + publication_epoch: None, dirty_usage_clear, remote_dirty_usage_acknowledgements: Vec::new(), failed_dirty_usage: false, @@ -597,6 +604,15 @@ impl ScannerCycleResult { } } + pub(crate) fn with_publication_epoch(mut self, publication_epoch: Option) -> Self { + self.publication_epoch = publication_epoch; + self + } + + pub(crate) fn publication_epoch(&self) -> Option { + self.publication_epoch + } + fn with_failed_dirty_usage(mut self, failed_dirty_usage: bool) -> Self { self.failed_dirty_usage = failed_dirty_usage; self diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index 001901187..070a89deb 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -395,6 +395,7 @@ pub(super) async fn persist_and_publish_cache_snapshot( updates: &mpsc::Sender, mut cache_snapshot: DataUsageCache, cache_cycle_floor: &AtomicU64, + expected_publication_epoch: u64, ) -> Option { let source = cache_snapshot.info.source?; let guard = match acquire_scanner_cache_locks(store.as_ref(), DATA_USAGE_CACHE_NAME, source).await { @@ -489,7 +490,7 @@ pub(super) async fn persist_and_publish_cache_snapshot( let done_save = Metrics::time(Metric::SaveUsage); if let Err(e) = cache_snapshot - .save_with_revisions(store, DATA_USAGE_CACHE_NAME, &revisions) + .save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch) .await { error!( @@ -519,6 +520,24 @@ pub(super) async fn persist_and_publish_cache_snapshot( ); return None; } + // The persisted-root fast path performs no PUT, so it also needs the + // cycle token re-admission before forwarding the root to the aggregate. + // This final check covers both the fast path and a successful save. + if scanner_publication_admission_for_epoch(store.clone(), expected_publication_epoch) + .await + .is_none() + { + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + cache_name = DATA_USAGE_CACHE_NAME, + state = "publication_epoch_changed_before_publish", + "Scanner cache root publish skipped after movement epoch change" + ); + return None; + } drop(guard); let last_update = cache_snapshot.info.last_update; diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index db52cb559..5b8246bc8 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -31,6 +31,7 @@ impl ScannerIOCache for SetDisks { all_buckets, digest: scan_plan_digest, leader_epoch, + publication_epoch, dirty_usage_buckets, bucket_failures, pending_maintenance_work, @@ -40,6 +41,12 @@ impl ScannerIOCache for SetDisks { let set_label = self.set_index.to_string(); let source = DataUsageCacheSource::new(self.pool_index, self.set_index); + let expected_publication_epoch = match publication_epoch { + Some(epoch) => epoch, + None => scanner_publication_epoch(self.clone()) + .await + .ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?, + }; let mut old_cache = DataUsageCache::default(); if let Err(e) = old_cache.load(self.clone(), DATA_USAGE_CACHE_NAME).await { warn!( @@ -76,10 +83,16 @@ impl ScannerIOCache for SetDisks { cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default()); } reset_disk_bucket_scan_gauges(&pool_label, &set_label); - return persist_and_publish_cache_snapshot(self, &updates, cache, cache_cycle_floor.as_ref()) - .await - .map(|_| ()) - .ok_or_else(|| StorageError::other("failed to persist empty scanner set scope")); + return persist_and_publish_cache_snapshot( + self, + &updates, + cache, + cache_cycle_floor.as_ref(), + expected_publication_epoch, + ) + .await + .map(|_| ()) + .ok_or_else(|| StorageError::other("failed to persist empty scanner set scope")); } let (disks, healing) = self.get_online_disks_with_healing(false).await; @@ -414,6 +427,7 @@ impl ScannerIOCache for SetDisks { let pending_maintenance_work_clone = pending_maintenance_work.clone(); let dirty_usage_buckets_clone = dirty_usage_buckets.clone(); let cache_cycle_floor_clone = cache_cycle_floor.clone(); + let expected_publication_epoch_clone = expected_publication_epoch; let remote_server_epoch = match worker_mode { NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch), NamespaceScannerWorkerMode::Coordinator => None, @@ -753,6 +767,23 @@ impl ScannerIOCache for SetDisks { ); continue; } + if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch) + .await + .is_none() + { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "publication_epoch_changed_before_reuse", + "Current scanner bucket cache root publish skipped after movement epoch change" + ); + continue; + } if let Err(e) = send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone) .await @@ -901,7 +932,12 @@ impl ScannerIOCache for SetDisks { { let done_save = Metrics::time(Metric::SaveUsage); if let Err(e) = cache - .save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions) + .save_with_revisions_for_epoch( + store_clone_clone.clone(), + cache_name.as_str(), + &revisions, + expected_publication_epoch_clone, + ) .await { error!( @@ -958,7 +994,12 @@ impl ScannerIOCache for SetDisks { false } else { match partial_cache - .save_with_revisions(store_clone_clone.clone(), cache_name.as_str(), &revisions) + .save_with_revisions_for_epoch( + store_clone_clone.clone(), + cache_name.as_str(), + &revisions, + expected_publication_epoch_clone, + ) .await { Ok(()) => true, @@ -1029,7 +1070,12 @@ impl ScannerIOCache for SetDisks { let done_save = Metrics::time(Metric::SaveUsage); if let Err(e) = cache - .save_with_revisions(store_clone_clone.clone(), &cache_name, &revisions) + .save_with_revisions_for_epoch( + store_clone_clone.clone(), + &cache_name, + &revisions, + expected_publication_epoch_clone, + ) .await { done_save(); @@ -1064,6 +1110,24 @@ impl ScannerIOCache for SetDisks { continue; } + if scanner_publication_admission_for_epoch(store_clone_clone.clone(), expected_publication_epoch_clone) + .await + .is_none() + { + record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await; + error!( + target: "rustfs::scanner::io", + event = EVENT_SCANNER_CACHE_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_IO, + bucket = %bucket.name, + cache_name = %cache_name, + state = "publication_epoch_changed_after_save", + "Scanner bucket cache root publish skipped after movement epoch change" + ); + continue; + } + debug!( target: "rustfs::scanner::io", event = EVENT_SCANNER_DATA_USAGE_STREAM, @@ -1159,7 +1223,14 @@ impl ScannerIOCache for SetDisks { cache.info.lkg_scan_plan_digest = None; cache.clone() }; - let _ = persist_and_publish_cache_snapshot(self.clone(), &updates, cache_snapshot, cache_cycle_floor.as_ref()).await; + let _ = persist_and_publish_cache_snapshot( + self.clone(), + &updates, + cache_snapshot, + cache_cycle_floor.as_ref(), + expected_publication_epoch, + ) + .await; } else { let mut incomplete_scope = cache_mutex.lock().await.clone(); incomplete_scope.info.name = DATA_USAGE_ROOT.to_string(); diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index defa8a47c..7a74e95f8 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -68,6 +68,19 @@ impl ScannerIOCycle for ECStore { )); } + // Capture one storage-owned movement epoch for the entire cycle. Set + // workers must not each observe a fresh epoch: a movement transition + // between sets would otherwise allow a mixed-generation aggregate. + let publication_epoch = match self.scanner_data_usage_publication_admission().await { + Some(admission) => Some(admission.epoch()), + None => { + return Ok(ScannerCycleResult::new( + ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement), + None, + )); + } + }; + let distributed = self.setup_is_dist_erasure().await; let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) { ScannerActivityPreflight::Ready(snapshot) => snapshot, @@ -131,7 +144,9 @@ impl ScannerIOCycle for ECStore { if all_buckets.is_empty() { reset_set_scan_gauges(); if !bucket_plan_complete { - return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); + return Ok( + ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch) + ); } let activity_status = scanner_cycle_activity_status(self, distributed, &activity_before).await; let dirty_usage_status = dirty_usage_snapshot_status(&dirty_usage_snapshot); @@ -155,7 +170,7 @@ impl ScannerIOCycle for ECStore { ) .await? { - return Ok(ScannerCycleResult::new(status, None)); + return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch)); } let dirty_usage_clear = (status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone()); @@ -165,6 +180,7 @@ impl ScannerIOCycle for ECStore { Vec::new() }; return Ok(ScannerCycleResult::new(status, dirty_usage_clear) + .with_publication_epoch(publication_epoch) .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements)); } @@ -180,7 +196,7 @@ impl ScannerIOCycle for ECStore { "Scanner set state update detected missing disk sets" ); reset_set_scan_gauges(); - return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)); + return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch)); } let set_scan_limit = scanner_budgeted_concurrency_limit( @@ -250,6 +266,7 @@ impl ScannerIOCycle for ECStore { all_buckets: Arc::clone(&all_buckets), digest: scan_plan_digest, leader_epoch, + publication_epoch, dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(), bucket_failures: bucket_failures.clone(), pending_maintenance_work: pending_maintenance_work.clone(), @@ -430,6 +447,7 @@ impl ScannerIOCycle for ECStore { Vec::new() }; Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear) + .with_publication_epoch(publication_epoch) .with_remote_dirty_usage_acknowledgements(remote_dirty_usage_acknowledgements) .with_failed_dirty_usage(!failed_buckets.is_empty()) .with_pending_maintenance_work(pending_maintenance_work) diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 9d2c1586d..7931a44f9 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -145,6 +145,50 @@ async fn scanner_cache_locks_allow_cross_source_workers() { assert!(!second.is_lock_lost()); } +#[tokio::test] +async fn scanner_set_cache_admission_tracks_owner_snapshot_and_fails_closed() { + let (_temp_dir, store) = setup_two_pool_scanner_store().await; + let set = store.pools[0].disk_set[0].clone(); + + assert!( + set.scanner_data_usage_publication_admission_guard().await.is_none(), + "a set must not publish before the owner has refreshed its movement snapshot" + ); + assert!(!store.scanner_data_usage_publication_blocked().await); + assert!( + set.scanner_data_usage_publication_admission_guard().await.is_some(), + "an idle owner snapshot should admit the set cache" + ); + + let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()]; + pool_stats[0] = EcstoreRebalanceStats { + participating: true, + info: EcstoreRebalanceInfo { + start_time: Some(OffsetDateTime::now_utc()), + status: EcstoreRebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }; + *store.rebalance_meta.write().await = Some(EcstoreRebalanceMeta { + id: Uuid::new_v4().to_string(), + pool_stats, + ..Default::default() + }); + assert!(store.scanner_data_usage_publication_blocked().await); + assert!( + set.scanner_data_usage_publication_admission_guard().await.is_none(), + "active movement must keep set cache publication blocked" + ); + + *store.rebalance_meta.write().await = None; + assert!(!store.scanner_data_usage_publication_blocked().await); + assert!( + set.scanner_data_usage_publication_admission_guard().await.is_some(), + "an idle owner refresh must make set cache publication live again" + ); +} + #[tokio::test] async fn scanner_cycle_is_deferred_while_rebalance_is_active() { let (_temp_dir, store) = setup_two_pool_scanner_store().await; From 76a863b3eafde7404394a348210fabf3262424ab Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 17:28:52 +0800 Subject: [PATCH 18/41] fix(scanner): unify unknown metadata size accounting (#6394) * fix(scanner): unify unknown metadata size accounting * fix(scanner): preserve restore expiry semantics * fix(ci): resolve ecstore clippy warnings * fix(scanner): close lifecycle review gaps --------- Signed-off-by: houseme Co-authored-by: houseme --- crates/data-usage/src/data_usage.rs | 90 +- crates/lifecycle/src/core.rs | 37 +- crates/scanner/src/data_usage_define.rs | 56 +- crates/scanner/src/data_usage_define/tests.rs | 39 + crates/scanner/src/scanner_folder.rs | 134 ++- .../src/scanner_folder/item_actions.rs | 874 ++++++++++++++++-- crates/scanner/src/scanner_folder/tests.rs | 63 ++ 7 files changed, 1212 insertions(+), 81 deletions(-) diff --git a/crates/data-usage/src/data_usage.rs b/crates/data-usage/src/data_usage.rs index dd8c3f158..fec5930fc 100644 --- a/crates/data-usage/src/data_usage.rs +++ b/crates/data-usage/src/data_usage.rs @@ -332,6 +332,38 @@ pub struct DiskUsageStatus { pub snapshot_exists: bool, } +/// A bounded reconciliation record for an object whose logical size could not +/// be trusted at the scanner boundary. The scanner persists these records in +/// its cache; keeping the model here avoids a second, incompatible accounting +/// representation in storage-facing crates. +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SizeReconciliationEntry { + /// Stable object/version identity key (not a metrics label). + pub key: String, + pub bucket: String, + pub object: String, + #[serde(default)] + pub version_id: Option, + #[serde(default)] + pub generation: Option, + /// Structured reason label; raw metadata values must never be stored here. + pub reason: String, + #[serde(default)] + pub physical_size: Option, + #[serde(default)] + pub first_seen: u64, + #[serde(default)] + pub attempts: u32, +} + +/// Object scope refreshed by one scanner pass. Existing debts in this scope +/// are removed before the pass's unresolved records are inserted. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SizeReconciliationScope { + pub bucket: String, + pub object: String, +} + /// Size summary for a single object or group of objects #[derive(Debug, Default, Clone)] pub struct SizeSummary { @@ -361,6 +393,16 @@ pub struct SizeSummary { pub repl_target_stats: HashMap, /// Per-tier accounting, keyed by storage class or remote tier name pub tier_stats: HashMap, + /// Size-resolution debts observed while scanning this summary. + pub size_reconciliation: Vec, + /// True when the per-object summary exceeded its bounded debt buffer. + /// Callers must retain prior ledger entries rather than treating the + /// partial list as a complete refresh. + pub size_reconciliation_truncated: bool, + /// Object scopes refreshed by this summary. They let the durable ledger + /// remove versions that resolved without allocating one key per healthy + /// version on the hot path. + pub reconciliation_scopes: Vec, } /// Replication target size summary @@ -858,7 +900,8 @@ impl DataUsageEntry { /// /// The canonical wire format is written by the hand-written map-encoded /// `Serialize` on the scanner-side `DataUsageCacheInfo` -/// (`crates/scanner/src/data_usage_define.rs`), which carries 16 fields. +/// (`crates/scanner/src/data_usage_define.rs`), which carries the original 16 +/// fields plus an optional reconciliation field. /// This type decodes only the shared subset and is deliberately not /// `Serialize`: a derived (array) encoding of this 6-field subset would /// corrupt the cache for scanner readers, so no write path may exist here. @@ -1835,6 +1878,51 @@ impl SizeSummary { entry.pending_count = entry.pending_count.saturating_add(stats.pending_count); entry.failed_count = entry.failed_count.saturating_add(stats.failed_count); } + + for entry in &other.size_reconciliation { + self.record_size_reconciliation(entry.clone()); + } + self.size_reconciliation_truncated |= other.size_reconciliation_truncated; + for scope in &other.reconciliation_scopes { + self.record_reconciliation_scope(&scope.bucket, &scope.object); + } + } + + /// Add one reconciliation debt, coalescing repeated observations in the + /// same object summary. The scanner cache applies its own larger bound. + pub fn record_size_reconciliation(&mut self, entry: SizeReconciliationEntry) { + const MAX_SUMMARY_RECONCILIATION_ENTRIES: usize = 1024; + if let Some(existing) = self.size_reconciliation.iter_mut().find(|value| value.key == entry.key) { + existing.reason = entry.reason; + existing.physical_size = entry.physical_size; + existing.generation = entry.generation; + existing.version_id = entry.version_id; + return; + } + if self.size_reconciliation.len() < MAX_SUMMARY_RECONCILIATION_ENTRIES { + self.size_reconciliation.push(entry); + } else { + self.size_reconciliation_truncated = true; + } + } + + /// Mark one object scope as refreshed. Duplicate scopes are suppressed so + /// merging summaries remains bounded and deterministic. + pub fn record_reconciliation_scope(&mut self, bucket: &str, object: &str) { + if !self + .reconciliation_scopes + .iter() + .any(|scope| scope.bucket == bucket && scope.object == object) + { + if self.reconciliation_scopes.len() >= 1024 { + self.size_reconciliation_truncated = true; + return; + } + self.reconciliation_scopes.push(SizeReconciliationScope { + bucket: bucket.to_string(), + object: object.to_string(), + }); + } } } diff --git a/crates/lifecycle/src/core.rs b/crates/lifecycle/src/core.rs index 6180d0022..7dea47b71 100644 --- a/crates/lifecycle/src/core.rs +++ b/crates/lifecycle/src/core.rs @@ -54,6 +54,7 @@ const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS: &str = "Days must be a positive integer and Date must not be specified inside Expiration with ExpiredObjectAllVersions"; const ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS: &str = "Days must be a positive integer with DelMarkerExpiration"; const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters"; +const ERR_LIFECYCLE_INVALID_RULE_ID_EMPTY: &str = "Rule ID must not be empty"; const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled"; const ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS: &str = "Rule with DelMarkerExpiration cannot have tags based filtering"; const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str = @@ -402,10 +403,13 @@ impl Lifecycle for BucketLifecycleConfiguration { NoncurrentVersionTransitionOps::validate(transition)?; } } - if let Some(id) = &r.id - && id.len() > 255 - { - return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG)); + if let Some(id) = &r.id { + if id.is_empty() { + return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_EMPTY)); + } + if id.len() > 255 { + return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG)); + } } r.validate()?; if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref() @@ -3730,6 +3734,31 @@ mod tests { .expect("empty prefix with filter should be valid"); } + #[tokio::test] + async fn validate_rejects_empty_rule_id() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(30), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some(String::new()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + let error = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err(); + assert_eq!(error.to_string(), ERR_LIFECYCLE_INVALID_RULE_ID_EMPTY); + } + // --- TASK-004 tests: ExpiredObjectAllVersions --- #[tokio::test] diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 3c69d2173..79d8fc9b7 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -29,8 +29,8 @@ use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS; pub use rustfs_data_usage::{ AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, DataUsageSnapshotSetState, LEGACY_DATA_USAGE_OBJECT_NAME, - PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path, - prefix_usage_in_cache, + PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeReconciliationEntry, + SizeReconciliationScope, SizeSummary, TierStats, hash_path, prefix_usage_in_cache, }; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use tokio::time::{Duration, Instant, sleep, timeout}; @@ -201,6 +201,10 @@ const MAX_DATA_USAGE_CACHE_DEPTH: usize = 1024; pub trait ScannerSizeSummaryExt { /// Fold one object's contribution into the summary, including its tier. fn actions_accounting(&mut self, oi: &ObjectInfo, size: i64, actual_size: i64); + /// Fold counters and physical tier usage for an object whose metadata is + /// valid but whose logical size is currently unavailable. Logical totals + /// stay unchanged. + fn actions_accounting_unknown(&mut self, oi: &ObjectInfo); } impl ScannerSizeSummaryExt for SizeSummary { @@ -234,6 +238,34 @@ impl ScannerSizeSummaryExt for SizeSummary { }); } } + + fn actions_accounting_unknown(&mut self, oi: &ObjectInfo) { + if oi.delete_marker { + self.delete_markers = self.delete_markers.saturating_add(1); + return; + } + + if oi.version_id.is_some_and(|v| !v.is_nil()) { + self.versions = self.versions.saturating_add(1); + } + + if oi.transitioned_object.free_version { + return; + } + + let tier = if oi.transitioned_object.status == TRANSITION_COMPLETE { + oi.transitioned_object.tier.clone() + } else { + oi.storage_class.clone().unwrap_or_else(|| storageclass::STANDARD.to_string()) + }; + if let Some(tier_stats) = self.tier_stats.get_mut(&tier) { + *tier_stats = tier_stats.add(&TierStats { + total_size: u64::try_from(oi.size).unwrap_or(0), + num_versions: 1, + num_objects: u64::from(oi.is_latest), + }); + } + } } // ===== Cache-related data structures ===== @@ -353,6 +385,10 @@ pub struct DataUsageCacheInfo { pub scan_plan_digest: Option, #[serde(default)] pub cache_key_format: u16, + /// Bounded durable debts for versions whose logical size was not trusted. + /// The map key is an identity key, never a user-controlled metric label. + #[serde(default)] + pub size_reconciliation: HashMap, /// Whether the entries retained while a set scan was incomplete come /// from a prior complete set snapshot. This is observational input only. #[serde(default)] @@ -374,7 +410,8 @@ impl Serialize for DataUsageCacheInfo { { // Keep this metadata map-encoded so older readers can ignore fields // appended by newer scanner versions during rolling upgrades. - let mut state = serializer.serialize_map(Some(21))?; + let field_count = 21 + usize::from(!self.size_reconciliation.is_empty()); + let mut state = serializer.serialize_map(Some(field_count))?; state.serialize_entry("name", &self.name)?; state.serialize_entry("next_cycle", &self.next_cycle)?; state.serialize_entry("leader_epoch", &self.leader_epoch)?; @@ -391,6 +428,9 @@ impl Serialize for DataUsageCacheInfo { state.serialize_entry("snapshot_complete", &self.snapshot_complete)?; state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?; state.serialize_entry("cache_key_format", &self.cache_key_format)?; + if !self.size_reconciliation.is_empty() { + state.serialize_entry("size_reconciliation", &self.size_reconciliation)?; + } state.serialize_entry("lkg_snapshot_complete", &self.lkg_snapshot_complete)?; state.serialize_entry("lkg_next_cycle", &self.lkg_next_cycle)?; state.serialize_entry("lkg_last_update", &self.lkg_last_update)?; @@ -454,14 +494,18 @@ impl DataUsageCache { self.checked_flatten(name).is_some() }); if !reusable { - let pending_heals = if self.info.name == name { - std::mem::take(&mut self.info.pending_heals) + let (pending_heals, size_reconciliation) = if self.info.name == name { + ( + std::mem::take(&mut self.info.pending_heals), + std::mem::take(&mut self.info.size_reconciliation), + ) } else { - Vec::new() + (Vec::new(), HashMap::new()) }; *self = Self::default(); self.info.name = name.to_string(); self.info.pending_heals = pending_heals; + self.info.size_reconciliation = size_reconciliation; } self.info.next_cycle = next_cycle; diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index 9a458d4b0..5c4d55f05 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -721,6 +721,34 @@ fn size_summary_actions_accounting_accumulates_tier_stats() { ); } +#[test] +fn size_summary_unknown_accounting_keeps_physical_tier_and_version_only() { + let mut summary = SizeSummary::default(); + summary + .tier_stats + .insert(storageclass::STANDARD.to_string(), TierStats::default()); + let object = ObjectInfo { + size: 12, + storage_class: Some(storageclass::STANDARD.to_string()), + version_id: Some(uuid::Uuid::new_v4()), + is_latest: true, + ..Default::default() + }; + + summary.actions_accounting_unknown(&object); + + assert_eq!(summary.total_size, 0, "unknown logical size must not become zero or physical bytes"); + assert_eq!(summary.versions, 1); + assert_eq!( + summary.tier_stats.get(storageclass::STANDARD), + Some(&TierStats { + total_size: 12, + num_versions: 1, + num_objects: 1, + }) + ); +} + #[test] fn test_data_usage_entry_merge_sums_failed_objects() { let mut left = DataUsageEntry { @@ -1127,6 +1155,16 @@ fn data_usage_cache_prepare_for_scan_preserves_pending_heal_only_progress() { scan_plan_digest: Some(TEST_PLAN_DIGEST), cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT, pending_heals: vec![pending_heal.clone()], + size_reconciliation: HashMap::from([( + "size-key".to_string(), + SizeReconciliationEntry { + key: "size-key".to_string(), + bucket: "bucket".to_string(), + object: "prefix/object".to_string(), + reason: "invalid_declared_size".to_string(), + ..Default::default() + }, + )]), ..Default::default() }, ..Default::default() @@ -1136,6 +1174,7 @@ fn data_usage_cache_prepare_for_scan_preserves_pending_heal_only_progress() { assert_eq!(outcome, DataUsageCachePrepareOutcome::Reused); assert_eq!(cache.info.pending_heals, vec![pending_heal]); + assert!(cache.info.size_reconciliation.contains_key("size-key")); assert!(cache.cache.is_empty()); assert!(!cache.info.snapshot_complete); } diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 148249ea8..bd9727154 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -20,8 +20,9 @@ use std::time::{Duration, Instant, SystemTime}; use crate::ReplTargetSizeSummary; use crate::data_usage_define::{ - DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageScanCheckpoint, - DataUsageScanCheckpointReason, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeSummary, hash_path, + DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap, + DataUsageScanCheckpoint, DataUsageScanCheckpointReason, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, + SizeReconciliationEntry, SizeSummary, hash_path, }; use crate::error::ScannerError; use crate::runtime_config::{ @@ -105,6 +106,9 @@ const METRIC_SCANNER_HEAL_DISCOVERY_UNVERIFIED_TOTAL: &str = "rustfs_scanner_hea const METRIC_SCANNER_HEAL_DISCOVERY_QUEUED_TOTAL: &str = "rustfs_scanner_heal_discovery_queued_total"; const METRIC_SCANNER_HEAL_DISCOVERY_TRUNCATED_TOTAL: &str = "rustfs_scanner_heal_discovery_truncated_total"; const MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET: usize = 128; +const MAX_SIZE_RECONCILIATION_ENTRIES_PER_BUCKET: usize = 10_000; +const MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET: usize = 8 * 1024 * 1024; +const MAX_SIZE_RECONCILIATION_AGE_SECS: u64 = 7 * 24 * 60 * 60; // --- scanner excess alerts as S3 notification events (rustfs/backlog#1868) -- // @@ -372,7 +376,7 @@ impl PendingScannerAccounting<'_> { fn apply(self, size_summary: &mut SizeSummary, cumulative_size: &mut i64, queued: bool) { let size = if queued { self.expired_size } else { self.retained_size }; size_summary.actions_accounting(self.object, size, self.retained_size); - *cumulative_size += size; + *cumulative_size = cumulative_size.saturating_add(size); } } @@ -679,10 +683,65 @@ pub struct FolderScanner { skip_heal: Arc, local_disk: Arc, pending_heals_changed: bool, + pending_size_reconciliation_keys: HashSet, + pending_size_reconciliation_scopes: HashSet, + pending_size_reconciliation_truncated: bool, #[cfg(test)] list_path_raw_options_observer: Option>, } +fn size_reconciliation_entry_bytes(entry: &SizeReconciliationEntry) -> usize { + entry.key.len() + + entry.bucket.len() + + entry.object.len() + + entry.version_id.as_deref().map_or(0, str::len) + + entry.generation.as_deref().map_or(0, str::len) + + entry.reason.len() + + std::mem::size_of::() + + std::mem::size_of::() +} + +fn size_reconciliation_scope_key(bucket: &str, object: &str) -> String { + format!("{}:{}|{}:{}", bucket.len(), bucket, object.len(), object) +} + +fn prune_size_reconciliation(info: &mut DataUsageCacheInfo, now: u64) { + info.size_reconciliation.retain(|key, entry| { + if entry.first_seen == 0 || entry.first_seen > now { + entry.first_seen = now; + } + key == &entry.key + && entry.key.len() <= 4096 + && entry.bucket.len() <= 512 + && entry.object.len() <= 512 + && entry.version_id.as_deref().is_none_or(|value| value.len() <= 64) + && entry.generation.as_deref().is_none_or(|value| value.len() <= 64) + && entry.reason.len() <= 64 + && now.saturating_sub(entry.first_seen) <= MAX_SIZE_RECONCILIATION_AGE_SECS + }); + + while info.size_reconciliation.len() > MAX_SIZE_RECONCILIATION_ENTRIES_PER_BUCKET + || info + .size_reconciliation + .values() + .map(size_reconciliation_entry_bytes) + .sum::() + > MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET + { + let oldest = info + .size_reconciliation + .iter() + .min_by(|(left_key, left), (right_key, right)| { + left.first_seen.cmp(&right.first_seen).then_with(|| left_key.cmp(right_key)) + }) + .map(|(key, _)| key.clone()); + let Some(oldest) = oldest else { + break; + }; + info.size_reconciliation.remove(&oldest); + } +} + impl FolderScanner { fn now_secs() -> u64 { SystemTime::now() @@ -756,6 +815,60 @@ impl FolderScanner { } } + /// Apply the per-object size-resolution ledger updates in one place. The + /// scanner cache is the durable boundary; both working copies are updated + /// so an incremental publication cannot lose a debt or its resolution. + fn apply_size_reconciliation(&mut self, summary: &SizeSummary) { + let now = Self::now_secs(); + self.pending_size_reconciliation_keys + .extend(summary.size_reconciliation.iter().map(|entry| entry.key.clone())); + self.pending_size_reconciliation_scopes.extend( + summary + .reconciliation_scopes + .iter() + .map(|scope| size_reconciliation_scope_key(&scope.bucket, &scope.object)), + ); + self.pending_size_reconciliation_truncated |= summary.size_reconciliation_truncated; + + for info in [&mut self.new_cache.info, &mut self.update_cache.info] { + for incoming in &summary.size_reconciliation { + if let Some(existing) = info.size_reconciliation.get_mut(&incoming.key) { + existing.reason = incoming.reason.clone(); + existing.physical_size = incoming.physical_size; + existing.generation = incoming.generation.clone(); + existing.version_id = incoming.version_id.clone(); + existing.attempts = existing.attempts.saturating_add(1); + continue; + } + + if size_reconciliation_entry_bytes(incoming) > MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET { + continue; + } + + let mut entry = incoming.clone(); + entry.first_seen = now; + entry.attempts = 1; + info.size_reconciliation.insert(entry.key.clone(), entry); + } + } + } + + fn finish_size_reconciliation_batch(&mut self) { + let now = Self::now_secs(); + let current_keys = std::mem::take(&mut self.pending_size_reconciliation_keys); + let scopes = std::mem::take(&mut self.pending_size_reconciliation_scopes); + let truncated = std::mem::replace(&mut self.pending_size_reconciliation_truncated, false); + + for info in [&mut self.new_cache.info, &mut self.update_cache.info] { + if !truncated { + info.size_reconciliation.retain(|key, entry| { + !scopes.contains(&size_reconciliation_scope_key(&entry.bucket, &entry.object)) || current_keys.contains(key) + }); + } + prune_size_reconciliation(info, now); + } + } + fn record_scan_resume_hint(&mut self, folder: &str) { self.new_cache.info.scan_resume_after = Some(folder.to_string()); self.update_cache.info.scan_resume_after = Some(folder.to_string()); @@ -1435,6 +1548,7 @@ impl FolderScanner { abandoned_children.remove(&path_join_buf(&[&item.bucket, &item.object_path()])); apply_scanner_size_summary(into, &sz); + self.apply_size_reconciliation(&sz); into.objects += 1; object_count += 1; self.budget.record_object_scanned(); @@ -2155,6 +2269,7 @@ impl FolderScanner { } } + self.finish_size_reconciliation_batch(); done_folder(); let scanned_objects = u64::try_from(into.objects).unwrap_or(u64::MAX); emit_scanner_folder_trace(&self.root, &folder.name, scanned_objects, trace_started_at, "completed"); @@ -2240,10 +2355,17 @@ pub async fn scan_data_folder( skip_heal, local_disk, pending_heals_changed: false, + pending_size_reconciliation_keys: HashSet::new(), + pending_size_reconciliation_scopes: HashSet::new(), + pending_size_reconciliation_truncated: false, #[cfg(test)] list_path_raw_options_observer: None, }; + let now = FolderScanner::now_secs(); + prune_size_reconciliation(&mut scanner.new_cache.info, now); + prune_size_reconciliation(&mut scanner.update_cache.info, now); + // Check if context is cancelled if ctx.is_cancelled() { return Err(ScannerError::Other("Operation cancelled".to_string())); @@ -2267,7 +2389,9 @@ pub async fn scan_data_folder( new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN); new_cache.info.last_update = Some(SystemTime::now()); new_cache.info.next_cycle = cache.info.next_cycle; - let unresolved_objects = root.failed_objects > 0 || !new_cache.info.failed_objects.is_empty(); + let unresolved_objects = root.failed_objects > 0 + || !new_cache.info.failed_objects.is_empty() + || !new_cache.info.size_reconciliation.is_empty(); new_cache.info.snapshot_complete = !unresolved_objects; let had_scan_checkpoint = cache.info.scan_checkpoint.is_some() || new_cache.info.scan_checkpoint.is_some(); new_cache.info.scan_resume_after = None; @@ -2295,7 +2419,7 @@ pub async fn scan_data_folder( if root_has_progress { new_cache.replace_hashed(&root_hash, &None, &root); } - if partial_cache_is_useful(&root, pending_heals_changed) { + if partial_cache_is_useful(&root, pending_heals_changed) || !new_cache.info.size_reconciliation.is_empty() { if new_cache.root().is_some() { new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN); } diff --git a/crates/scanner/src/scanner_folder/item_actions.rs b/crates/scanner/src/scanner_folder/item_actions.rs index 09a97e42b..635637199 100644 --- a/crates/scanner/src/scanner_folder/item_actions.rs +++ b/crates/scanner/src/scanner_folder/item_actions.rs @@ -15,6 +15,7 @@ use super::*; #[cfg(test)] use rustfs_filemeta::MetadataResolutionParams; +use sha2::{Digest as _, Sha256}; /// Cached folder information for scanning #[derive(Clone, Debug)] @@ -34,6 +35,263 @@ pub(super) enum GetSizeFailureAction { HealMetadata { object: String }, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum SizeResolutionReason { + CompressedSizeUnknown, + InvalidPhysicalSize, + UnsupportedCompression, + InvalidObjectSize, + InvalidPartSize, + InvalidDeclaredSize, + SizeOverflowOrMismatch, +} + +impl SizeResolutionReason { + fn as_str(self) -> &'static str { + match self { + Self::CompressedSizeUnknown => "compressed_size_unknown", + Self::InvalidPhysicalSize => "invalid_physical_size", + Self::UnsupportedCompression => "unsupported_compression", + Self::InvalidObjectSize => "invalid_object_size", + Self::InvalidPartSize => "invalid_part_size", + Self::InvalidDeclaredSize => "invalid_declared_size", + Self::SizeOverflowOrMismatch => "size_overflow_or_mismatch", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) enum SizeResolution { + Known { logical: i64, physical: i64 }, + Unknown { physical: i64, reason: SizeResolutionReason }, + Corrupt { physical: i64, reason: SizeResolutionReason }, +} + +impl SizeResolution { + fn known_size(&self) -> Option { + match self { + Self::Known { logical, .. } => Some(*logical), + Self::Unknown { .. } | Self::Corrupt { .. } => None, + } + } +} + +fn size_reconciliation_key(oi: &ObjectInfo, reason: SizeResolutionReason) -> String { + let version = oi + .version_id + .filter(|version| !version.is_nil()) + .map(|version| version.to_string()) + .unwrap_or_default(); + let generation = oi + .data_dir + .filter(|generation| !generation.is_nil()) + .map(|generation| generation.to_string()) + .unwrap_or_default(); + // Length-prefix each component so an object key containing the separator + // cannot alias another identity. S3 keys are bounded in normal operation; + // oversized persisted values use a digest so a corrupt metadata record + // cannot grow the ledger without bound. + fn component(value: &str) -> String { + const MAX_COMPONENT_LEN: usize = 512; + if value.len() <= MAX_COMPONENT_LEN { + return format!("{}:{}", value.len(), value); + } + let digest = Sha256::digest(value.as_bytes()); + let digest = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower); + format!("hash:{}:{}", value.len(), digest) + } + format!( + "{}|{}|{}|{}|{}", + component(&oi.bucket), + component(&oi.name), + component(&version), + component(&generation), + component(reason.as_str()) + ) +} + +pub(super) fn bounded_reconciliation_field(value: &str) -> String { + const MAX_FIELD_LEN: usize = 512; + if value.len() <= MAX_FIELD_LEN { + return value.to_string(); + } + let digest = hex_simd::encode_to_string(Sha256::digest(value.as_bytes()), hex_simd::AsciiCase::Lower); + let prefix_len = MAX_FIELD_LEN - 65; + let prefix = value + .char_indices() + .take_while(|(offset, ch)| offset.saturating_add(ch.len_utf8()) <= prefix_len) + .map(|(_, ch)| ch) + .collect::(); + format!("{}~{}", prefix, digest) +} + +fn record_size_resolution(summary: &mut SizeSummary, oi: &ObjectInfo, resolution: &SizeResolution) { + match resolution { + SizeResolution::Known { .. } => {} + SizeResolution::Unknown { physical, reason } | SizeResolution::Corrupt { physical, reason } => { + summary.record_size_reconciliation(SizeReconciliationEntry { + key: size_reconciliation_key(oi, *reason), + bucket: bounded_reconciliation_field(&oi.bucket), + object: bounded_reconciliation_field(&oi.name), + version_id: oi + .version_id + .filter(|version| !version.is_nil()) + .map(|version| version.to_string()), + generation: oi + .data_dir + .filter(|generation| !generation.is_nil()) + .map(|generation| generation.to_string()), + reason: reason.as_str().to_string(), + physical_size: u64::try_from(*physical).ok(), + first_seen: 0, + attempts: 0, + }); + } + } +} + +/// Resolve the size metadata once at the scanner trust boundary. A compressed +/// -1 sentinel is valid legacy metadata, but it cannot participate in normal +/// logical-size accounting or size-filtered lifecycle rules. +pub(super) fn resolve_size(oi: &ObjectInfo) -> SizeResolution { + let physical = oi.size; + if physical < 0 { + return SizeResolution::Corrupt { + physical, + reason: SizeResolutionReason::InvalidPhysicalSize, + }; + } + + let compressed = match oi.compression_read_plan() { + Ok((_, _, compressed)) => compressed, + Err(_) => { + return SizeResolution::Corrupt { + physical, + reason: SizeResolutionReason::UnsupportedCompression, + }; + } + }; + + if oi.actual_size < -1 || (oi.actual_size == -1 && !compressed) { + return SizeResolution::Corrupt { + physical, + reason: SizeResolutionReason::InvalidObjectSize, + }; + } + + // Match ObjectInfo::get_actual_size: a positive in-memory value is the + // authoritative decoded size. Stale declared/part metadata must not turn + // an otherwise valid object into a false corruption report. + if oi.actual_size > 0 { + return SizeResolution::Known { + logical: oi.actual_size, + physical, + }; + } + + if oi + .parts + .iter() + .any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed)) + { + return SizeResolution::Corrupt { + physical, + reason: SizeResolutionReason::InvalidPartSize, + }; + } + + let declared = rustfs_utils::http::get_str(&oi.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE); + let declared = match declared { + Some(value) if value.is_empty() => { + return SizeResolution::Corrupt { + physical, + reason: SizeResolutionReason::InvalidDeclaredSize, + }; + } + Some(value) => match value.parse::() { + Ok(value) if value >= 0 => Some(value), + _ => { + return SizeResolution::Corrupt { + physical, + reason: SizeResolutionReason::InvalidDeclaredSize, + }; + } + }, + None => None, + }; + + let logical = match oi.get_actual_size() { + Ok(size) if size == -1 && compressed && declared.is_none() => { + return SizeResolution::Unknown { + physical, + reason: SizeResolutionReason::CompressedSizeUnknown, + }; + } + Ok(size) if size >= 0 => size, + Ok(_) | Err(_) => { + return SizeResolution::Corrupt { + physical, + reason: SizeResolutionReason::SizeOverflowOrMismatch, + }; + } + }; + + if compressed && logical == 0 && physical != 0 && oi.parts.is_empty() && declared.is_none() { + return SizeResolution::Corrupt { + physical, + reason: SizeResolutionReason::SizeOverflowOrMismatch, + }; + } + + SizeResolution::Known { logical, physical } +} + +fn resolve_sizes(object_infos: &[ObjectInfo]) -> Vec { + object_infos.iter().map(resolve_size).collect() +} + +fn lifecycle_rule_has_size_filter(lifecycle: &BucketLifecycleConfiguration, rule_id: &str) -> bool { + let filter_has_size = |filter: &s3s::dto::LifecycleRuleFilter| { + filter.object_size_greater_than.is_some() + || filter.object_size_less_than.is_some() + || filter + .and + .as_ref() + .is_some_and(|and| and.object_size_greater_than.is_some() || and.object_size_less_than.is_some()) + }; + lifecycle + .rules + .iter() + .find(|rule| { + if rule_id.is_empty() { + rule.id.as_deref().is_none_or(str::is_empty) + } else { + rule.id.as_deref() == Some(rule_id) + } + }) + .and_then(|rule| rule.filter.as_ref()) + .is_some_and(filter_has_size) +} + +fn lifecycle_event_allowed(resolution: &SizeResolution, event: &Event, lifecycle: &BucketLifecycleConfiguration) -> bool { + match resolution { + // Missing or invalid logical size only defers actions whose selected + // rule actually depends on that size. Time/version-only actions retain + // their existing semantics, including intrinsic events without a rule ID. + SizeResolution::Unknown { .. } | SizeResolution::Corrupt { .. } => { + !lifecycle_rule_has_size_filter(lifecycle, &event.rule_id) + } + SizeResolution::Known { .. } => true, + } +} + +/// A successful newer-noncurrent batch consumes both known and unresolved +/// versions from the retained-version alert count. The two accounting paths +/// are separate because only known sizes can contribute byte totals. +fn remaining_versions_after_queued_noncurrent(remaining_versions: usize, known_count: usize, unknown_count: usize) -> usize { + remaining_versions.saturating_sub(known_count.saturating_add(unknown_count)) +} + /// How the corrupt-metadata branch records the repair after attempting an /// MRF intent (backlog#1894 axis A). #[derive(Debug, PartialEq, Eq)] @@ -338,34 +596,48 @@ impl ScannerItem { "Scanner lifecycle evaluation started" ); + let resolved_sizes = resolve_sizes(&object_infos); + if let Some(first) = object_infos.first() { + size_summary.record_reconciliation_scope( + &bounded_reconciliation_field(&first.bucket), + &bounded_reconciliation_field(&first.name), + ); + } + for (oi, resolution) in object_infos.iter().zip(resolved_sizes.iter()) { + record_size_resolution(size_summary, oi, resolution); + } + let has_corrupt_size = resolved_sizes + .iter() + .any(|resolution| matches!(resolution, SizeResolution::Corrupt { .. })); + // `versioning_config` is resolved once per object by the caller // (`get_size`) and handed in; only `prefix_enabled` is consulted here. - let Some(lifecycle) = self.lifecycle.as_ref() else { - let mut cumulative_size = 0; - for oi in object_infos.iter() { - let actual_size = match oi.get_actual_size() { - Ok(size) => size, - Err(_) => { - warn!( - target: "rustfs::scanner::folder", - event = EVENT_SCANNER_LIFECYCLE_ACTION, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - bucket = %self.bucket, - object = %oi.name, - state = "size_lookup_failed", - "Scanner lifecycle action used fallback size" - ); + let Some(lifecycle) = self.lifecycle.clone() else { + let mut cumulative_size: i64 = 0; + for (oi, resolved_size) in object_infos.iter().zip(resolved_sizes.iter()) { + let accounting_size = match resolved_size { + SizeResolution::Known { logical, .. } => *logical, + // A valid compressed legacy sentinel has no logical size, + // but heal and replication still need to run. The + // physical size is only an input to those operations; it + // is not folded into the logical total below. + SizeResolution::Unknown { physical, .. } => { + self.heal_actions(oi, *physical, size_summary).await; + size_summary.actions_accounting_unknown(oi); + continue; + } + SizeResolution::Corrupt { .. } => { + size_summary.actions_accounting_unknown(oi); continue; } }; - let size = self.heal_actions(oi, actual_size, size_summary).await; + let size = self.heal_actions(oi, accounting_size, size_summary).await; - size_summary.actions_accounting(oi, size, actual_size); + size_summary.actions_accounting(oi, size, accounting_size); - cumulative_size += size; + cumulative_size = cumulative_size.saturating_add(size); } self.alert_excessive_versions(object_infos.len(), cumulative_size); @@ -419,25 +691,108 @@ impl ScannerItem { let mut to_delete_objs: Vec = Vec::new(); let mut noncurrent_events: Vec = Vec::new(); let mut noncurrent_accounting: Vec> = Vec::new(); + let mut noncurrent_unknown: Vec<&ObjectInfo> = Vec::new(); let mut cumulative_size = 0; let mut remaining_versions = object_infos.len(); 'eventLoop: { for (i, event) in events.iter().enumerate() { let oi = &object_infos[i]; - let actual_size = match oi.get_actual_size() { - Ok(size) => size, - Err(_) => { - warn!( - target: "rustfs::scanner::folder", - event = EVENT_SCANNER_LIFECYCLE_ACTION, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - bucket = %self.bucket, - object = %oi.name, - state = "size_lookup_failed", - "Scanner lifecycle action used fallback size" - ); - 0 + let known_size = resolved_sizes[i].known_size(); + if has_corrupt_size + && matches!( + event.action, + IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction + ) + { + // An all-version delete would also remove a corrupt + // sibling that could not be reconciled safely. + continue; + } + if !lifecycle_event_allowed(&resolved_sizes[i], event, &lifecycle) { + // An unknown logical size must not make an otherwise + // non-destructive scan disappear from heal/physical-tier + // accounting. Size-filtered or deferred events remain + // pending, so retain the version-only physical counters. + if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] { + self.heal_actions(oi, *physical, size_summary).await; + size_summary.actions_accounting_unknown(oi); + } + continue; + } + let actual_size = match known_size { + Some(size) => size, + None => { + match event.action { + IlmAction::DeleteAction + | IlmAction::DeleteRestoredAction + | IlmAction::DeleteRestoredVersionAction + | IlmAction::DeleteAllVersionsAction + | IlmAction::DelMarkerDeleteAllVersionsAction => { + let done_ilm = Metrics::time_ilm(event.action); + let trace_started_at = trace_start_instant(); + let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); + if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { + done_ilm(1)(); + if event.action == IlmAction::DeleteAllVersionsAction + || event.action == IlmAction::DelMarkerDeleteAllVersionsAction + { + remaining_versions = 0; + } + } else if matches!( + event.action, + IlmAction::DeleteAction + | IlmAction::DeleteRestoredAction + | IlmAction::DeleteRestoredVersionAction + ) { + size_summary.actions_accounting_unknown(oi); + } else { + size_summary.actions_accounting_unknown(oi); + for (j, retained) in object_infos.iter().enumerate().skip(i + 1) { + match &resolved_sizes[j] { + SizeResolution::Known { logical, .. } => PendingScannerAccounting { + object: retained, + retained_size: *logical, + expired_size: 0, + } + .apply(size_summary, &mut cumulative_size, false), + SizeResolution::Unknown { .. } => { + size_summary.actions_accounting_unknown(retained); + } + SizeResolution::Corrupt { .. } => {} + } + } + } + } + IlmAction::DeleteVersionAction => { + if let Some(opt) = object_opts.get(i) { + to_delete_objs.push(ObjectToDelete { + object_name: opt.name.clone(), + version_id: opt.version_id, + ..Default::default() + }); + noncurrent_events.push(event.clone()); + noncurrent_unknown.push(oi); + } + } + IlmAction::TransitionAction | IlmAction::TransitionVersionAction => { + let trace_started_at = trace_start_instant(); + let queued = apply_transition_rule(event, &LcEventSrc::Scanner, oi).await; + emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at); + if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) { + let done_ilm = Metrics::time_ilm(event.action); + done_ilm(1)(); + } + size_summary.actions_accounting_unknown(oi); + } + IlmAction::NoneAction | IlmAction::ActionCount => { + if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] { + self.heal_actions(oi, *physical, size_summary).await; + } + size_summary.actions_accounting_unknown(oi); + } + } + continue; } }; @@ -465,36 +820,24 @@ impl ScannerItem { done_ilm(1)(); remaining_versions = 0; } else { - PendingScannerAccounting { - object: oi, - retained_size: actual_size, - expired_size: 0, - } - .apply(size_summary, &mut cumulative_size, false); - for retained in object_infos.iter().skip(i + 1) { - let retained_size = match retained.get_actual_size() { - Ok(size) => size, - Err(_) => { - warn!( - target: "rustfs::scanner::folder", - event = EVENT_SCANNER_LIFECYCLE_ACTION, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_LIFECYCLE, - bucket = %self.bucket, - object = %retained.name, - state = "size_lookup_failed", - "Scanner lifecycle action used fallback size" - ); - 0 - } - }; + if let Some(actual_size) = known_size { PendingScannerAccounting { - object: retained, - retained_size, + object: oi, + retained_size: actual_size, expired_size: 0, } .apply(size_summary, &mut cumulative_size, false); } + for (j, retained) in object_infos.iter().enumerate().skip(i + 1) { + if let Some(retained_size) = resolved_sizes[j].known_size() { + PendingScannerAccounting { + object: retained, + retained_size, + expired_size: 0, + } + .apply(size_summary, &mut cumulative_size, false); + } + } } break 'eventLoop; } @@ -530,11 +873,13 @@ impl ScannerItem { version_id: opt.version_id, ..Default::default() }); - noncurrent_accounting.push(PendingScannerAccounting { - object: oi, - retained_size: actual_size, - expired_size: 0, - }); + if let Some(actual_size) = known_size { + noncurrent_accounting.push(PendingScannerAccounting { + object: oi, + retained_size: actual_size, + expired_size: 0, + }); + } account_now = false; } noncurrent_events.push(event.clone()); @@ -567,7 +912,7 @@ impl ScannerItem { if account_now { size_summary.actions_accounting(oi, size, actual_size); - cumulative_size += size; + cumulative_size = cumulative_size.saturating_add(size); } } } @@ -595,11 +940,20 @@ impl ScannerItem { } if record_scanner_ilm_action_if_queued(global_metrics(), action, count, queued) { done_ilm(count)(); - remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len()); + remaining_versions = remaining_versions_after_queued_noncurrent( + remaining_versions, + noncurrent_accounting.len(), + noncurrent_unknown.len(), + ); } for pending in noncurrent_accounting { pending.apply(size_summary, &mut cumulative_size, queued); } + if !queued { + for object in noncurrent_unknown { + size_summary.actions_accounting_unknown(object); + } + } } self.alert_excessive_versions(remaining_versions, cumulative_size); } @@ -948,4 +1302,394 @@ mod tests { assert_eq!(item.object_name, "object"); assert_eq!(item.object_path(), "object"); } + + #[test] + fn size_resolution_rejects_negative_overflow_and_unknown_compression() { + let compressed = |actual_size: i64, declared: Option<&str>| { + let mut user_defined = HashMap::new(); + rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); + if let Some(declared) = declared { + rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, declared.to_string()); + } + ObjectInfo { + size: 12, + actual_size, + user_defined: Arc::new(user_defined), + ..Default::default() + } + }; + + let normal = ObjectInfo { + size: 12, + actual_size: 10, + ..Default::default() + }; + assert_eq!( + resolve_size(&normal), + SizeResolution::Known { + logical: 10, + physical: 12 + } + ); + + let stale_declared_metadata = ObjectInfo { + size: 12, + actual_size: 10, + user_defined: Arc::new(HashMap::from([("x-rustfs-internal-actual-size".to_string(), "not-a-size".to_string())])), + parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + actual_size: -2, + ..Default::default() + }]), + ..Default::default() + }; + assert_eq!( + resolve_size(&stale_declared_metadata), + SizeResolution::Known { + logical: 10, + physical: 12 + } + ); + + assert_eq!( + resolve_size(&compressed(0, Some("9"))), + SizeResolution::Known { + logical: 9, + physical: 12 + } + ); + assert_eq!( + resolve_size(&compressed(-1, None)), + SizeResolution::Unknown { + physical: 12, + reason: SizeResolutionReason::CompressedSizeUnknown, + } + ); + assert!(matches!( + resolve_size(&compressed(0, Some("not-a-size"))), + SizeResolution::Corrupt { + reason: SizeResolutionReason::InvalidDeclaredSize, + .. + } + )); + assert!(matches!( + resolve_size(&ObjectInfo { + size: 12, + actual_size: -2, + ..Default::default() + }), + SizeResolution::Corrupt { .. } + )); + assert!(matches!(resolve_size(&compressed(0, Some("-1"))), SizeResolution::Corrupt { .. })); + assert!(matches!(resolve_size(&compressed(0, Some(""))), SizeResolution::Corrupt { .. })); + + let unsupported = { + let mut object = compressed(0, None); + let mut metadata = (*object.user_defined).clone(); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "unsupported".to_string()); + object.user_defined = Arc::new(metadata); + object + }; + assert!(matches!(resolve_size(&unsupported), SizeResolution::Corrupt { .. })); + + let invalid_part = { + let mut object = compressed(0, None); + object.parts = Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + size: 12, + actual_size: -2, + ..Default::default() + }]); + object + }; + assert!(matches!(resolve_size(&invalid_part), SizeResolution::Corrupt { .. })); + + let overflow = { + let mut object = compressed(0, None); + object.parts = Arc::new(vec![ + rustfs_filemeta::ObjectPartInfo { + size: 1, + actual_size: i64::MAX, + ..Default::default() + }, + rustfs_filemeta::ObjectPartInfo { + size: 1, + actual_size: 1, + ..Default::default() + }, + ]); + object + }; + assert!(matches!(resolve_size(&overflow), SizeResolution::Corrupt { .. })); + + let mismatch = compressed(0, None); + assert!(matches!(resolve_size(&mismatch), SizeResolution::Corrupt { .. })); + assert_eq!( + resolve_size(&ObjectInfo { + size: 0, + actual_size: 0, + ..Default::default() + }), + SizeResolution::Known { logical: 0, physical: 0 } + ); + } + + #[test] + fn size_resolution_records_and_replays_one_identity() { + let version_id = uuid::Uuid::new_v4(); + let generation = uuid::Uuid::new_v4(); + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "not-a-number".to_string()); + let corrupt = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: 12, + version_id: Some(version_id), + data_dir: Some(generation), + user_defined: Arc::new(metadata), + ..Default::default() + }; + + let mut summary = SizeSummary::default(); + let resolution = resolve_size(&corrupt); + record_size_resolution(&mut summary, &corrupt, &resolution); + record_size_resolution(&mut summary, &corrupt, &resolution); + assert_eq!(summary.size_reconciliation.len(), 1); + assert_eq!(summary.size_reconciliation[0].reason, "invalid_declared_size"); + assert_eq!(summary.size_reconciliation[0].physical_size, Some(12)); + + let known = ObjectInfo { + actual_size: 12, + user_defined: Arc::new(HashMap::new()), + ..corrupt.clone() + }; + record_size_resolution(&mut summary, &known, &resolve_size(&known)); + summary.record_reconciliation_scope(&known.bucket, &known.name); + assert_eq!(summary.reconciliation_scopes.len(), 1); + assert_eq!(summary.reconciliation_scopes[0].bucket, "bucket"); + } + + #[test] + fn malformed_size_has_same_ilm_accounting() { + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "invalid".to_string()); + let object = ObjectInfo { + bucket: "bucket".to_string(), + name: "object".to_string(), + size: 12, + user_defined: Arc::new(metadata), + ..Default::default() + }; + let resolution = resolve_size(&object); + let mut without_ilm = SizeSummary::default(); + let mut with_ilm = SizeSummary::default(); + record_size_resolution(&mut without_ilm, &object, &resolution); + record_size_resolution(&mut with_ilm, &object, &resolution); + assert_eq!(without_ilm.size_reconciliation, with_ilm.size_reconciliation); + assert_eq!(without_ilm.total_size, 0); + assert_eq!(with_ilm.total_size, 0); + assert!(without_ilm.tier_stats.is_empty()); + assert!(with_ilm.tier_stats.is_empty()); + } + + #[test] + fn size_resolution_parses_once_per_version() { + let objects = vec![ + ObjectInfo { + bucket: "bucket".to_string(), + name: "one".to_string(), + size: 1, + actual_size: 1, + ..Default::default() + }, + ObjectInfo { + bucket: "bucket".to_string(), + name: "two".to_string(), + size: 2, + actual_size: -2, + ..Default::default() + }, + ]; + let resolutions = resolve_sizes(&objects); + assert_eq!(resolutions.len(), objects.len()); + assert!(matches!(resolutions[0], SizeResolution::Known { logical: 1, .. })); + assert!(matches!(resolutions[1], SizeResolution::Corrupt { .. })); + } + + #[test] + fn queued_unknown_noncurrent_versions_are_removed_from_alert_count() { + assert_eq!(remaining_versions_after_queued_noncurrent(3, 1, 2), 0); + assert_eq!(remaining_versions_after_queued_noncurrent(7, 2, 1), 4); + assert_eq!(remaining_versions_after_queued_noncurrent(usize::MAX, usize::MAX, usize::MAX), 0); + } + + #[test] + fn malformed_size_blocks_size_dependent_transition_but_allows_time_only_expiry() { + let size_filtered = BucketLifecycleConfiguration { + rules: vec![s3s::dto::LifecycleRule { + status: s3s::dto::ExpirationStatus::from_static(s3s::dto::ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + id: Some("size".to_string()), + filter: Some(s3s::dto::LifecycleRuleFilter { + object_size_greater_than: Some(1), + ..Default::default() + }), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + ..Default::default() + }; + let unknown = SizeResolution::Unknown { + physical: 12, + reason: SizeResolutionReason::CompressedSizeUnknown, + }; + let size_event = Event { + action: IlmAction::DeleteAction, + rule_id: "size".to_string(), + ..Default::default() + }; + assert!(!lifecycle_event_allowed(&unknown, &size_event, &size_filtered)); + assert!(!lifecycle_event_allowed( + &unknown, + &Event { + action: IlmAction::TransitionAction, + rule_id: "size".to_string(), + ..Default::default() + }, + &size_filtered + )); + let mixed_filters = BucketLifecycleConfiguration { + rules: vec![ + size_filtered.rules[0].clone(), + s3s::dto::LifecycleRule { + status: s3s::dto::ExpirationStatus::from_static(s3s::dto::ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + id: Some("time".to_string()), + filter: None, + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }, + ], + ..Default::default() + }; + assert!(lifecycle_event_allowed( + &unknown, + &Event { + action: IlmAction::DeleteAction, + rule_id: "time".to_string(), + ..Default::default() + }, + &mixed_filters + )); + assert!(lifecycle_event_allowed( + &unknown, + &Event { + action: IlmAction::TransitionAction, + ..Default::default() + }, + &BucketLifecycleConfiguration::default() + )); + assert!(lifecycle_event_allowed( + &SizeResolution::Corrupt { + physical: 12, + reason: SizeResolutionReason::InvalidDeclaredSize, + }, + &Event { + action: IlmAction::DeleteAction, + ..Default::default() + }, + &BucketLifecycleConfiguration::default() + )); + assert!(!lifecycle_event_allowed( + &SizeResolution::Corrupt { + physical: 12, + reason: SizeResolutionReason::InvalidDeclaredSize, + }, + &Event { + action: IlmAction::DeleteAction, + rule_id: "size".to_string(), + ..Default::default() + }, + &size_filtered + )); + assert!(lifecycle_rule_has_size_filter( + &BucketLifecycleConfiguration { + rules: vec![s3s::dto::LifecycleRule { + status: s3s::dto::ExpirationStatus::from_static(s3s::dto::ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + id: None, + filter: Some(s3s::dto::LifecycleRuleFilter { + object_size_greater_than: Some(1), + ..Default::default() + }), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + ..Default::default() + }, + "" + )); + assert!(lifecycle_event_allowed( + &SizeResolution::Known { + logical: 10, + physical: 12, + }, + &Event { + action: IlmAction::DeleteAllVersionsAction, + ..Default::default() + }, + &BucketLifecycleConfiguration::default() + )); + assert!(lifecycle_event_allowed( + &unknown, + &Event { + action: IlmAction::DeleteAction, + rule_id: "time-only".to_string(), + ..Default::default() + }, + &BucketLifecycleConfiguration::default() + )); + } + + #[tokio::test] + async fn long_object_size_reconciliation_scope_uses_bounded_identity() { + let object_name = "o".repeat(600); + let mut item = scanner_item_with_prefix(""); + item.object_name = object_name.clone(); + + let mut metadata = HashMap::new(); + rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string()); + let object = ObjectInfo { + bucket: item.bucket.clone(), + name: object_name.clone(), + size: 12, + actual_size: -1, + version_id: Some(uuid::Uuid::new_v4()), + user_defined: Arc::new(metadata), + ..Default::default() + }; + let mut summary = SizeSummary::default(); + item.apply_actions(vec![object], None, VersioningConfiguration::default(), &mut summary) + .await; + + let bounded_bucket = bounded_reconciliation_field(&item.bucket); + let bounded_object = bounded_reconciliation_field(&object_name); + assert_eq!(summary.reconciliation_scopes[0].bucket, bounded_bucket); + assert_eq!(summary.reconciliation_scopes[0].object, bounded_object); + assert_eq!(summary.size_reconciliation[0].object, bounded_object); + assert_eq!(summary.versions, 1); + assert_eq!(summary.total_size, 0); + } } diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index 56d1f1ddc..393ff50cc 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -338,6 +338,9 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) { skip_heal: Arc::new(AtomicBool::new(false)), local_disk: disk, pending_heals_changed: false, + pending_size_reconciliation_keys: HashSet::new(), + pending_size_reconciliation_scopes: HashSet::new(), + pending_size_reconciliation_truncated: false, list_path_raw_options_observer: None, }; @@ -400,6 +403,66 @@ async fn test_record_failed_ttl_zero_noop() { assert!(!scanner.should_skip_failed("path2")); } +#[tokio::test] +async fn malformed_size_reconciliation_replays_after_restart() { + let (mut scanner, temp_dir) = build_test_scanner().await; + let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir); + + let entry = SizeReconciliationEntry { + key: "1:b|6:object|0:|0:".to_string(), + bucket: "b".to_string(), + object: "object".to_string(), + reason: "invalid_declared_size".to_string(), + physical_size: Some(12), + ..Default::default() + }; + let mut summary = SizeSummary::default(); + summary.record_size_reconciliation(entry.clone()); + summary.record_reconciliation_scope("b", "object"); + scanner.apply_size_reconciliation(&summary); + scanner.apply_size_reconciliation(&summary); + + assert_eq!(scanner.new_cache.info.size_reconciliation.len(), 1); + assert_eq!(scanner.update_cache.info.size_reconciliation.len(), 1); + assert_eq!(scanner.new_cache.info.size_reconciliation[&entry.key].attempts, 2); + + let encoded = rmp_serde::to_vec_named(&scanner.new_cache.info).expect("size ledger should encode"); + let decoded: crate::data_usage_define::DataUsageCacheInfo = + rmp_serde::from_slice(&encoded).expect("size ledger should decode"); + assert_eq!(decoded.size_reconciliation.len(), 1); + assert_eq!(decoded.size_reconciliation[&entry.key].reason, "invalid_declared_size"); + + let mut resolved = SizeSummary::default(); + resolved.record_reconciliation_scope("b", "object"); + scanner.apply_size_reconciliation(&resolved); + assert!(scanner.new_cache.info.size_reconciliation.is_empty()); + assert!(scanner.update_cache.info.size_reconciliation.is_empty()); +} + +#[tokio::test] +async fn malformed_size_reconciliation_clears_bounded_long_object_scope() { + let (mut scanner, temp_dir) = build_test_scanner().await; + let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir); + let long_object = "o".repeat(600); + let bounded_object = item_actions::bounded_reconciliation_field(&long_object); + let entry = SizeReconciliationEntry { + key: "long-object-key".to_string(), + bucket: "b".to_string(), + object: bounded_object, + reason: "invalid_declared_size".to_string(), + ..Default::default() + }; + let mut summary = SizeSummary::default(); + summary.record_size_reconciliation(entry); + scanner.apply_size_reconciliation(&summary); + assert_eq!(scanner.new_cache.info.size_reconciliation.len(), 1); + + let mut resolved = SizeSummary::default(); + resolved.record_reconciliation_scope("b", &long_object); + scanner.apply_size_reconciliation(&resolved); + assert!(scanner.new_cache.info.size_reconciliation.is_empty()); +} + #[test] fn test_classify_get_size_failure_marks_metadata_heal_object_path() { let temp_dir = std::env::temp_dir(); From 14e3eb787db126facbe9c3f2010c291393384d47 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 17:29:45 +0800 Subject: [PATCH 19/41] fix(heal): correct progress accounting (#6382) * fix(heal): correct progress accounting * fix(heal): atomically persist page progress * fix(heal): preserve terminal progress counters * fix(heal): make resume handoff crash safe * fix(heal): preserve resumable bucket checkpoints * fix(heal): satisfy checkpoint outcome lint * fix(heal): preserve progress status across nodes * fix(heal): stabilize progress generations * style: restore rebalance formatting * test(heal): cover cross-set baseline generation --------- Signed-off-by: houseme Co-authored-by: overtrue Co-authored-by: houseme Co-authored-by: heihutu --- .../src/cluster/rpc/peer_rest_client.rs | 4 +- crates/heal/src/heal/erasure_healer.rs | 569 ++++++++++++++++-- crates/heal/src/heal/manager.rs | 29 +- crates/heal/src/heal/progress.rs | 509 ++++++++++++++-- crates/heal/src/heal/resume.rs | 134 ++++- crates/heal/src/heal/resume/checkpoint.rs | 205 ++++++- crates/heal/src/heal/resume/tests.rs | 157 ++++- crates/heal/src/heal/storage.rs | 49 +- crates/heal/src/heal/task.rs | 10 +- crates/heal/src/heal/task/heal_bucket.rs | 55 +- crates/heal/src/heal/task/heal_erasure_set.rs | 16 +- crates/heal/src/heal/task/heal_metadata.rs | 20 +- crates/heal/src/heal/task/heal_object.rs | 16 +- crates/heal/src/heal/task/tests.rs | 96 +++ crates/heal/src/lib.rs | 2 +- .../src/generated/proto_gen/node_service.rs | 5 +- crates/protos/src/lib.rs | 19 + crates/protos/src/node.proto | 4 +- rustfs/src/admin/handlers/heal.rs | 72 ++- rustfs/src/storage/rpc/node_service.rs | 4 +- rustfs/src/storage/rpc/node_service/heal.rs | 203 ++++++- 21 files changed, 1939 insertions(+), 239 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs index edfec184f..f28a1d17e 100644 --- a/crates/ecstore/src/cluster/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/cluster/rpc/peer_rest_client.rs @@ -1090,7 +1090,9 @@ impl PeerRestClient { .await? .max_decoding_message_size(BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE); let response = match client - .background_heal_status(Request::new(BackgroundHealStatusRequest::default())) + .background_heal_status(Request::new(BackgroundHealStatusRequest { + protocol_version: rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION, + })) .await { Ok(response) => response.into_inner(), diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index a0ace01b4..92a245c92 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -13,10 +13,10 @@ // limitations under the License. use crate::heal::{ - progress::HealProgress, + progress::{HealProgress, add_bytes, increment_counter}, resume::{ - CheckpointManager, ReplacementTargetIdentity, ResumeManager, ResumeUtils, compose_key, - replacement_target_identities_match, + CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager, + ResumeUtils, compose_key, replacement_target_identities_match, }, storage::{HealStorageAPI, next_heal_listing_token}, task::{demote_to_debug_when, is_missing_object_dir_heal_result, take_failure_log_sample}, @@ -415,6 +415,9 @@ impl ErasureSetHealer { && state.successful_objects == 0 && state.failed_objects == 0 && state.skipped_objects == 0 + && state.skipped_new_versions == 0 + && state.skipped_ilm_expired == 0 + && state.processed_bytes == 0 { // schedule_retry persists the authoritative resume reset before // resetting the checkpoint. Reapply the checkpoint reset after @@ -479,6 +482,23 @@ impl ErasureSetHealer { // 2. initialize progress self.initialize_progress(buckets, &state).await; + let (baseline_known, baseline_count, baseline_size, baseline_generation) = { + let baseline = self.progress.read().await; + ( + baseline.baseline_known, + baseline.objects_total_count, + baseline.objects_total_size, + baseline.baseline_generation, + ) + }; + if baseline_known { + resume_manager + .set_progress_baseline(baseline_count, baseline_size, baseline_generation) + .await?; + checkpoint_manager + .set_progress_baseline(baseline_count, baseline_size, baseline_generation) + .await?; + } // 3. continue from checkpoint let current_bucket_index = checkpoint.current_bucket_index; @@ -488,12 +508,66 @@ impl ErasureSetHealer { let mut successful_objects = state.successful_objects; let mut failed_objects = state.failed_objects; let mut skipped_objects = state.skipped_objects; + let checkpoint_has_progress = checkpoint.baseline_known + || checkpoint.successful_objects > 0 + || checkpoint.failed_object_count > 0 + || checkpoint.skipped_object_count > 0 + || checkpoint.skipped_new_versions > 0 + || checkpoint.skipped_ilm_expired > 0 + || checkpoint.processed_bytes > 0 + || checkpoint.total_objects > 0 + || checkpoint.total_bytes > 0 + || checkpoint.baseline_generation.is_some() + || checkpoint.counter_unknown; + let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation; + let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown; + if checkpoint_has_progress { + successful_objects = checkpoint.successful_objects; + failed_objects = checkpoint.failed_object_count; + skipped_objects = checkpoint.skipped_object_count; + let restored_processed_objects = successful_objects + .checked_add(failed_objects) + .and_then(|value| value.checked_add(skipped_objects)) + .and_then(|value| value.checked_add(checkpoint.skipped_new_versions)) + .and_then(|value| value.checked_add(checkpoint.skipped_ilm_expired)); + let checkpoint_counter_overflow = restored_processed_objects.is_none(); + restored_counter_unknown |= checkpoint_counter_overflow; + processed_objects = restored_processed_objects.unwrap_or(u64::MAX); + let mut progress = self.progress.write().await; + progress.objects_scanned = processed_objects; + progress.objects_healed = successful_objects; + progress.objects_failed = failed_objects; + progress.skipped_objects = skipped_objects; + progress.skipped_new_versions = checkpoint.skipped_new_versions; + progress.skipped_ilm_expired = checkpoint.skipped_ilm_expired; + if checkpoint.baseline_known && !checkpoint_generation_mismatch { + progress.objects_total_count = checkpoint.total_objects; + progress.objects_total_size = checkpoint.total_bytes; + progress.baseline_generation = checkpoint.baseline_generation; + progress.baseline_known = true; + } + progress.bytes_processed = checkpoint.processed_bytes; + progress.counter_unknown = state.counter_unknown || checkpoint.counter_unknown; + progress.refresh_progress_percentage(); + if checkpoint_generation_mismatch || checkpoint_counter_overflow || progress.counter_unknown { + progress.mark_unknown(); + } + } + if checkpoint_generation_mismatch { + restored_counter_unknown = true; + } + if restored_counter_unknown { + checkpoint_manager.mark_counter_unknown().await?; + resume_manager.mark_counter_unknown().await?; + } let mut failed_buckets = 0u64; // 4. process remaining buckets for (bucket_idx, bucket) in buckets.iter().enumerate().skip(current_bucket_index) { // check if completed if state.completed_buckets.contains(bucket) { + checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?; + current_object_index = 0; continue; } @@ -521,13 +595,42 @@ impl ErasureSetHealer { return bucket_result; } - // update checkpoint position - checkpoint_manager.update_position(bucket_idx, current_object_index).await?; - // update progress - resume_manager - .update_progress(processed_objects, successful_objects, failed_objects, skipped_objects) + let progress_snapshot = self.progress.read().await; + let bytes_processed = progress_snapshot.bytes_processed; + let skipped_new_versions = progress_snapshot.skipped_new_versions; + let skipped_ilm_expired = progress_snapshot.skipped_ilm_expired; + let counter_unknown = progress_snapshot.counter_unknown; + drop(progress_snapshot); + // The checkpoint is the recovery authority for object progress. + // Publish its counters and fence before the resume summary so a + // crash between the two stores cannot make recovery select newer + // summary bytes with an older checkpoint ledger. + if counter_unknown { + checkpoint_manager.mark_counter_unknown().await?; + } + checkpoint_manager + .update_progress(successful_objects, failed_objects, skipped_objects, bytes_processed) .await?; + checkpoint_manager + .set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired) + .await?; + checkpoint_manager.update_position(bucket_idx, current_object_index).await?; + resume_manager + .update_progress_with_bytes( + processed_objects, + successful_objects, + failed_objects, + skipped_objects, + bytes_processed, + ) + .await?; + resume_manager + .set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired) + .await?; + if counter_unknown { + resume_manager.mark_counter_unknown().await?; + } // check cancel status if self.cancel_token.is_cancelled() { @@ -547,6 +650,7 @@ impl ErasureSetHealer { match bucket_result { Ok(_) => { resume_manager.complete_bucket(bucket).await?; + checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?; debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_BUCKET_STATE, @@ -572,7 +676,9 @@ impl ErasureSetHealer { error = %e, "Erasure set bucket heal failed" ); - // continue to next bucket, do not interrupt the whole process + // A single durable cursor and ledger cannot safely preserve + // this bucket while processing a later one. + break; } } @@ -780,20 +886,49 @@ impl ErasureSetHealer { // Per-version dedup identity — the single canonical key. let key = compose_key(&item.name, item.version_id.as_deref()); - if checkpoint.processed_objects.contains(&key) || checkpoint.skipped_objects.contains(&key) { + if checkpoint.processed_objects.contains(&key) + || checkpoint.failed_objects.contains(&key) + || checkpoint.skipped_objects.contains(&key) + { continue; } if should_skip_new_version(item.mod_time_unix_nanos, started_at_secs) { - checkpoint_manager.add_processed_object(key).await?; - *processed_objects = processed_objects.saturating_add(1); + let counter_ok = increment_counter(processed_objects); completed_in_page = completed_in_page.saturating_add(1); counter!("rustfs_heal_skipped_new_versions_total").increment(1); - { + let (outcome_record, counter_unknown) = { let mut progress = self.progress.write().await; progress.record_skipped_new_version(); progress.set_current_object(Some(format!("skipped_new: {bucket}/{}", item.name))); - progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + progress.update_object_progress( + *processed_objects, + *successful_objects, + *failed_objects, + *skipped_objects, + bytes_processed, + ); + if !counter_ok { + progress.mark_unknown(); + } + ( + CheckpointObjectOutcomeRecord { + object: key, + outcome: CheckpointObjectOutcome::Processed, + successful: progress.objects_healed, + failed: progress.objects_failed, + skipped: progress.skipped_objects, + bytes: progress.bytes_processed, + skipped_new_versions: progress.skipped_new_versions, + skipped_ilm_expired: progress.skipped_ilm_expired, + counter_unknown: progress.counter_unknown, + }, + progress.counter_unknown, + ) + }; + checkpoint_manager.record_object_outcome(outcome_record).await?; + if counter_unknown { + resume_manager.mark_counter_unknown().await?; } debug!( target: "rustfs::heal::erasure_healer", @@ -825,15 +960,41 @@ impl ErasureSetHealer { ) .await? { - checkpoint_manager.add_processed_object(key).await?; - *processed_objects = processed_objects.saturating_add(1); + let counter_ok = increment_counter(processed_objects); completed_in_page = completed_in_page.saturating_add(1); counter!("rustfs_heal_skipped_ilm_expired_total").increment(1); - { + let (outcome_record, counter_unknown) = { let mut progress = self.progress.write().await; progress.record_skipped_ilm_expired(); progress.set_current_object(Some(format!("skipped_ilm: {bucket}/{}", item.name))); - progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + progress.update_object_progress( + *processed_objects, + *successful_objects, + *failed_objects, + *skipped_objects, + bytes_processed, + ); + if !counter_ok { + progress.mark_unknown(); + } + ( + CheckpointObjectOutcomeRecord { + object: key, + outcome: CheckpointObjectOutcome::Processed, + successful: progress.objects_healed, + failed: progress.objects_failed, + skipped: progress.skipped_objects, + bytes: progress.bytes_processed, + skipped_new_versions: progress.skipped_new_versions, + skipped_ilm_expired: progress.skipped_ilm_expired, + counter_unknown: progress.counter_unknown, + }, + progress.counter_unknown, + ) + }; + checkpoint_manager.record_object_outcome(outcome_record).await?; + if counter_unknown { + resume_manager.mark_counter_unknown().await?; } debug!( target: "rustfs::heal::erasure_healer", @@ -959,11 +1120,11 @@ impl ErasureSetHealer { while let Some((key, object, version_id, result)) = page_tasks.next().await { let (object_size, result) = result; - match result { + let mut telemetry_unknown = false; + let checkpoint_outcome = match result { Ok(true) => { - *successful_objects += 1; - bytes_processed = bytes_processed.saturating_add(object_size); - checkpoint_manager.add_processed_object(key).await?; + telemetry_unknown |= !increment_counter(successful_objects); + telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size); debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -976,11 +1137,11 @@ impl ErasureSetHealer { state = "healed", "Erasure set object healed" ); + CheckpointObjectOutcome::Processed } Ok(false) => { - checkpoint_manager.add_processed_object(key).await?; - *successful_objects += 1; - bytes_processed = bytes_processed.saturating_add(object_size); + telemetry_unknown |= !increment_counter(successful_objects); + telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size); debug!( target: "rustfs::heal::erasure_healer", event = EVENT_HEAL_ERASURE_OBJECT_STATE, @@ -993,12 +1154,12 @@ impl ErasureSetHealer { state = "missing_treated_as_ok", "Erasure set missing object treated as ok" ); + CheckpointObjectOutcome::Processed } Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err), Err(Error::TransientSkip { message }) => { - *skipped_objects += 1; - bytes_processed = bytes_processed.saturating_add(object_size); - checkpoint_manager.add_skipped_object(key).await?; + telemetry_unknown |= !increment_counter(skipped_objects); + telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size); demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", { event = EVENT_HEAL_ERASURE_OBJECT_STATE, component = LOG_COMPONENT_HEAL, @@ -1011,11 +1172,11 @@ impl ErasureSetHealer { error = %message, "Erasure set object heal skipped due to transient error" }); + CheckpointObjectOutcome::Skipped } Err(err) => { - *failed_objects += 1; - bytes_processed = bytes_processed.saturating_add(object_size); - checkpoint_manager.add_failed_object(key).await?; + telemetry_unknown |= !increment_counter(failed_objects); + telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size); demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", { event = EVENT_HEAL_ERASURE_OBJECT_STATE, component = LOG_COMPONENT_HEAL, @@ -1028,15 +1189,43 @@ impl ErasureSetHealer { error = %err, "Erasure set object heal failed" }); + CheckpointObjectOutcome::Failed } - } + }; - *processed_objects += 1; + telemetry_unknown |= !increment_counter(processed_objects); completed_in_page += 1; - { + let (outcome_record, counter_unknown) = { let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("{bucket}/{object}"))); - progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed); + progress.update_object_progress( + *processed_objects, + *successful_objects, + *failed_objects, + *skipped_objects, + bytes_processed, + ); + if telemetry_unknown { + progress.mark_unknown(); + } + ( + CheckpointObjectOutcomeRecord { + object: key, + outcome: checkpoint_outcome, + successful: progress.objects_healed, + failed: progress.objects_failed, + skipped: progress.skipped_objects, + bytes: progress.bytes_processed, + skipped_new_versions: progress.skipped_new_versions, + skipped_ilm_expired: progress.skipped_ilm_expired, + counter_unknown: progress.counter_unknown, + }, + progress.counter_unknown, + ) + }; + checkpoint_manager.record_object_outcome(outcome_record).await?; + if counter_unknown { + resume_manager.mark_counter_unknown().await?; } if completed_in_page.is_multiple_of(100) { @@ -1046,16 +1235,22 @@ impl ErasureSetHealer { *current_object_index = global_obj_idx; - // Persist the authoritative cursor FIRST (points at the next page - // boundary), then prune the per-version dedup sets. Both are - // idempotent under crash: heal_object re-heals safely. - let next_cursor = if is_truncated { next_token.clone() } else { None }; - resume_manager.set_resume_cursor(next_cursor.clone()).await?; - checkpoint_manager.complete_page(bucket_index, *current_object_index).await?; + // Persist the checkpoint ledger and page position before exposing + // the next resume cursor. A crash before cursor publication keeps + // the page identities available for exact-once replay. + checkpoint_manager.advance_page(bucket_index, *current_object_index).await?; // Check if there are more pages if !is_truncated { break; } + continuation_token = next_heal_listing_token(bucket, "", next_token, is_truncated)?; + if continuation_token.is_none() { + // A truncated page without a continuation token is terminal. + // Retain its ledger until bucket completion is durable. + break; + } + resume_manager.set_resume_cursor(continuation_token.clone()).await?; + checkpoint_manager.prune_completed_page().await?; // Anti-loop guard: an empty page reported as truncated cannot advance // the cursor (there is no last identity to move past), so treat it as a @@ -1074,12 +1269,6 @@ impl ErasureSetHealer { ))); } previous_page_last = page_last; - - continuation_token = next_heal_listing_token(bucket, "", next_token, is_truncated)?; - if continuation_token.is_none() { - // Truncated but no continuation token: treat as end of listing. - break; - } } Ok(()) @@ -1088,10 +1277,66 @@ impl ErasureSetHealer { /// initialize progress tracking async fn initialize_progress(&self, _buckets: &[String], state: &crate::heal::resume::ResumeState) { let mut progress = self.progress.write().await; - progress.objects_scanned = state.total_objects; + let existing_baseline = ( + progress.objects_total_count, + progress.objects_total_size, + progress.baseline_generation, + progress.progress_state, + progress.baseline_known, + ); + let baseline_generation_mismatch = + state.baseline_known && existing_baseline.4 && state.baseline_generation != existing_baseline.2; + let use_persisted_baseline = state.baseline_known && !baseline_generation_mismatch; + progress.objects_scanned = state.processed_objects; progress.objects_healed = state.successful_objects; progress.objects_failed = state.failed_objects; - progress.bytes_processed = 0; // Resume state tracks object counts, not byte counters. + progress.skipped_objects = state.skipped_objects; + progress.skipped_new_versions = state.skipped_new_versions; + progress.skipped_ilm_expired = state.skipped_ilm_expired; + progress.bytes_processed = state.processed_bytes; + progress.counter_unknown = state.counter_unknown; + if use_persisted_baseline + || existing_baseline.0 > 0 + || existing_baseline.1 > 0 + || existing_baseline.2.is_some() + || existing_baseline.4 + { + progress.objects_total_count = if use_persisted_baseline { + state.total_objects + } else { + existing_baseline.0 + }; + progress.objects_total_size = if use_persisted_baseline { + state.total_bytes + } else { + existing_baseline.1 + }; + progress.baseline_generation = if use_persisted_baseline { + state.baseline_generation + } else { + existing_baseline.2 + }; + progress.baseline_known = use_persisted_baseline + || existing_baseline.0 > 0 + || existing_baseline.1 > 0 + || existing_baseline.2.is_some() + || existing_baseline.4; + } + progress.progress_state = if use_persisted_baseline + || existing_baseline.0 > 0 + || existing_baseline.1 > 0 + || existing_baseline.2.is_some() + || existing_baseline.4 + { + crate::heal::progress::HealProgressState::Running + } else { + crate::heal::progress::HealProgressState::Indeterminate + }; + if baseline_generation_mismatch || state.counter_unknown { + progress.mark_unknown(); + } + progress.ledger_complete = false; + progress.refresh_progress_percentage(); progress.start_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.start_time)); progress.last_update_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.last_update)); progress.set_current_object(state.current_object.clone()); @@ -1269,8 +1514,8 @@ mod resume_loop_tests { }; use crate::heal::progress::HealProgress; use crate::heal::resume::{ - CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, - compose_key, + CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE, + ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, compose_key, }; use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI}; use crate::heal::storage_api::status::BucketInfo; @@ -1410,6 +1655,7 @@ mod resume_loop_tests { list_include_lifecycle_object_info: Mutex>, replacement_target_identity_sequences: Mutex>>, fail_listing: AtomicBool, + fail_listing_buckets: Mutex>, } impl FakeStorage { @@ -1446,6 +1692,9 @@ mod resume_loop_tests { fn fail_listing(&self) { self.fail_listing.store(true, Ordering::SeqCst); } + fn fail_bucket_listing(&self, bucket: &str) { + self.fail_listing_buckets.lock().unwrap().insert(bucket.to_string()); + } } #[async_trait::async_trait] @@ -1536,7 +1785,7 @@ mod resume_loop_tests { } async fn list_objects_for_heal_page( &self, - _bucket: &str, + bucket: &str, _prefix: &str, continuation_token: Option<&str>, include_lifecycle_object_info: bool, @@ -1545,7 +1794,7 @@ mod resume_loop_tests { .lock() .unwrap() .push(include_lifecycle_object_info); - if self.fail_listing.load(Ordering::SeqCst) { + if self.fail_listing.load(Ordering::SeqCst) || self.fail_listing_buckets.lock().unwrap().contains(bucket) { return Err(Error::other("injected listing failure")); } let key = continuation_token.map(str::to_string); @@ -1923,6 +2172,49 @@ mod resume_loop_tests { assert!(state.completed_buckets.is_empty(), "the failed bucket must remain resumable"); } + #[tokio::test] + async fn bucket_failure_stops_before_a_later_bucket_checkpoint() { + let env = make_env().await; + let task_id = ResumeUtils::generate_task_id(); + let buckets = vec!["a".to_string(), "b".to_string()]; + let resume = ResumeManager::new( + env.healer.disk.clone(), + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + buckets.clone(), + ) + .await + .unwrap(); + let checkpoint = CheckpointManager::new(env.healer.disk.clone(), task_id.clone()) + .await + .unwrap(); + env.storage.fail_bucket_listing("a"); + for _ in 0..3 { + assert!(resume.schedule_retry().await.unwrap()); + } + + env.healer + .execute_heal_with_resume(&buckets, "pool_0_set_0", &resume, &checkpoint) + .await + .expect_err("the first bucket failure must keep the pass incomplete"); + let persisted = checkpoint.get_checkpoint().await; + assert_eq!(persisted.current_bucket_index, 0); + assert!(resume.get_state().await.completed_buckets.is_empty()); + + let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &task_id) + .await + .unwrap(); + let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &task_id) + .await + .unwrap(); + env.healer + .execute_heal_with_resume(&buckets, "pool_0_set_0", &resumed, &checkpoint) + .await + .expect_err("recovery must retry the earlier failed bucket"); + assert!(!resumed.get_state().await.completed); + } + #[tokio::test] async fn completed_resume_state_is_not_selected_for_a_new_heal() { let env = make_env().await; @@ -2112,8 +2404,175 @@ mod resume_loop_tests { let mut names: Vec = env.storage.calls().into_iter().map(|(n, _)| n).collect(); names.sort(); assert_eq!(names, vec!["a", "b", "c", "d"], "every object exactly once, none dropped/doubled"); - // Final page not truncated => cursor cleared. - assert_eq!(env.resume.resume_cursor().await, None); + // Keep the final page cursor until the outer loop durably completes the + // bucket, so a crash can replay only this page against its identities. + assert_eq!(env.resume.resume_cursor().await, Some("t1".to_string())); + } + + #[tokio::test] + async fn persisted_failure_waits_for_the_bounded_retry_after_page_replay() { + let env = make_env().await; + env.storage.set_page( + None, + Page { + items: vec![item("object", Some("v1"), false)], + next: None, + truncated: false, + }, + ); + env.checkpoint + .record_object_outcome(CheckpointObjectOutcomeRecord { + object: compose_key("object", Some("v1")), + outcome: CheckpointObjectOutcome::Failed, + successful: 0, + failed: 1, + skipped: 0, + bytes: 0, + skipped_new_versions: 0, + skipped_ilm_expired: 0, + counter_unknown: false, + }) + .await + .unwrap(); + env.checkpoint.advance_page(0, 1).await.unwrap(); + + let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .unwrap(); + let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .unwrap(); + + env.healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint) + .await + .expect_err("the persisted failure must schedule a bounded retry"); + assert!( + env.storage.calls().is_empty(), + "the failed identity must not be repeated in the same pass" + ); + + env.healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint) + .await + .expect("the bounded retry must heal the object"); + assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]); + let state = resumed.get_state().await; + assert_eq!(state.successful_objects, 1); + assert_eq!(state.failed_objects, 0); + } + + #[tokio::test] + async fn final_page_crash_replays_only_the_retained_page_identities() { + let env = make_env().await; + env.storage.set_page( + None, + Page { + items: vec![item("first", Some("v1"), false)], + next: Some("final-page".to_string()), + truncated: true, + }, + ); + env.storage.set_page( + Some("final-page"), + Page { + items: vec![item("last", Some("v1"), false)], + next: None, + truncated: false, + }, + ); + + let (processed, successful, failed, skipped, result) = run(&env).await; + result.expect("the bucket pass must finish before the simulated crash"); + assert_eq!((processed, successful, failed, skipped), (2, 2, 0, 0)); + + let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .unwrap(); + let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .unwrap(); + env.healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint) + .await + .expect("the retained final-page ledger must make recovery exact"); + + assert_eq!( + env.storage.calls(), + vec![ + ("first".to_string(), Some("v1".to_string())), + ("last".to_string(), Some("v1".to_string())) + ] + ); + let state = resumed.get_state().await; + assert_eq!(state.successful_objects, 2); + assert_eq!(state.processed_objects, 2); + } + + #[tokio::test] + async fn truncated_page_without_token_retains_its_replay_ledger() { + let env = make_env().await; + env.storage.set_page( + None, + Page { + items: vec![item("object", Some("v1"), false)], + next: None, + truncated: true, + }, + ); + + let (processed, successful, failed, skipped, result) = run(&env).await; + result.expect("the tokenless truncated page is a terminal page"); + assert_eq!((processed, successful, failed, skipped), (1, 1, 0, 0)); + + let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .unwrap(); + let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .unwrap(); + env.healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint) + .await + .expect("terminal-page recovery must not replay a durable identity"); + + assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]); + let state = resumed.get_state().await; + assert_eq!(state.successful_objects, 1); + assert_eq!(state.processed_objects, 1); + } + + #[tokio::test] + async fn completed_bucket_reconciles_its_final_page_checkpoint_after_crash() { + let env = make_env().await; + env.storage.set_page( + None, + Page { + items: vec![item("object", Some("v1"), false)], + next: None, + truncated: false, + }, + ); + + let (_, _, _, _, result) = run(&env).await; + result.expect("the bucket pass must finish before the simulated crash"); + env.resume.complete_bucket("b").await.unwrap(); + + let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .unwrap(); + let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id) + .await + .unwrap(); + env.healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint) + .await + .expect("recovery must finish the checkpoint transition without replaying the bucket"); + + assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]); + let checkpoint = checkpoint.get_checkpoint().await; + assert_eq!(checkpoint.current_bucket_index, 1); + assert!(checkpoint.processed_objects.is_empty()); } #[tokio::test] diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 7c98b91f5..e092520f9 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -2110,34 +2110,11 @@ impl HealManager { return None; } - let mut snapshot = HealProgress::default(); + let mut progresses = Vec::with_capacity(active_tasks.len()); for task in active_tasks { - let progress = task.get_progress().await; - snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned); - snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed); - snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed); - snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions); - snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired); - snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count); - snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size); - snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed); - snapshot.start_time = match (snapshot.start_time, progress.start_time) { - (Some(current), Some(next)) => Some(current.min(next)), - (None, next) => next, - (current, None) => current, - }; - snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) { - (Some(current), Some(next)) => Some(current.max(next)), - (None, next) => next, - (current, None) => current, - }; - if progress.current_object.is_some() { - snapshot.current_object = progress.current_object; - } + progresses.push(task.get_progress().await); } - snapshot.refresh_progress_percentage(); - snapshot.refresh_estimated_completion_time(); - Some(snapshot) + crate::heal::progress::aggregate_heal_progress(progresses) } } diff --git a/crates/heal/src/heal/progress.rs b/crates/heal/src/heal/progress.rs index d30b82ad6..7f4ba3b26 100644 --- a/crates/heal/src/heal/progress.rs +++ b/crates/heal/src/heal/progress.rs @@ -15,15 +15,91 @@ use serde::{Deserialize, Serialize}; use std::time::{Duration, SystemTime}; -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub(crate) fn stable_generation(parts: &[&[u8]]) -> u64 { + let mut hash = 0xcbf29ce484222325u64; + for part in parts { + for byte in (part.len() as u64).to_be_bytes().into_iter().chain(part.iter().copied()) { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x100000001b3); + } + } + hash +} + +#[cfg(test)] +mod stable_generation_tests { + use super::stable_generation; + + #[test] + fn stable_generation_has_a_fixed_vector() { + assert_eq!(stable_generation(&[b"rustfs", b"heal", b"42"]), 11_007_672_338_488_385_056); + } +} + +pub(crate) fn increment_counter(counter: &mut u64) -> bool { + match counter.checked_add(1) { + Some(next) => { + *counter = next; + true + } + None => { + *counter = u64::MAX; + false + } + } +} + +pub(crate) fn add_bytes(total: &mut u64, amount: u64) -> bool { + match total.checked_add(amount) { + Some(next) => { + *total = next; + true + } + None => { + *total = u64::MAX; + false + } + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] +pub enum HealProgressKind { + #[default] + Unknown, + Stage, + ObjectSweep, +} + +/// Whether the object ledger can produce a meaningful percentage. +/// +/// A zero-valued baseline is not a completed scan: it means that no complete +/// usage snapshot was available. Keep this state explicit so callers do not +/// mistake the legacy `0.0` wire value for a measured zero-percent result. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum HealProgressState { + #[default] + Unknown, + Indeterminate, + Running, + Completed, +} + +#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] pub struct HealProgress { + #[serde(default)] + pub kind: HealProgressKind, /// Objects scanned pub objects_scanned: u64, /// Objects healed pub objects_healed: u64, /// Objects failed pub objects_failed: u64, + /// Versions deferred for a later retry pass. + #[serde(default)] + pub skipped_objects: u64, /// Versions skipped because they were written after this heal started pub skipped_new_versions: u64, /// Versions skipped because lifecycle already selected them for expiry @@ -44,11 +120,38 @@ pub struct HealProgress { pub last_update_time: Option, /// Estimated completion time pub estimated_completion_time: Option, + /// Current stage number. Stage updates are intentionally independent from + /// the object ledger below. + #[serde(default)] + pub stage_current: u64, + /// Number of stages in the current task. + #[serde(default)] + pub stage_total: u64, + /// Explicitly distinguishes a missing usage baseline from measured 0%. + #[serde(default)] + pub progress_state: HealProgressState, + /// True only after the task's durable completion ledger was committed. + #[serde(default)] + pub ledger_complete: bool, + /// Generation of the usage snapshot used for the baseline, if available. + #[serde(default)] + pub baseline_generation: Option, + /// Whether the baseline was explicitly observed. This is separate from + /// the counters so a known empty scope (0 objects, 0 bytes) is not + /// confused with a legacy snapshot that omitted the baseline fields. + #[serde(default)] + pub baseline_known: bool, + /// Internal telemetry fence set when an aggregate counter overflows or + /// becomes inconsistent. It prevents a later refresh from fabricating a + /// percentage from the poisoned values. + #[serde(default)] + pub counter_unknown: bool, } impl HealProgress { pub fn new() -> Self { Self { + kind: HealProgressKind::Unknown, start_time: Some(SystemTime::now()), last_update_time: Some(SystemTime::now()), ..Default::default() @@ -56,12 +159,87 @@ impl HealProgress { } pub fn update_progress(&mut self, scanned: u64, healed: u64, failed: u64, bytes: u64) { + self.update_object_sweep_progress(scanned, healed, failed, bytes); + } + + pub fn update_object_sweep_progress(&mut self, scanned: u64, healed: u64, failed: u64, bytes: u64) { + self.kind = HealProgressKind::ObjectSweep; self.objects_scanned = scanned; self.objects_healed = healed; self.objects_failed = failed; self.bytes_processed = bytes; self.last_update_time = Some(SystemTime::now()); + let explicit_skipped = match self.skipped_new_versions.checked_add(self.skipped_ilm_expired) { + Some(value) => value, + None => { + self.mark_unknown(); + 0 + } + }; + let skipped = healed + .checked_add(failed) + .and_then(|value| value.checked_add(explicit_skipped)) + .and_then(|value| scanned.checked_sub(value)) + .unwrap_or(0); + self.update_object_progress(scanned, healed, failed, skipped, bytes); + } + + /// Update task stage progress without modifying object counters. + pub fn update_stage(&mut self, current: u64, total: u64) { + let object_sweep_active = matches!(self.kind, HealProgressKind::ObjectSweep); + if !object_sweep_active { + self.kind = HealProgressKind::Stage; + } + self.ledger_complete = false; + self.stage_current = current.min(total); + self.stage_total = total; + if object_sweep_active { + self.last_update_time = Some(SystemTime::now()); + self.refresh_progress_percentage(); + return; + } + self.progress_state = if total == 0 { + HealProgressState::Indeterminate + } else { + HealProgressState::Running + }; + self.progress_percentage = if total == 0 { + 0.0 + } else { + (current as f64 / total as f64 * 100.0).min(100.0) + }; + self.last_update_time = Some(SystemTime::now()); + } + + /// Update the disjoint object ledger. `scanned` is the number of terminal + /// object outcomes and must equal healed + failed + deferred skipped plus + /// the two terminal skip classes. Overflow is a corrupt/unknown counter + /// state, not a reason to abort a completed heal. + pub fn update_object_progress(&mut self, scanned: u64, healed: u64, failed: u64, skipped: u64, bytes: u64) { + self.kind = HealProgressKind::ObjectSweep; + // `skipped` is the transient/deferred class. The two explicit skip + // counters are terminal classifications too, so include them in the + // same ledger without making callers maintain a second aggregate. + let outcomes = healed + .checked_add(failed) + .and_then(|value| value.checked_add(skipped)) + .and_then(|value| value.checked_add(self.skipped_new_versions)) + .and_then(|value| value.checked_add(self.skipped_ilm_expired)); + self.objects_scanned = scanned; + self.objects_healed = healed; + self.objects_failed = failed; + self.skipped_objects = skipped; + self.bytes_processed = bytes; + self.last_update_time = Some(SystemTime::now()); + self.ledger_complete = false; + if outcomes != Some(scanned) { + // Telemetry corruption must not abort a heal. Preserve the + // counters for diagnostics, but do not derive a percentage from a + // double-counted or overflowing ledger. + self.mark_unknown(); + return; + } self.refresh_progress_percentage(); self.refresh_estimated_completion_time(); } @@ -69,50 +247,88 @@ impl HealProgress { pub fn set_total_baseline(&mut self, objects_total_count: u64, objects_total_size: u64) { self.objects_total_count = objects_total_count; self.objects_total_size = objects_total_size; + self.baseline_known = true; self.last_update_time = Some(SystemTime::now()); self.refresh_progress_percentage(); self.refresh_estimated_completion_time(); } + pub fn set_total_baseline_with_generation(&mut self, objects_total_count: u64, objects_total_size: u64, generation: u64) { + self.baseline_generation = Some(generation); + self.set_total_baseline(objects_total_count, objects_total_size); + } + pub fn record_skipped_new_version(&mut self) { - self.skipped_new_versions = self.skipped_new_versions.saturating_add(1); + let Some(next) = self.skipped_new_versions.checked_add(1) else { + self.mark_unknown(); + return; + }; + self.skipped_new_versions = next; self.last_update_time = Some(SystemTime::now()); self.refresh_progress_percentage(); self.refresh_estimated_completion_time(); } pub fn record_skipped_ilm_expired(&mut self) { - self.skipped_ilm_expired = self.skipped_ilm_expired.saturating_add(1); + let Some(next) = self.skipped_ilm_expired.checked_add(1) else { + self.mark_unknown(); + return; + }; + self.skipped_ilm_expired = next; self.last_update_time = Some(SystemTime::now()); self.refresh_progress_percentage(); self.refresh_estimated_completion_time(); } - fn completed_for_baseline(&self) -> u64 { + fn completed_for_baseline(&self) -> Option { self.objects_healed - .saturating_add(self.objects_failed) - .saturating_add(self.skipped_new_versions) - .saturating_add(self.skipped_ilm_expired) + .checked_add(self.objects_failed)? + .checked_add(self.skipped_objects)? + .checked_add(self.skipped_new_versions)? + .checked_add(self.skipped_ilm_expired) } pub(crate) fn refresh_progress_percentage(&mut self) { + if self.ledger_complete { + self.progress_state = HealProgressState::Completed; + self.progress_percentage = 100.0; + return; + } + if self.counter_unknown { + self.progress_state = HealProgressState::Unknown; + self.progress_percentage = 0.0; + return; + } + if !self.baseline_known { + self.progress_state = HealProgressState::Indeterminate; + self.progress_percentage = 0.0; + self.estimated_completion_time = None; + return; + } if self.objects_total_size > 0 { self.progress_percentage = ((self.bytes_processed as f64 / self.objects_total_size as f64) * 100.0).min(100.0); + self.progress_percentage = self.progress_percentage.min(99.999); + self.progress_state = HealProgressState::Running; return; } if self.objects_total_count > 0 { - let completed = self.completed_for_baseline(); + let Some(completed) = self.completed_for_baseline() else { + self.progress_state = HealProgressState::Unknown; + self.progress_percentage = 0.0; + return; + }; self.progress_percentage = ((completed as f64 / self.objects_total_count as f64) * 100.0).min(100.0); + self.progress_percentage = self.progress_percentage.min(99.999); + self.progress_state = HealProgressState::Running; return; } - - let total = self - .objects_scanned - .saturating_add(self.objects_healed) - .saturating_add(self.objects_failed); - if total > 0 { - self.progress_percentage = (self.objects_healed as f64 / total as f64) * 100.0; + if self.baseline_known { + self.progress_state = HealProgressState::Running; + self.progress_percentage = 0.0; + return; } + self.progress_state = HealProgressState::Indeterminate; + self.progress_percentage = 0.0; } pub fn set_current_object(&mut self, object: Option) { @@ -125,7 +341,11 @@ impl HealProgress { self.estimated_completion_time = None; return; }; - if self.is_completed() || !(0.0..100.0).contains(&self.progress_percentage) || self.bytes_processed == 0 { + if self.is_completed() + || self.progress_percentage <= 0.0 + || self.progress_percentage >= 100.0 + || self.bytes_processed == 0 + { self.estimated_completion_time = None; return; } @@ -142,18 +362,39 @@ impl HealProgress { } pub fn is_completed(&self) -> bool { - if self.progress_percentage >= 100.0 { - return true; - } - if self.objects_total_count > 0 || self.objects_total_size > 0 { - return false; - } + self.ledger_complete + } - self.objects_scanned > 0 && self.objects_healed.saturating_add(self.objects_failed) >= self.objects_scanned + /// Mark telemetry unknown while allowing the underlying heal operation to + /// continue. This is used for corrupt/overflowing counters at the + /// observability boundary; it must never turn a successful heal into an + /// execution error. + pub fn mark_unknown(&mut self) { + self.counter_unknown = true; + self.progress_state = HealProgressState::Unknown; + self.ledger_complete = false; + self.progress_percentage = 0.0; + self.estimated_completion_time = None; + self.last_update_time = Some(SystemTime::now()); + } + + /// Mark the object ledger terminal only after the enclosing task has + /// committed all durable resume state and cleanup fences. + pub fn mark_completed(&mut self) { + let telemetry_unknown = self.counter_unknown || self.progress_state == HealProgressState::Unknown; + self.ledger_complete = true; + if !telemetry_unknown { + self.progress_state = HealProgressState::Completed; + } + self.progress_percentage = 100.0; + self.last_update_time = Some(SystemTime::now()); + self.estimated_completion_time = None; } pub fn get_success_rate(&self) -> f64 { - let total = self.objects_healed + self.objects_failed; + let Some(total) = self.objects_healed.checked_add(self.objects_failed) else { + return 0.0; + }; if total > 0 { (self.objects_healed as f64 / total as f64) * 100.0 } else { @@ -162,6 +403,101 @@ impl HealProgress { } } +pub fn aggregate_heal_progress(progresses: impl IntoIterator) -> Option { + let mut snapshot = HealProgress::default(); + let mut found = false; + let mut has_object_sweep = false; + let mut all_object_baselines_known = true; + let mut baseline_generation = None; + let mut baseline_generation_consistent = true; + let mut all_ledgers_complete = true; + let mut counter_overflow = false; + + for progress in progresses { + found = true; + let object_sweep = matches!(progress.kind, HealProgressKind::ObjectSweep); + has_object_sweep |= object_sweep; + all_ledgers_complete &= progress.ledger_complete; + if object_sweep { + all_object_baselines_known &= progress.baseline_known; + match baseline_generation { + None => baseline_generation = Some(progress.baseline_generation), + Some(generation) => baseline_generation_consistent &= generation == progress.baseline_generation, + } + } + counter_overflow |= progress.counter_unknown || matches!(progress.progress_state, HealProgressState::Unknown); + for (target, value) in [ + (&mut snapshot.objects_scanned, progress.objects_scanned), + (&mut snapshot.objects_healed, progress.objects_healed), + (&mut snapshot.objects_failed, progress.objects_failed), + (&mut snapshot.skipped_objects, progress.skipped_objects), + (&mut snapshot.skipped_new_versions, progress.skipped_new_versions), + (&mut snapshot.skipped_ilm_expired, progress.skipped_ilm_expired), + (&mut snapshot.objects_total_count, progress.objects_total_count), + (&mut snapshot.objects_total_size, progress.objects_total_size), + (&mut snapshot.bytes_processed, progress.bytes_processed), + (&mut snapshot.stage_current, progress.stage_current), + (&mut snapshot.stage_total, progress.stage_total), + ] { + match target.checked_add(value) { + Some(sum) => *target = sum, + None => { + *target = u64::MAX; + counter_overflow = true; + } + } + } + snapshot.start_time = match (snapshot.start_time, progress.start_time) { + (Some(current), Some(next)) => Some(current.min(next)), + (None, next) => next, + (current, None) => current, + }; + snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) { + (Some(current), Some(next)) => Some(current.max(next)), + (None, next) => next, + (current, None) => current, + }; + if progress.current_object.is_some() { + snapshot.current_object = progress.current_object; + } + } + + if !found { + return None; + } + + snapshot.kind = if has_object_sweep { + HealProgressKind::ObjectSweep + } else { + HealProgressKind::Stage + }; + snapshot.baseline_known = has_object_sweep && all_object_baselines_known && baseline_generation_consistent; + snapshot.baseline_generation = if snapshot.baseline_known && baseline_generation_consistent { + baseline_generation.flatten() + } else { + None + }; + snapshot.ledger_complete = all_ledgers_complete; + snapshot.counter_unknown = counter_overflow; + if counter_overflow { + snapshot.progress_state = HealProgressState::Unknown; + snapshot.progress_percentage = if snapshot.ledger_complete { 100.0 } else { 0.0 }; + } else if snapshot.ledger_complete { + snapshot.progress_state = HealProgressState::Completed; + snapshot.progress_percentage = 100.0; + } else if has_object_sweep { + snapshot.refresh_progress_percentage(); + } else if snapshot.stage_total == 0 { + snapshot.progress_state = HealProgressState::Indeterminate; + snapshot.progress_percentage = 0.0; + } else { + snapshot.progress_state = HealProgressState::Running; + snapshot.progress_percentage = ((snapshot.stage_current as f64 / snapshot.stage_total as f64) * 100.0).min(99.999); + } + snapshot.refresh_estimated_completion_time(); + Some(snapshot) +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HealStatistics { /// Total heal tasks @@ -230,6 +566,7 @@ mod tests { assert_eq!(progress.objects_scanned, 0); assert_eq!(progress.objects_healed, 0); assert_eq!(progress.objects_failed, 0); + assert_eq!(progress.skipped_objects, 0); assert_eq!(progress.skipped_new_versions, 0); assert_eq!(progress.skipped_ilm_expired, 0); assert_eq!(progress.objects_total_count, 0); @@ -250,10 +587,8 @@ mod tests { assert_eq!(progress.objects_healed, 8); assert_eq!(progress.objects_failed, 2); assert_eq!(progress.bytes_processed, 1024); - // Progress percentage should be calculated based on healed/total - // total = scanned + healed + failed = 10 + 8 + 2 = 20 - // healed/total = 8/20 = 0.4 = 40% - assert!((progress.progress_percentage - 40.0).abs() < 0.001); + assert_eq!(progress.progress_state, HealProgressState::Indeterminate); + assert_eq!(progress.progress_percentage, 0.0); assert!(progress.last_update_time.is_some()); } @@ -262,7 +597,8 @@ mod tests { let mut progress = HealProgress::new(); progress.start_time = Some(SystemTime::now() - Duration::from_secs(10)); - progress.update_progress(100, 25, 0, 4096); + progress.set_total_baseline(100, 16384); + progress.update_progress(25, 25, 0, 4096); let eta = progress .estimated_completion_time @@ -275,7 +611,7 @@ mod tests { let mut progress = HealProgress::new(); progress.set_total_baseline(10, 8192); - progress.update_progress(100, 25, 0, 4096); + progress.update_progress(25, 25, 0, 4096); assert!((progress.progress_percentage - 50.0).abs() < 0.001); } @@ -285,7 +621,7 @@ mod tests { let mut progress = HealProgress::new(); progress.set_total_baseline(10, 0); - progress.update_progress(100, 3, 2, 0); + progress.update_progress(5, 3, 2, 0); assert!((progress.progress_percentage - 50.0).abs() < 0.001); } @@ -295,7 +631,7 @@ mod tests { let mut progress = HealProgress::new(); progress.set_total_baseline(10, 0); - progress.update_progress(100, 3, 2, 0); + progress.update_progress(5, 3, 2, 0); progress.record_skipped_new_version(); assert_eq!(progress.skipped_new_versions, 1); @@ -336,7 +672,8 @@ mod tests { fn test_heal_progress_update_progress_all_healed() { let mut progress = HealProgress::new(); // When scanned=0, healed=10, failed=0: total=10, progress = 10/10 = 100% - progress.update_progress(0, 10, 0, 2048); + progress.update_progress(10, 10, 0, 2048); + progress.mark_completed(); // All healed, should be 100% assert!((progress.progress_percentage - 100.0).abs() < 0.001); @@ -394,6 +731,7 @@ mod tests { assert_eq!(json["objectsScanned"], 10); assert_eq!(json["objectsHealed"], 8); assert_eq!(json["objectsFailed"], 2); + assert_eq!(json["skippedObjects"], 0); assert_eq!(json["skippedNewVersions"], 0); assert_eq!(json["skippedIlmExpired"], 0); assert_eq!(json["bytesProcessed"], 1024); @@ -405,6 +743,7 @@ mod tests { fn test_heal_progress_is_completed_by_percentage() { let mut progress = HealProgress::new(); progress.update_progress(10, 10, 0, 1024); + progress.mark_completed(); assert!(progress.is_completed()); } @@ -415,7 +754,7 @@ mod tests { progress.objects_scanned = 10; progress.objects_healed = 8; progress.objects_failed = 2; - // healed + failed = 8 + 2 = 10 >= scanned = 10 + progress.mark_completed(); assert!(progress.is_completed()); } @@ -455,6 +794,108 @@ mod tests { assert!((progress.get_success_rate() - 100.0).abs() < 0.001); } + #[test] + fn single_object_progress_reaches_terminal_100() { + let mut progress = HealProgress::new(); + progress.update_object_progress(1, 1, 0, 0, 128); + assert!(!progress.is_completed()); + progress.mark_completed(); + assert!(progress.is_completed()); + assert_eq!(progress.progress_percentage, 100.0); + } + + #[test] + fn progress_without_baseline_is_indeterminate() { + let mut progress = HealProgress::new(); + progress.update_object_progress(1, 1, 0, 0, 128); + assert_eq!(progress.progress_state, HealProgressState::Indeterminate); + assert_eq!(progress.progress_percentage, 0.0); + assert!(progress.estimated_completion_time.is_none()); + } + + #[test] + fn progress_retry_is_exactly_once() { + let mut progress = HealProgress::new(); + progress.set_total_baseline(1, 128); + progress.update_object_progress(1, 1, 0, 0, 128); + progress.update_object_progress(1, 1, 0, 0, 128); + assert_eq!(progress.objects_scanned, 1); + assert_eq!(progress.objects_healed, 1); + assert_eq!(progress.bytes_processed, 128); + } + + #[test] + fn progress_never_triggers_cleanup_before_terminal_ledger_empty() { + let mut progress = HealProgress::new(); + progress.progress_percentage = 100.0; + assert!(!progress.is_completed()); + progress.mark_completed(); + assert!(progress.is_completed()); + } + + #[test] + fn progress_counter_overflow_is_marked_unknown_without_aborting_completed_heal() { + let mut progress = HealProgress::new(); + progress.update_object_progress(u64::MAX, u64::MAX, 1, 0, 0); + assert_eq!(progress.progress_state, HealProgressState::Unknown); + progress.mark_completed(); + assert!(progress.is_completed()); + assert_eq!(progress.progress_state, HealProgressState::Unknown); + + let aggregate = aggregate_heal_progress([progress]).expect("progress should aggregate"); + assert!(aggregate.ledger_complete); + assert!(aggregate.counter_unknown); + assert_eq!(aggregate.progress_state, HealProgressState::Unknown); + assert_eq!(aggregate.progress_percentage, 100.0); + } + + #[test] + fn aggregate_rejects_mixed_baseline_generations() { + let progress = |generation| HealProgress { + kind: HealProgressKind::ObjectSweep, + objects_scanned: 5, + objects_total_count: 10, + progress_state: HealProgressState::Running, + baseline_generation: Some(generation), + baseline_known: true, + ..Default::default() + }; + + let aggregate = aggregate_heal_progress([progress(1), progress(2)]).expect("progress should aggregate"); + assert!(!aggregate.baseline_known); + assert_eq!(aggregate.baseline_generation, None); + assert_eq!(aggregate.progress_state, HealProgressState::Indeterminate); + assert_eq!(aggregate.progress_percentage, 0.0); + } + + #[test] + fn aggregate_accepts_multiple_sets_from_one_snapshot_generation() { + let progress = |objects_scanned| HealProgress { + kind: HealProgressKind::ObjectSweep, + objects_scanned, + objects_total_count: 10, + progress_state: HealProgressState::Running, + baseline_generation: Some(7), + baseline_known: true, + ..Default::default() + }; + + let aggregate = aggregate_heal_progress([progress(5), progress(3)]).expect("progress should aggregate"); + assert!(aggregate.baseline_known); + assert_eq!(aggregate.baseline_generation, Some(7)); + } + + #[test] + fn stage_updates_do_not_double_count_object_outcomes() { + let mut progress = HealProgress::new(); + progress.update_object_progress(2, 1, 0, 1, 256); + progress.update_stage(3, 4); + assert_eq!(progress.kind, HealProgressKind::ObjectSweep); + assert_eq!(progress.objects_scanned, 2); + assert_eq!(progress.objects_healed, 1); + assert_eq!(progress.skipped_objects, 1); + } + #[test] fn test_heal_statistics_new() { let stats = HealStatistics::new(); diff --git a/crates/heal/src/heal/resume.rs b/crates/heal/src/heal/resume.rs index b19a2090c..19d9522ae 100644 --- a/crates/heal/src/heal/resume.rs +++ b/crates/heal/src/heal/resume.rs @@ -32,7 +32,7 @@ mod gc; mod replacement; mod utils; -pub use checkpoint::{CheckpointManager, ResumeCheckpoint}; +pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint}; pub(crate) use gc::ResumeGc; pub(crate) use replacement::replacement_target_identities_match; use replacement::replacement_targets_match_identities; @@ -343,6 +343,12 @@ pub struct ResumeState { pub failed_objects: u64, /// skipped objects pub skipped_objects: u64, + /// Terminal versions skipped because they were newer than the heal start. + #[serde(default)] + pub skipped_new_versions: u64, + /// Terminal versions handed to lifecycle expiry. + #[serde(default)] + pub skipped_ilm_expired: u64, /// current bucket pub current_bucket: Option, /// current object @@ -357,6 +363,24 @@ pub struct ResumeState { pub retry_count: u32, /// max retries pub max_retries: u32, + /// Bytes accounted by the object ledger; additive for old snapshots. + #[serde(default)] + pub processed_bytes: u64, + /// Total bytes from a complete usage snapshot, when available. + #[serde(default)] + pub total_bytes: u64, + /// Generation of the usage snapshot used for the baseline. + #[serde(default)] + pub baseline_generation: Option, + /// Whether the usage baseline is known. Missing in old snapshots means + /// indeterminate rather than a measured zero baseline. + #[serde(default)] + pub baseline_known: bool, + /// Persistent telemetry fence for counter/byte overflow or corruption. + /// It must survive a restart so a saturated snapshot is never presented as + /// a measured percentage on the next resume. + #[serde(default)] + pub counter_unknown: bool, } impl ResumeState { @@ -380,6 +404,8 @@ impl ResumeState { successful_objects: 0, failed_objects: 0, skipped_objects: 0, + skipped_new_versions: 0, + skipped_ilm_expired: 0, current_bucket: None, current_object: None, completed_buckets: Vec::new(), @@ -387,6 +413,11 @@ impl ResumeState { error_message: None, retry_count: 0, max_retries: 3, + processed_bytes: 0, + total_bytes: 0, + baseline_generation: None, + baseline_known: false, + counter_unknown: false, } } @@ -415,6 +446,39 @@ impl ResumeState { self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); } + pub fn update_progress_with_bytes( + &mut self, + processed: u64, + successful: u64, + failed: u64, + skipped: u64, + processed_bytes: u64, + ) { + self.update_progress(processed, successful, failed, skipped); + self.processed_bytes = processed_bytes; + } + + pub fn set_skipped_version_counts(&mut self, new_versions: u64, ilm_expired: u64) { + self.skipped_new_versions = new_versions; + self.skipped_ilm_expired = ilm_expired; + self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + } + + pub fn set_progress_baseline(&mut self, total_objects: u64, total_bytes: u64, generation: Option) { + self.total_objects = total_objects; + self.total_bytes = total_bytes; + self.baseline_generation = generation; + // This method is called only after a complete usage snapshot has been + // validated. A complete but empty snapshot is still a known baseline. + self.baseline_known = true; + self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + } + + pub fn mark_counter_unknown(&mut self) { + self.counter_unknown = true; + self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + } + pub fn set_current_item(&mut self, bucket: Option, object: Option) { self.current_bucket = bucket; self.current_object = object; @@ -440,6 +504,7 @@ impl ResumeState { if let Some(pos) = self.pending_buckets.iter().position(|b| b == bucket) { self.pending_buckets.remove(pos); } + self.resume_cursor = None; self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); } @@ -457,6 +522,10 @@ impl ResumeState { self.successful_objects = 0; self.failed_objects = 0; self.skipped_objects = 0; + self.skipped_new_versions = 0; + self.skipped_ilm_expired = 0; + self.processed_bytes = 0; + self.counter_unknown = false; self.completed = false; // A retry re-scans every bucket from the beginning, so the version // cursor must be cleared too — otherwise the retry would resume mid-scan. @@ -479,14 +548,28 @@ impl ResumeState { } pub fn get_progress_percentage(&self) -> f64 { + if self.completed { + return 100.0; + } + if self.counter_unknown { + return 0.0; + } + if !self.baseline_known { + return 0.0; + } + if self.total_bytes > 0 { + return ((self.processed_bytes as f64 / self.total_bytes as f64) * 100.0).min(99.999); + } if self.total_objects == 0 { return 0.0; } - (self.processed_objects as f64 / self.total_objects as f64) * 100.0 + ((self.processed_objects as f64 / self.total_objects as f64) * 100.0).min(99.999) } pub fn get_success_rate(&self) -> f64 { - let total = self.successful_objects + self.failed_objects; + let Some(total) = self.successful_objects.checked_add(self.failed_objects) else { + return 0.0; + }; if total == 0 { return 0.0; } @@ -757,6 +840,14 @@ impl ResumeManager { state.successful_objects = 0; state.failed_objects = 0; state.skipped_objects = 0; + state.skipped_new_versions = 0; + state.skipped_ilm_expired = 0; + state.processed_bytes = 0; + state.total_objects = 0; + state.total_bytes = 0; + state.baseline_generation = None; + state.baseline_known = false; + state.counter_unknown = false; state.completed = false; state.completed_buckets.clear(); state.schema_version = CURRENT_RESUME_SCHEMA; @@ -841,6 +932,41 @@ impl ResumeManager { self.save_state_throttled().await } + pub async fn update_progress_with_bytes( + &self, + processed: u64, + successful: u64, + failed: u64, + skipped: u64, + processed_bytes: u64, + ) -> Result<()> { + let mut state = self.state.write().await; + state.update_progress_with_bytes(processed, successful, failed, skipped, processed_bytes); + drop(state); + self.save_state_throttled().await + } + + pub async fn set_progress_baseline(&self, total_objects: u64, total_bytes: u64, generation: Option) -> Result<()> { + let mut state = self.state.write().await; + state.set_progress_baseline(total_objects, total_bytes, generation); + drop(state); + self.save_state_throttled().await + } + + pub async fn mark_counter_unknown(&self) -> Result<()> { + let mut state = self.state.write().await; + state.mark_counter_unknown(); + drop(state); + self.save_state().await + } + + pub async fn set_skipped_version_counts(&self, new_versions: u64, ilm_expired: u64) -> Result<()> { + let mut state = self.state.write().await; + state.set_skipped_version_counts(new_versions, ilm_expired); + drop(state); + self.save_state_throttled().await + } + /// Set current item. Called once per healed object, so persistence is /// throttled: the in-memory state always updates, but the snapshot is only /// written every `PERSIST_EVERY_MUTATIONS` calls or `PERSIST_INTERVAL`. @@ -885,7 +1011,7 @@ impl ResumeManager { let mut state = self.state.write().await; state.complete_bucket(bucket); drop(state); - self.save_state_throttled().await + self.save_state().await } /// mark task completed diff --git a/crates/heal/src/heal/resume/checkpoint.rs b/crates/heal/src/heal/resume/checkpoint.rs index 18e159386..b3f097c9f 100644 --- a/crates/heal/src/heal/resume/checkpoint.rs +++ b/crates/heal/src/heal/resume/checkpoint.rs @@ -34,11 +34,31 @@ const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state"; const RESUME_CHECKPOINT_DIGEST_FILE: &str = "ahm_checkpoint.sha256"; const CHECKPOINT_PER_VERSION_SCHEMA: u32 = 5; -/// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as -/// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable -/// to the new `compose_key` identities, so a stale checkpoint is discarded. +/// Current on-disk schema version for `ResumeCheckpoint`. Schema 5 could +/// persist dedup identities without the aggregate counters needed to restore +/// them safely, so stale checkpoints are discarded and replayed. pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 6; +#[derive(Debug, Clone, Copy)] +pub enum CheckpointObjectOutcome { + Processed, + Failed, + Skipped, +} + +#[derive(Debug)] +pub struct CheckpointObjectOutcomeRecord { + pub object: String, + pub outcome: CheckpointObjectOutcome, + pub successful: u64, + pub failed: u64, + pub skipped: u64, + pub bytes: u64, + pub skipped_new_versions: u64, + pub skipped_ilm_expired: u64, + pub counter_unknown: bool, +} + /// resume checkpoint #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ResumeCheckpoint { @@ -62,6 +82,30 @@ pub struct ResumeCheckpoint { pub failed_objects: HashSet, /// skipped objects pub skipped_objects: HashSet, + /// Aggregate object ledger counters restored alongside the dedup sets. + #[serde(default)] + pub successful_objects: u64, + #[serde(default)] + pub failed_object_count: u64, + #[serde(default)] + pub skipped_object_count: u64, + #[serde(default)] + pub skipped_new_versions: u64, + #[serde(default)] + pub skipped_ilm_expired: u64, + #[serde(default)] + pub processed_bytes: u64, + #[serde(default)] + pub total_objects: u64, + #[serde(default)] + pub total_bytes: u64, + #[serde(default)] + pub baseline_generation: Option, + #[serde(default)] + pub baseline_known: bool, + /// Persistent telemetry fence for counter/byte overflow or corruption. + #[serde(default)] + pub counter_unknown: bool, /// Integrity digest over the checkpoint with this field set to `None`. /// Keeping it in the checkpoint makes the payload and its authentication /// record one CAS generation instead of two independently-written files. @@ -80,6 +124,17 @@ impl ResumeCheckpoint { processed_objects: HashSet::new(), failed_objects: HashSet::new(), skipped_objects: HashSet::new(), + successful_objects: 0, + failed_object_count: 0, + skipped_object_count: 0, + skipped_new_versions: 0, + skipped_ilm_expired: 0, + processed_bytes: 0, + total_objects: 0, + total_bytes: 0, + baseline_generation: None, + baseline_known: false, + counter_unknown: false, integrity_digest: None, } } @@ -102,6 +157,34 @@ impl ResumeCheckpoint { self.skipped_objects.insert(object); } + pub fn update_progress(&mut self, successful: u64, failed: u64, skipped: u64, bytes: u64) { + self.successful_objects = successful; + self.failed_object_count = failed; + self.skipped_object_count = skipped; + self.processed_bytes = bytes; + } + + pub fn set_progress_baseline(&mut self, total_objects: u64, total_bytes: u64, generation: Option) { + self.total_objects = total_objects; + self.total_bytes = total_bytes; + self.baseline_generation = generation; + // The caller has already validated that this is a complete snapshot; + // preserve the distinction between a known empty scope and an old + // checkpoint that omitted all baseline fields. + self.baseline_known = true; + } + + pub fn mark_counter_unknown(&mut self) { + self.counter_unknown = true; + self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + } + + pub fn set_skipped_version_counts(&mut self, new_versions: u64, ilm_expired: u64) { + self.skipped_new_versions = new_versions; + self.skipped_ilm_expired = ilm_expired; + self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + } + /// Advance past a fully-processed page: objects below `object_index` are /// skipped by position on resume, so the per-object sets no longer need /// their entries and would otherwise grow with the whole bucket. @@ -118,6 +201,17 @@ impl ResumeCheckpoint { self.update_position(0, 0); self.processed_objects.clear(); self.skipped_objects.clear(); + self.successful_objects = 0; + self.failed_object_count = 0; + self.skipped_object_count = 0; + self.skipped_new_versions = 0; + self.skipped_ilm_expired = 0; + self.processed_bytes = 0; + self.total_objects = 0; + self.total_bytes = 0; + self.baseline_generation = None; + self.baseline_known = false; + self.counter_unknown = false; self.failed_objects.clear(); } } @@ -275,10 +369,9 @@ impl CheckpointManager { }); } - // A checkpoint from an older schema stored latest-only dedup identities - // that are not comparable to the new per-version `compose_key` - // identities. Discard the stale sets and position, then stamp the - // current schema so the scan restarts cleanly. + // Older checkpoints can contain identities that are not comparable to + // the current keys or lack their corresponding aggregate counters. + // Discard the stale sets and position so the scan restarts cleanly. if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA { Self::block_invalid_snapshot(&disk, task_id).await; return Err(Error::TaskExecutionFailed { @@ -341,6 +434,17 @@ impl CheckpointManager { checkpoint.processed_objects.clear(); checkpoint.failed_objects.clear(); checkpoint.skipped_objects.clear(); + checkpoint.successful_objects = 0; + checkpoint.failed_object_count = 0; + checkpoint.skipped_object_count = 0; + checkpoint.skipped_new_versions = 0; + checkpoint.skipped_ilm_expired = 0; + checkpoint.processed_bytes = 0; + checkpoint.total_objects = 0; + checkpoint.total_bytes = 0; + checkpoint.baseline_generation = None; + checkpoint.baseline_known = false; + checkpoint.counter_unknown = false; checkpoint.current_bucket_index = 0; checkpoint.current_object_index = 0; } @@ -383,7 +487,7 @@ impl CheckpointManager { self.save_checkpoint_throttled().await } - /// Advance past a completed page and prune the per-object sets, then persist. + /// Persist a completed page position while retaining its identities. pub async fn complete_page(&self, bucket_index: usize, object_index: usize) -> Result<()> { let mut checkpoint = self.checkpoint.write().await; checkpoint.complete_page(bucket_index, object_index); @@ -391,6 +495,35 @@ impl CheckpointManager { self.save_checkpoint_throttled().await } + /// Persist the page position while retaining identities until the resume + /// cursor is durable. + pub async fn advance_page(&self, bucket_index: usize, object_index: usize) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.update_position(bucket_index, object_index); + drop(checkpoint); + self.save_checkpoint().await + } + + /// Remove the previous page's dedup identities only after its resume cursor + /// has been durably exposed. + pub async fn prune_completed_page(&self) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.processed_objects.clear(); + checkpoint.skipped_objects.clear(); + checkpoint.failed_objects.clear(); + drop(checkpoint); + self.save_checkpoint().await + } + + /// Advance to the next bucket and clear the final page identities after the + /// resume state has durably recorded the completed bucket. + pub async fn complete_bucket(&self, next_bucket_index: usize) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.complete_page(next_bucket_index, 0); + drop(checkpoint); + self.save_checkpoint().await + } + /// Reset the checkpoint to the start of the scan for a retry, then persist. pub async fn reset_for_retry(&self) -> Result<()> { let mut checkpoint = self.checkpoint.write().await; @@ -425,6 +558,62 @@ impl CheckpointManager { self.save_checkpoint_if_due().await } + /// Atomically persist an object's dedup identity with its aggregate result. + pub async fn record_object_outcome(&self, record: CheckpointObjectOutcomeRecord) -> Result<()> { + let CheckpointObjectOutcomeRecord { + object, + outcome, + successful, + failed, + skipped, + bytes, + skipped_new_versions, + skipped_ilm_expired, + counter_unknown, + } = record; + let mut checkpoint = self.checkpoint.write().await; + match outcome { + CheckpointObjectOutcome::Processed => checkpoint.add_processed_object(object), + CheckpointObjectOutcome::Failed => checkpoint.add_failed_object(object), + CheckpointObjectOutcome::Skipped => checkpoint.add_skipped_object(object), + } + checkpoint.update_progress(successful, failed, skipped, bytes); + checkpoint.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired); + if counter_unknown { + checkpoint.mark_counter_unknown(); + } + drop(checkpoint); + self.save_checkpoint_if_due().await + } + + pub async fn update_progress(&self, successful: u64, failed: u64, skipped: u64, bytes: u64) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.update_progress(successful, failed, skipped, bytes); + drop(checkpoint); + self.save_checkpoint_if_due().await + } + + pub async fn set_progress_baseline(&self, total_objects: u64, total_bytes: u64, generation: Option) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.set_progress_baseline(total_objects, total_bytes, generation); + drop(checkpoint); + self.save_checkpoint_throttled().await + } + + pub async fn mark_counter_unknown(&self) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.mark_counter_unknown(); + drop(checkpoint); + self.save_checkpoint().await + } + + pub async fn set_skipped_version_counts(&self, new_versions: u64, ilm_expired: u64) -> Result<()> { + let mut checkpoint = self.checkpoint.write().await; + checkpoint.set_skipped_version_counts(new_versions, ilm_expired); + drop(checkpoint); + self.save_checkpoint_throttled().await + } + async fn save_checkpoint_if_due(&self) -> Result<()> { let should_save = self.throttle.lock().map(|mut throttle| throttle.record()).unwrap_or(true); if !should_save { diff --git a/crates/heal/src/heal/resume/tests.rs b/crates/heal/src/heal/resume/tests.rs index fd76c9924..73d379b04 100644 --- a/crates/heal/src/heal/resume/tests.rs +++ b/crates/heal/src/heal/resume/tests.rs @@ -1296,6 +1296,7 @@ async fn test_resume_state_progress() { assert_eq!(progress, 0.0); // total_objects is 0 state.total_objects = 100; + state.baseline_known = true; let progress = state.get_progress_percentage(); assert_eq!(progress, 10.0); } @@ -1475,6 +1476,40 @@ fn test_checkpoint_object_sets_dedupe_and_prune() { assert!(checkpoint.failed_objects.is_empty()); } +#[tokio::test] +async fn checkpoint_page_commit_keeps_ledger_until_cursor_is_durable() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let checkpoint = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap(); + + checkpoint + .record_object_outcome(CheckpointObjectOutcomeRecord { + object: "bucket/object:v1".to_string(), + outcome: CheckpointObjectOutcome::Processed, + successful: 1, + failed: 0, + skipped: 0, + bytes: 128, + skipped_new_versions: 0, + skipped_ilm_expired: 0, + counter_unknown: false, + }) + .await + .unwrap(); + checkpoint.advance_page(0, 1).await.unwrap(); + + let reloaded = CheckpointManager::load_from_disk(disk.clone(), &task_id).await.unwrap(); + let snapshot = reloaded.get_checkpoint().await; + assert_eq!(snapshot.current_object_index, 1); + assert_eq!(snapshot.successful_objects, 1); + assert_eq!(snapshot.processed_bytes, 128); + assert!(snapshot.processed_objects.contains("bucket/object:v1")); + + checkpoint.prune_completed_page().await.unwrap(); + let reloaded = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap(); + assert!(reloaded.get_checkpoint().await.processed_objects.is_empty()); +} + #[test] fn test_checkpoint_loads_legacy_vec_format() { // Checkpoints written before the HashSet migration stored the object @@ -1568,14 +1603,14 @@ async fn test_resumestate_schema_v0_discarded_on_load() { } #[tokio::test] -async fn test_checkpoint_schema_v4_discarded_on_load() { +async fn test_checkpoint_schema_v5_discarded_on_load() { let (temp_dir, disk) = schema_test_disk().await; - // The previous checkpoint schema is unsafe once its paired resume - // state is discarded: retaining either position would skip work. + // Schema v5 can persist failed identities without the aggregate counters + // that make those identities safe to deduplicate after an upgrade. let task_id = "00000000-0000-4000-8000-000000000002"; let legacy = r#"{ - "schema_version": 4, + "schema_version": 5, "task_id": "00000000-0000-4000-8000-000000000002", "checkpoint_time": 1700000000, "current_bucket_index": 2, @@ -1665,6 +1700,120 @@ async fn current_normal_resume_schema_preserves_progress() { temp_dir.close().expect("remove schema test directory"); } +#[test] +fn progress_checkpoint_restores_bytes_and_generation() { + let mut checkpoint = ResumeCheckpoint::new("progress-checkpoint".to_string()); + checkpoint.set_progress_baseline(9, 4096, Some(77)); + checkpoint.update_progress(4, 1, 2, 2048); + checkpoint.set_skipped_version_counts(3, 1); + checkpoint.mark_counter_unknown(); + + let restored: ResumeCheckpoint = + serde_json::from_slice(&serde_json::to_vec(&checkpoint).expect("serialize checkpoint")).expect("deserialize checkpoint"); + assert_eq!(restored.processed_bytes, 2048); + assert_eq!(restored.total_objects, 9); + assert_eq!(restored.total_bytes, 4096); + assert_eq!(restored.baseline_generation, Some(77)); + assert!(restored.baseline_known); + assert_eq!(restored.skipped_new_versions, 3); + assert_eq!(restored.skipped_ilm_expired, 1); + assert!(restored.counter_unknown); +} + +#[test] +fn old_progress_schema_migrates_missing_fields_to_unknown() { + let state = ResumeState::new( + "legacy-progress".to_string(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + Vec::new(), + ); + let mut value = serde_json::to_value(state).expect("serialize legacy-compatible state"); + let object = value.as_object_mut().expect("state must be an object"); + for field in [ + "processed_bytes", + "total_bytes", + "baseline_generation", + "baseline_known", + "skipped_new_versions", + "skipped_ilm_expired", + ] { + object.remove(field); + } + object.insert("total_objects".to_string(), serde_json::json!(10)); + object.insert("processed_objects".to_string(), serde_json::json!(5)); + let restored: ResumeState = serde_json::from_value(value).expect("deserialize old progress state"); + assert_eq!(restored.processed_bytes, 0); + assert_eq!(restored.total_bytes, 0); + assert_eq!(restored.baseline_generation, None); + assert!(!restored.baseline_known, "missing baseline must remain unknown"); + assert_eq!(restored.get_progress_percentage(), 0.0); + assert_eq!(restored.skipped_new_versions, 0); + assert_eq!(restored.skipped_ilm_expired, 0); +} + +#[test] +fn progress_counter_unknown_survives_resume_round_trip() { + let mut state = ResumeState::new( + "overflow-progress".to_string(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + Vec::new(), + ); + state.mark_counter_unknown(); + + let restored: ResumeState = + serde_json::from_slice(&serde_json::to_vec(&state).expect("serialize resume state")).expect("deserialize resume state"); + assert!(restored.counter_unknown); +} + +#[tokio::test] +async fn checkpoint_progress_survives_a_torn_resume_summary_write() { + let (_temp_dir, disk) = schema_test_disk().await; + let task_id = ResumeUtils::generate_task_id(); + let _resume = ResumeManager::new( + disk.clone(), + task_id.clone(), + "erasure_set".to_string(), + "pool_0_set_0".to_string(), + vec!["bucket".to_string()], + ) + .await + .expect("resume state should persist"); + let checkpoint = CheckpointManager::new(disk.clone(), task_id.clone()) + .await + .expect("checkpoint should persist"); + + // This is the ordering used by the erasure-set loop: the checkpoint is + // durable before the summary write. Stop here to model a crash in the + // inter-store window and verify that the recovery authority retains the + // telemetry fence and bytes. + checkpoint + .update_progress(3, 0, 0, 1024) + .await + .expect("checkpoint progress should persist"); + checkpoint.mark_counter_unknown().await.expect("unknown fence should persist"); + checkpoint + .update_position(0, 3) + .await + .expect("checkpoint position should persist"); + + let restored_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id) + .await + .expect("checkpoint should reload") + .get_checkpoint() + .await; + let restored_resume = ResumeManager::load_from_disk(disk, &task_id) + .await + .expect("resume summary should reload") + .get_state() + .await; + assert!(restored_checkpoint.counter_unknown); + assert_eq!(restored_checkpoint.processed_bytes, 1024); + assert_eq!(restored_checkpoint.current_object_index, 3); + assert!(!restored_resume.counter_unknown, "summary is intentionally the torn/older store"); +} + #[tokio::test] async fn future_resume_and_checkpoint_schemas_are_rejected() { let (temp_dir, disk) = schema_test_disk().await; diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index de3cd149a..a3e7c0cdf 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use tracing::{debug, error, warn}; +use super::progress::stable_generation; use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached}; use super::storage_api::storage::{ BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _, @@ -34,6 +35,9 @@ pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader}; pub struct HealBucketUsageBaseline { pub objects_count: u64, pub bytes: u64, + /// Stable identity of the validated usage snapshot and selected scope. + /// `None` is retained for test/legacy providers that cannot expose one. + pub generation: Option, } pub struct HealLifecycleExpiryContext { @@ -785,11 +789,52 @@ impl HealStorageAPI for ECStoreHealStorage { let mut baseline = HealBucketUsageBaseline::default(); for bucket in buckets { if let Some(usage) = info.buckets_usage.get(bucket) { - baseline.objects_count = baseline.objects_count.saturating_add(usage.objects_count); - baseline.bytes = baseline.bytes.saturating_add(usage.size); + baseline.objects_count = match baseline.objects_count.checked_add(usage.objects_count) { + Some(total) => total, + // A corrupt/overflowing usage snapshot is not a usable + // denominator. Leave progress indeterminate instead of + // turning saturation into a plausible percentage. + None => return Ok(None), + }; + baseline.bytes = match baseline.bytes.checked_add(usage.size) { + Some(total) => total, + None => return Ok(None), + }; } } + let identity = info.snapshot_identity(); + let mut canonical = Vec::new(); + match identity.last_update { + Some(last_update) => { + canonical.push(1); + canonical.extend_from_slice( + &last_update + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + .to_be_bytes(), + ); + } + None => canonical.push(0), + } + for value in [identity.scanner_cycle, identity.scanner_epoch] { + match value { + Some(value) => { + canonical.push(1); + canonical.extend_from_slice(&value.to_be_bytes()); + } + None => canonical.push(0), + } + } + let mut scope = buckets.to_vec(); + scope.sort_unstable(); + for bucket in scope { + canonical.extend_from_slice(&(bucket.len() as u64).to_be_bytes()); + canonical.extend_from_slice(bucket.as_bytes()); + } + baseline.generation = Some(stable_generation(&[&canonical])); + Ok(Some(baseline)) } diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 5a047c4dd..0d142e15d 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -649,7 +649,7 @@ impl HealTask { let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("skipped: {bucket}/{object}"))); - progress.update_progress(0, 1, 0, 0); + progress.update_stage(1, 1); Ok(()) } @@ -733,7 +733,7 @@ impl HealTask { "Heal object skipped for data usage cache after transient error" ); let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); true } @@ -757,7 +757,7 @@ impl HealTask { ); let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("skipped: {bucket}/{object}"))); - progress.update_progress(4, 4, 0, 0); + progress.update_stage(4, 4); true } @@ -831,6 +831,10 @@ impl HealTask { match &result { Ok(_) => { + // A stage can reach its final step before the durable resume + // ledger and cleanup fences commit. Publish terminal 100 only + // after the enclosing operation has returned success. + self.progress.write().await.mark_completed(); let mut status = self.status.write().await; *status = HealTaskStatus::Completed; demote_to_debug_when!(self.heal_type.is_per_object(), info, target: "rustfs::heal::task", { diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index 7c80fa75a..8a4a28476 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -13,6 +13,7 @@ // limitations under the License. /// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline use super::*; +use crate::heal::progress::{add_bytes, increment_counter, stable_generation}; impl HealTask { pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> { @@ -32,7 +33,7 @@ impl HealTask { { let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("bucket: {bucket}"))); - progress.update_progress(0, 3, 0, 0); + progress.update_stage(0, 3); } // Step 1: Check if bucket exists @@ -66,7 +67,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(1, 3, 0, 0); + progress.update_stage(1, 3); } // Step 2: Perform bucket heal using ecstore @@ -122,7 +123,7 @@ impl HealTask { if !self.options.recursive { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } Ok(()) } @@ -142,7 +143,7 @@ impl HealTask { ); { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } Err(Error::TaskExecutionFailed { message: format!("Failed to heal bucket {bucket}: {e}"), @@ -245,6 +246,7 @@ impl HealTask { let mut scanned = 0u64; let mut healed = 0u64; let mut failed = 0u64; + let mut skipped = 0u64; let mut retryable_failed = 0u64; let mut permanent_failed = 0u64; let mut bytes = 0u64; @@ -286,16 +288,14 @@ impl HealTask { let mut retry = Vec::with_capacity(pending.len()); for item in pending { self.check_control_flags().await?; + let mut telemetry_unknown = false; let object = item.name.as_str(); - if retry_attempt == 0 { - scanned = scanned.saturating_add(1); - } { let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("{bucket}/{object}"))); - progress.update_progress(scanned, healed, failed, bytes); } + let mut terminal_outcome = true; let error = match self .await_with_control( self.storage @@ -304,13 +304,13 @@ impl HealTask { .await { Ok((result, None)) => { - healed = healed.saturating_add(1); - bytes = bytes.saturating_add(u64::try_from(result.object_size).unwrap_or_default()); + telemetry_unknown |= !increment_counter(&mut healed); + telemetry_unknown |= !add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX)); self.record_result_item(result).await; None } Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => { - healed = healed.saturating_add(1); + telemetry_unknown |= !increment_counter(&mut healed); debug!( target: "rustfs::heal::task", event = EVENT_HEAL_BUCKET_RESULT, @@ -329,6 +329,7 @@ impl HealTask { if let Some(err) = error { if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) { + telemetry_unknown |= !increment_counter(&mut skipped); warn!( target: "rustfs::heal::task", event = EVENT_HEAL_BUCKET_RESULT, @@ -342,6 +343,7 @@ impl HealTask { "Heal bucket object repair skipped due to transient metadata error" ); } else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES { + terminal_outcome = false; debug!( target: "rustfs::heal::task", event = EVENT_HEAL_BUCKET_RESULT, @@ -357,7 +359,7 @@ impl HealTask { ); retry.push(item); } else { - failed = failed.saturating_add(1); + telemetry_unknown |= !increment_counter(&mut failed); if err.is_recoverable_heal() { retryable_failed = retryable_failed.saturating_add(1); } else { @@ -383,8 +385,19 @@ impl HealTask { } } + if terminal_outcome { + telemetry_unknown |= !increment_counter(&mut scanned); + } + + if !terminal_outcome { + continue; + } + let mut progress = self.progress.write().await; - progress.update_progress(scanned, healed, failed, bytes); + progress.update_object_progress(scanned, healed, failed, skipped, bytes); + if telemetry_unknown { + progress.mark_unknown(); + } } pending = retry; retry_attempt = retry_attempt.saturating_add(1); @@ -432,6 +445,9 @@ impl HealTask { } pub(super) async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> { + if matches!(self.options.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal) { + return Ok(()); + } let baseline = match self .await_with_control(self.storage.erasure_set_usage_baseline(buckets)) .await @@ -442,9 +458,18 @@ impl HealTask { Err(_) => return Ok(()), }; - let HealBucketUsageBaseline { objects_count, bytes } = baseline; + let HealBucketUsageBaseline { + objects_count, + bytes, + generation, + } = baseline; + let generation = generation.map(|snapshot_generation| stable_generation(&[&snapshot_generation.to_be_bytes()])); let mut progress = self.progress.write().await; - progress.set_total_baseline(objects_count, bytes); + if let Some(generation) = generation { + progress.set_total_baseline_with_generation(objects_count, bytes, generation); + } else { + progress.set_total_baseline(objects_count, bytes); + } Ok(()) } } diff --git a/crates/heal/src/heal/task/heal_erasure_set.rs b/crates/heal/src/heal/task/heal_erasure_set.rs index 7b25bcba2..d983e9b96 100644 --- a/crates/heal/src/heal/task/heal_erasure_set.rs +++ b/crates/heal/src/heal/task/heal_erasure_set.rs @@ -32,7 +32,7 @@ impl HealTask { { let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("erasure_set: {} ({} buckets)", set_disk_id, buckets.len()))); - progress.update_progress(0, 4, 0, 0); + progress.update_stage(0, 4); } let is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty(); @@ -248,7 +248,7 @@ impl HealTask { ); { let mut progress = self.progress.write().await; - progress.update_progress(4, 4, 0, 0); + progress.update_stage(4, 4); } return Err(Error::TaskExecutionFailed { message: format!("Failed to heal disk format for {set_disk_id}: {error}"), @@ -304,7 +304,7 @@ impl HealTask { ); { let mut progress = self.progress.write().await; - progress.update_progress(4, 4, 0, 0); + progress.update_stage(4, 4); } return Err(Error::TaskExecutionFailed { message: format!("Failed to heal disk format for {set_disk_id}: {e}"), @@ -314,7 +314,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(1, 4, 0, 0); + progress.update_stage(1, 4); } // The rebuilt disks are formatted now: mark them as healing so @@ -343,7 +343,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(2, 4, 0, 0); + progress.update_stage(2, 4); } // Step 3: Heal bucket structure @@ -427,7 +427,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(3, 4, 0, 0); + progress.update_stage(3, 4); } // Step 4: Execute erasure set heal with resume @@ -470,9 +470,7 @@ impl HealTask { }; { - let mut progress = self.progress.write().await; - let bytes_processed = progress.bytes_processed; - progress.update_progress(4, 4, 0, bytes_processed); + self.progress.write().await.update_stage(4, 4); } match result { diff --git a/crates/heal/src/heal/task/heal_metadata.rs b/crates/heal/src/heal/task/heal_metadata.rs index 55f3a87df..3b2129288 100644 --- a/crates/heal/src/heal/task/heal_metadata.rs +++ b/crates/heal/src/heal/task/heal_metadata.rs @@ -32,7 +32,7 @@ impl HealTask { { let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("metadata: {bucket}/{object}"))); - progress.update_progress(0, 3, 0, 0); + progress.update_stage(0, 3); } // Step 1: Check if object exists @@ -74,7 +74,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(1, 3, 0, 0); + progress.update_stage(1, 3); } // Step 2: Perform metadata heal using ecstore @@ -122,7 +122,7 @@ impl HealTask { ); { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } return Err(Error::TaskExecutionFailed { message: format!("Failed to heal metadata {bucket}/{object}: {e}"), @@ -145,7 +145,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } self.record_result_item(result).await; Ok(()) @@ -167,7 +167,7 @@ impl HealTask { ); { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } Err(Error::TaskExecutionFailed { message: format!("Failed to heal metadata {bucket}/{object}: {e}"), @@ -194,7 +194,7 @@ impl HealTask { { let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("ec_decode: {bucket}/{object}"))); - progress.update_progress(0, 3, 0, 0); + progress.update_stage(0, 3); } // Step 1: Check if object exists @@ -236,7 +236,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(1, 3, 0, 0); + progress.update_stage(1, 3); } // Step 2: Perform EC decode heal using ecstore @@ -284,7 +284,7 @@ impl HealTask { ); { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } return Err(Error::TaskExecutionFailed { message: format!("Failed to heal EC decode {bucket}/{object}: {e}"), @@ -309,7 +309,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, object_size); + progress.update_object_progress(1, 1, 0, 0, object_size); } self.record_result_item(result).await; Ok(()) @@ -331,7 +331,7 @@ impl HealTask { ); { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } Err(Error::TaskExecutionFailed { message: format!("Failed to heal EC decode {bucket}/{object}: {e}"), diff --git a/crates/heal/src/heal/task/heal_object.rs b/crates/heal/src/heal/task/heal_object.rs index 14d908a1d..037bc3d41 100644 --- a/crates/heal/src/heal/task/heal_object.rs +++ b/crates/heal/src/heal/task/heal_object.rs @@ -36,7 +36,7 @@ impl HealTask { { let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("{bucket}/{object}"))); - progress.update_progress(0, 4, 0, 0); + progress.update_stage(0, 4); } // Step 1: Check if object exists and get metadata @@ -132,7 +132,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(1, 3, 0, 0); + progress.update_stage(1, 3); } // Step 2: directly call ecstore to perform heal @@ -187,7 +187,7 @@ impl HealTask { ); { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } return Ok(()); } @@ -207,7 +207,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } if Self::should_return_typed_heal_error(&e) { @@ -249,7 +249,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, object_size); + progress.update_object_progress(1, 1, 0, 0, object_size); } self.record_result_item(result).await; Ok(()) @@ -275,7 +275,7 @@ impl HealTask { ); { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } return Ok(()); } @@ -295,7 +295,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(3, 3, 0, 0); + progress.update_stage(3, 3); } if Self::should_return_typed_heal_error(&e) { @@ -414,7 +414,7 @@ impl HealTask { { let mut progress = self.progress.write().await; - progress.update_progress(4, 4, 0, object_size); + progress.update_object_progress(1, 1, 0, 0, object_size); } self.record_result_item(result).await; Ok(()) diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index f2b442205..4d9635d01 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -22,6 +22,7 @@ use std::sync::Mutex; use tempfile::TempDir; use super::super::storage_api::status::BucketInfo; +use crate::heal::progress::{HealProgressState, aggregate_heal_progress}; #[tokio::test] async fn retry_request_carries_remaining_timeout_budget() { @@ -2124,6 +2125,7 @@ async fn erasure_set_heal_applies_usage_baseline_to_progress() { usage_baseline: Mutex::new(Some(HealBucketUsageBaseline { objects_count: 10, bytes: 8, + generation: Some(1), })), ..Default::default() }); @@ -2147,10 +2149,104 @@ async fn erasure_set_heal_applies_usage_baseline_to_progress() { let progress = task.get_progress().await; assert_eq!(progress.objects_total_count, 10); assert_eq!(progress.objects_total_size, 8); + assert!(progress.baseline_generation.is_some()); + assert!(progress.baseline_known); assert_eq!(progress.bytes_processed, 2); assert!((progress.progress_percentage - 25.0).abs() < 0.001); } +#[tokio::test] +async fn erasure_sets_from_one_usage_snapshot_share_baseline_generation() { + let storage: Arc = Arc::new(MockStorage { + usage_baseline: Mutex::new(Some(HealBucketUsageBaseline { + objects_count: 10, + bytes: 8, + generation: Some(7), + })), + ..Default::default() + }); + let buckets = vec!["bucket-a".to_string()]; + let task_for_set = |set_disk_id: &str| { + HealTask::from_request( + HealRequest::new( + HealType::ErasureSet { + buckets: buckets.clone(), + set_disk_id: set_disk_id.to_string(), + }, + HealOptions::default(), + HealPriority::Normal, + ), + storage.clone(), + ) + }; + let first = task_for_set("pool_0_set_0"); + let second = task_for_set("pool_0_set_1"); + + first + .apply_erasure_set_usage_baseline(&buckets) + .await + .expect("first baseline"); + second + .apply_erasure_set_usage_baseline(&buckets) + .await + .expect("second baseline"); + first.progress.write().await.update_object_progress(0, 0, 0, 0, 0); + second.progress.write().await.update_object_progress(0, 0, 0, 0, 0); + let first = first.get_progress().await; + let second = second.get_progress().await; + + let expected_generation = first.baseline_generation; + assert!(expected_generation.is_some()); + assert_eq!(second.baseline_generation, expected_generation); + let aggregate = aggregate_heal_progress([first, second]).expect("aggregate progress"); + assert!(aggregate.baseline_known); + assert_eq!(aggregate.baseline_generation, expected_generation); + assert_eq!(aggregate.progress_state, HealProgressState::Running); +} + +#[tokio::test] +async fn erasure_set_disk_walk_keeps_cluster_usage_baseline_indeterminate() { + for (scan_mode, source) in [ + (HealScanMode::Deep, HealRequestSource::Admin), + (HealScanMode::Normal, HealRequestSource::AutoHeal), + ] { + let temp = TempDir::new().expect("temporary directory should be created"); + let disk = make_resume_disk(&temp).await; + let storage = Arc::new(MockStorage { + resume_disk: Mutex::new(Some(disk)), + usage_baseline: Mutex::new(Some(HealBucketUsageBaseline { + objects_count: 10, + bytes: 8, + generation: Some(1), + })), + ..Default::default() + }); + let mut request = HealRequest::new( + HealType::ErasureSet { + buckets: vec!["bucket-a".to_string()], + set_disk_id: "pool_0_set_0".to_string(), + }, + HealOptions { + scan_mode, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ); + request.source = source; + let task = HealTask::from_request(request, storage); + + task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string()) + .await + .expect("erasure set heal should complete"); + + let progress = task.get_progress().await; + assert!(!progress.baseline_known); + assert_eq!(progress.baseline_generation, None); + assert_eq!(progress.progress_state, HealProgressState::Indeterminate); + } +} + #[tokio::test] async fn erasure_set_heal_ignores_usage_baseline_errors() { let temp = TempDir::new().expect("temporary directory should be created"); diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index 29444d0f5..a60946229 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -19,7 +19,7 @@ pub use error::{Error, Result}; pub use heal::{ HealManager, HealOperationsSnapshot, HealOptions, HealPriority, HealPriorityCounts, HealRequest, HealSourceCounts, HealType, channel::HealChannelProcessor, - progress::HealProgress, + progress::{HealProgress, aggregate_heal_progress}, resume::{ReplacementRecoveryRecord, ReplacementRecoveryState, ResumeUtils}, }; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index 3885be4a3..3d9fa01d9 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -1215,7 +1215,10 @@ pub struct ScannerActivityResponse { pub dirty_usage_pending: bool, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] -pub struct BackgroundHealStatusRequest {} +pub struct BackgroundHealStatusRequest { + #[prost(uint32, tag = "1")] + pub protocol_version: u32, +} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BackgroundHealStatusResponse { #[prost(bool, tag = "1")] diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index ccfb8197d..f4f0c44d8 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -170,6 +170,7 @@ pub fn internode_rpc_max_message_size() -> usize { pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SIZE + 1024; pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 3; pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1; +pub const BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION: u32 = 2; pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v3\0"; pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0"; pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0"; @@ -2346,10 +2347,28 @@ pub async fn evict_failed_connection_with_log_level(addr: &str, log_level: Conne #[cfg(test)] mod tests { use super::*; + use prost::Message as _; use std::sync::Mutex; static INTERNODE_RPC_MSGPACK_ONLY_ENV_LOCK: Mutex<()> = Mutex::new(()); + #[derive(Clone, PartialEq, prost::Message)] + struct BackgroundHealStatusRequestV1 {} + + #[test] + fn background_heal_status_request_remains_rolling_upgrade_compatible() { + let current = proto_gen::node_service::BackgroundHealStatusRequest { + protocol_version: BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION, + }; + let encoded = current.encode_to_vec(); + BackgroundHealStatusRequestV1::decode(encoded.as_slice()).expect("v1 server should ignore the version field"); + + let encoded = BackgroundHealStatusRequestV1 {}.encode_to_vec(); + let decoded = proto_gen::node_service::BackgroundHealStatusRequest::decode(encoded.as_slice()) + .expect("v2 server should accept a v1 request"); + assert_eq!(decoded.protocol_version, 0); + } + #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] struct CompatPayloadField { message: &'static str, diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index 70ca93565..38812b00f 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -846,7 +846,9 @@ message ScannerActivityResponse { bool dirty_usage_pending = 9; } -message BackgroundHealStatusRequest {} +message BackgroundHealStatusRequest { + uint32 protocol_version = 1; +} message BackgroundHealStatusResponse { bool success = 1; diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 9ae59353b..28999fa94 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -20,7 +20,7 @@ use crate::admin::storage_api::bucket::utils::is_valid_object_prefix; use crate::server::ADMIN_PREFIX; use crate::server::RemoteAddr; use crate::storage::rpc::node_service::heal::{ - HealControlCoordinator, NodeHealProgress, NodeHealStatusSnapshot, capture_node_heal_status, decode_node_heal_status, + HealControlCoordinator, NodeHealStatusSnapshot, capture_node_heal_status, decode_node_heal_status, decode_node_replacement_recovery_status, heal_control_coordinator, heal_topology_fingerprint, }; use bytes::Bytes; @@ -298,14 +298,7 @@ fn background_heal_runtime_state( } } -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct BackgroundHealProgress { - objects_scanned: u64, - objects_healed: u64, - objects_failed: u64, - bytes_processed: u64, -} +type BackgroundHealProgress = rustfs_heal::HealProgress; #[derive(Debug)] struct ClusterHealStatusSnapshot { @@ -344,17 +337,10 @@ fn add_operations(total: &mut rustfs_heal::HealOperationsSnapshot, next: rustfs_ add_source_counts(&mut total.retrying_by_source, next.retrying_by_source); } -fn add_progress(total: &mut BackgroundHealProgress, next: NodeHealProgress) { - total.objects_scanned = total.objects_scanned.saturating_add(next.objects_scanned); - total.objects_healed = total.objects_healed.saturating_add(next.objects_healed); - total.objects_failed = total.objects_failed.saturating_add(next.objects_failed); - total.bytes_processed = total.bytes_processed.saturating_add(next.bytes_processed); -} - fn aggregate_cluster_heal_status(snapshots: Vec) -> ClusterHealStatusSnapshot { let mut info = BackgroundHealInfo::default(); let mut operations = rustfs_heal::HealOperationsSnapshot::default(); - let mut progress = None; + let mut progress = Vec::new(); let mut any_services_enabled = false; let mut any_initialized = false; @@ -371,18 +357,12 @@ fn aggregate_cluster_heal_status(snapshots: Vec) -> Clus } add_operations(&mut operations, snapshot.operations); if let Some(next) = snapshot.progress { - add_progress( - progress.get_or_insert(BackgroundHealProgress { - objects_scanned: 0, - objects_healed: 0, - objects_failed: 0, - bytes_processed: 0, - }), - next, - ); + progress.push(next); } } + let progress = rustfs_heal::aggregate_heal_progress(progress); + let state = if operations.queue_length > 0 || operations.active_tasks > 0 || operations.retrying_tasks > 0 { HealRuntimeState::Active } else if any_initialized { @@ -2201,10 +2181,19 @@ mod tests { }; let progress = BackgroundHealProgress { + kind: rustfs_heal::heal::progress::HealProgressKind::ObjectSweep, objects_scanned: 7, objects_healed: 3, objects_failed: 1, + skipped_objects: 3, + objects_total_count: 10, + objects_total_size: 8192, bytes_processed: 4096, + progress_percentage: 50.0, + progress_state: rustfs_heal::heal::progress::HealProgressState::Running, + baseline_generation: Some(42), + baseline_known: true, + ..Default::default() }; let encoded = encode_background_heal_status( @@ -2220,7 +2209,14 @@ mod tests { assert_eq!(json["progress"]["objectsScanned"], 7); assert_eq!(json["progress"]["objectsHealed"], 3); assert_eq!(json["progress"]["objectsFailed"], 1); + assert_eq!(json["progress"]["skippedObjects"], 3); + assert_eq!(json["progress"]["objectsTotalCount"], 10); + assert_eq!(json["progress"]["objectsTotalSize"], 8192); assert_eq!(json["progress"]["bytesProcessed"], 4096); + assert_eq!(json["progress"]["progressState"], "running"); + assert_eq!(json["progress"]["baselineGeneration"], 42); + assert_eq!(json["progress"]["baselineKnown"], true); + assert_eq!(json["progress"]["counterUnknown"], false); } #[test] @@ -2242,10 +2238,18 @@ mod tests { ..Default::default() }, Some(NodeHealProgress { + kind: rustfs_heal::heal::progress::HealProgressKind::ObjectSweep, objects_scanned: 3, objects_healed: 1, objects_failed: 0, + skipped_objects: 2, + objects_total_count: 6, + objects_total_size: 400, bytes_processed: 100, + progress_state: rustfs_heal::heal::progress::HealProgressState::Running, + baseline_generation: Some(9), + baseline_known: true, + ..Default::default() }), ); let peer = NodeHealStatusSnapshot::for_test( @@ -2265,10 +2269,17 @@ mod tests { ..Default::default() }, Some(NodeHealProgress { + kind: rustfs_heal::heal::progress::HealProgressKind::ObjectSweep, objects_scanned: 5, objects_healed: 4, objects_failed: 1, + objects_total_count: 4, + objects_total_size: 1600, bytes_processed: 900, + progress_state: rustfs_heal::heal::progress::HealProgressState::Running, + baseline_generation: Some(9), + baseline_known: true, + ..Default::default() }), ); @@ -2287,7 +2298,15 @@ mod tests { assert_eq!(progress.objects_scanned, 8); assert_eq!(progress.objects_healed, 5); assert_eq!(progress.objects_failed, 1); + assert_eq!(progress.skipped_objects, 2); + assert_eq!(progress.objects_total_count, 10); + assert_eq!(progress.objects_total_size, 2000); assert_eq!(progress.bytes_processed, 1000); + assert_eq!(progress.progress_percentage, 50.0); + assert_eq!(progress.progress_state, rustfs_heal::heal::progress::HealProgressState::Running); + assert_eq!(progress.baseline_generation, Some(9)); + assert!(progress.baseline_known); + assert!(!progress.counter_unknown); assert_eq!(peer_first.state, HealRuntimeState::Active); assert_eq!(peer_first.operations, local_first.operations); @@ -2327,6 +2346,7 @@ mod tests { objects_healed: value, objects_failed: value, bytes_processed: value, + ..Default::default() }; let saturated = NodeHealStatusSnapshot::for_test( true, diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index d8eaadcb4..8c680aa9c 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -1912,7 +1912,7 @@ impl Node for NodeService { async fn background_heal_status( &self, - _request: Request, + request: Request, ) -> Result, Status> { if self.resolve_object_store().is_none() { return Ok(Response::new(BackgroundHealStatusResponse { @@ -1922,7 +1922,7 @@ impl Node for NodeService { })); } let snapshot = heal::capture_node_heal_status(rustfs_scanner::scanner::BackgroundHealInfo::default()).await; - match heal::encode_node_heal_status(&snapshot) { + match heal::encode_node_heal_status(&snapshot, request.into_inner().protocol_version) { Ok(bg_heal_state) => Ok(Response::new(BackgroundHealStatusResponse { success: true, bg_heal_state: bg_heal_state.into(), diff --git a/rustfs/src/storage/rpc/node_service/heal.rs b/rustfs/src/storage/rpc/node_service/heal.rs index 8f796726c..8bbd27572 100644 --- a/rustfs/src/storage/rpc/node_service/heal.rs +++ b/rustfs/src/storage/rpc/node_service/heal.rs @@ -25,7 +25,8 @@ use std::io::Cursor; use super::super::encode_msgpack_map; -const NODE_HEAL_STATUS_VERSION: u8 = 1; +const NODE_HEAL_STATUS_PREVIOUS_VERSION: u8 = 1; +const NODE_HEAL_STATUS_VERSION: u8 = 2; const NODE_HEAL_STATUS_MAX_SIZE: usize = 64 * 1024; const NODE_REPLACEMENT_RECOVERY_STATUS_VERSION: u8 = 1; const NODE_REPLACEMENT_RECOVERY_STATUS_MAX_SIZE: usize = 64 * 1024; @@ -172,13 +173,38 @@ pub(crate) fn heal_topology_fingerprint(endpoint_pools: &EndpointServerPools) -> Ok(hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower)) } -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) type NodeHealProgress = rustfs_heal::HealProgress; + +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -pub(crate) struct NodeHealProgress { - pub objects_scanned: u64, - pub objects_healed: u64, - pub objects_failed: u64, - pub bytes_processed: u64, +struct NodeHealProgressV1 { + objects_scanned: u64, + objects_healed: u64, + objects_failed: u64, + bytes_processed: u64, +} + +impl From<&NodeHealProgress> for NodeHealProgressV1 { + fn from(progress: &NodeHealProgress) -> Self { + Self { + objects_scanned: progress.objects_scanned, + objects_healed: progress.objects_healed, + objects_failed: progress.objects_failed, + bytes_processed: progress.bytes_processed, + } + } +} + +impl From for NodeHealProgress { + fn from(progress: NodeHealProgressV1) -> Self { + Self { + objects_scanned: progress.objects_scanned, + objects_healed: progress.objects_healed, + objects_failed: progress.objects_failed, + bytes_processed: progress.bytes_processed, + ..Default::default() + } + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -220,6 +246,48 @@ pub(crate) struct NodeHealStatusSnapshot { pub progress: Option, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct NodeHealStatusSnapshotV1 { + version: u8, + services_enabled: bool, + initialized: bool, + info: NodeHealInfo, + operations: HealOperationsSnapshot, + progress: Option, +} + +impl From<&NodeHealStatusSnapshot> for NodeHealStatusSnapshotV1 { + fn from(snapshot: &NodeHealStatusSnapshot) -> Self { + Self { + version: NODE_HEAL_STATUS_PREVIOUS_VERSION, + services_enabled: snapshot.services_enabled, + initialized: snapshot.initialized, + info: snapshot.info.clone(), + operations: snapshot.operations, + progress: snapshot.progress.as_ref().map(NodeHealProgressV1::from), + } + } +} + +impl From for NodeHealStatusSnapshot { + fn from(snapshot: NodeHealStatusSnapshotV1) -> Self { + Self { + version: snapshot.version, + services_enabled: snapshot.services_enabled, + initialized: snapshot.initialized, + info: snapshot.info, + operations: snapshot.operations, + progress: snapshot.progress.map(NodeHealProgress::from), + } + } +} + +#[derive(Deserialize)] +struct NodeHealStatusVersion { + version: u8, +} + impl NodeHealStatusSnapshot { #[cfg(test)] pub(crate) fn for_test( @@ -249,14 +317,7 @@ impl NodeHealStatusSnapshot { } pub(crate) async fn capture_node_heal_status(info: BackgroundHealInfo) -> NodeHealStatusSnapshot { - let progress = rustfs_heal::current_heal_progress_snapshot() - .await - .map(|progress| NodeHealProgress { - objects_scanned: progress.objects_scanned, - objects_healed: progress.objects_healed, - objects_failed: progress.objects_failed, - bytes_processed: progress.bytes_processed, - }); + let progress = rustfs_heal::current_heal_progress_snapshot().await; NodeHealStatusSnapshot { version: NODE_HEAL_STATUS_VERSION, @@ -268,22 +329,43 @@ pub(crate) async fn capture_node_heal_status(info: BackgroundHealInfo) -> NodeHe } } -pub(crate) fn encode_node_heal_status(snapshot: &NodeHealStatusSnapshot) -> Result, String> { - encode_msgpack_map(snapshot).map_err(|err| format!("failed to encode node heal status: {err}")) +pub(crate) fn encode_node_heal_status(snapshot: &NodeHealStatusSnapshot, protocol_version: u32) -> Result, String> { + let encoded = if protocol_version < rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION { + encode_msgpack_map(&NodeHealStatusSnapshotV1::from(snapshot)) + } else { + let mut snapshot = snapshot.clone(); + snapshot.version = NODE_HEAL_STATUS_VERSION; + encode_msgpack_map(&snapshot) + }; + encoded.map_err(|err| format!("failed to encode node heal status: {err}")) } pub(crate) fn decode_node_heal_status(data: &[u8]) -> Result { if data.len() > NODE_HEAL_STATUS_MAX_SIZE { return Err("node heal status exceeds size limit".to_string()); } - let mut deserializer = Deserializer::new(Cursor::new(data)); - let snapshot = NodeHealStatusSnapshot::deserialize(&mut deserializer) - .map_err(|err| format!("failed to decode node heal status: {err}"))?; + let decode_version = || { + let mut deserializer = Deserializer::new(Cursor::new(data)); + NodeHealStatusVersion::deserialize(&mut deserializer) + .map(|version| (version, deserializer)) + .map_err(|err| format!("failed to decode node heal status: {err}")) + }; + let (version, deserializer) = decode_version()?; if usize::try_from(deserializer.get_ref().position()).ok() != Some(data.len()) { return Err("node heal status contains trailing data".to_string()); } - if snapshot.version != NODE_HEAL_STATUS_VERSION { - return Err(format!("unsupported node heal status version: {}", snapshot.version)); + + let mut deserializer = Deserializer::new(Cursor::new(data)); + let snapshot = match version.version { + NODE_HEAL_STATUS_PREVIOUS_VERSION => { + NodeHealStatusSnapshotV1::deserialize(&mut deserializer).map(NodeHealStatusSnapshot::from) + } + NODE_HEAL_STATUS_VERSION => NodeHealStatusSnapshot::deserialize(&mut deserializer), + version => return Err(format!("unsupported node heal status version: {version}")), + } + .map_err(|err| format!("failed to decode node heal status: {err}"))?; + if usize::try_from(deserializer.get_ref().position()).ok() != Some(data.len()) { + return Err("node heal status contains trailing data".to_string()); } Ok(snapshot) } @@ -387,9 +469,10 @@ pub(crate) fn decode_node_replacement_recovery_status(data: &[u8]) -> Result Date: Sun, 23 Aug 2026 19:28:43 +0800 Subject: [PATCH 20/41] fix(scanner): fence unknown tier accounting (#6396) --- Cargo.lock | 161 +++-- Cargo.toml | 22 +- crates/data-usage/src/data_usage.rs | 679 +++++++++++++++++- crates/e2e_test/src/protocols/sftp_helpers.rs | 4 +- crates/ecstore/src/api/mod.rs | 12 +- crates/ecstore/src/cluster/rpc/http_auth.rs | 81 ++- .../cluster/rpc/internode_data_transport.rs | 229 +++++- crates/ecstore/src/cluster/rpc/mod.rs | 5 +- crates/ecstore/src/cluster/rpc/remote_disk.rs | 61 +- crates/ecstore/src/core/pools.rs | 28 +- .../ecstore/src/storage_api_contracts/mod.rs | 12 +- crates/scanner/src/data_usage_define.rs | 145 +++- crates/scanner/src/data_usage_define/tests.rs | 264 ++++++- crates/scanner/src/lib.rs | 296 +++++++- crates/scanner/src/remote_scanner/stream.rs | 39 +- .../src/remote_scanner/stream/tests.rs | 38 + crates/scanner/src/scanner_folder.rs | 36 +- .../src/scanner_folder/item_actions.rs | 113 ++- crates/scanner/src/scanner_folder/tests.rs | 5 + crates/scanner/src/scanner_io.rs | 22 +- crates/scanner/src/scanner_io/cache.rs | 205 +++++- crates/scanner/src/scanner_io/io_cache.rs | 26 +- crates/scanner/src/scanner_io/io_cycle.rs | 12 + crates/scanner/src/scanner_io/io_disk.rs | 42 +- .../src/scanner_io/publish_gate_tests.rs | 357 ++++++++- crates/scanner/src/scanner_io/tests.rs | 49 +- crates/storage-api/src/lib.rs | 3 + rustfs/src/storage/rpc/http_service.rs | 113 ++- rustfs/src/storage/storage_api.rs | 27 +- 29 files changed, 2827 insertions(+), 259 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d00968f01..be5c34172 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,16 +68,16 @@ dependencies = [ [[package]] name = "aes-gcm" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" +checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" dependencies = [ "aead", "aes 0.9.2", "cipher 0.5.2", "ctr", + "ctutils", "ghash", - "subtle", "zeroize", ] @@ -333,6 +333,12 @@ dependencies = [ "password-hash", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "arrayvec" version = "0.7.8" @@ -813,11 +819,12 @@ dependencies = [ [[package]] name = "async_zip" -version = "0.0.18" +version = "0.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d8c50d65ce1b0e0cb65a785ff615f78860d7754290647d3b983208daa4f85e6" +checksum = "fb7f5f40e1eb30949a266fc900d37fd3267c7baf50c3705ac09c5d8ced5def63" dependencies = [ "async-compression", + "binrw", "crc32fast", "futures-lite", "pin-project", @@ -869,9 +876,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4" +checksum = "a767267da9e2c2e189b2f9df8b5657e850ecf5352644734ba130d4a57095cf1b" dependencies = [ "aws-credential-types", "aws-runtime", @@ -964,9 +971,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.115.0" +version = "1.116.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5b034f8b7ceadb873d0bc607c30bb4b0be68e09a84c837174e7c2c6878ff882" +checksum = "484ecdbea2a1cfc0e6eea69ce0a665f93913671b303ba40b2361b1d826544e7e" dependencies = [ "arc-swap", "aws-credential-types", @@ -990,9 +997,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.142.0" +version = "1.143.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9e15a5c55e05f4b0b7e483160b3c85cccdf77cff02c95504f3e71d460855cd2" +checksum = "a0ade5433c9561daac7c0c6bc910f1240b4f8ec0d6148b0b463aac0691d747c9" dependencies = [ "arc-swap", "aws-credential-types", @@ -1027,9 +1034,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.106.0" +version = "1.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0efcee834347b6705eca3eea2defd88242f43774f55d7326604222e3c86260" +checksum = "769b0abd0f89cfe11da5099986dd493e4f94347ce9a4562cb86ddecfe926b6c0" dependencies = [ "arc-swap", "aws-credential-types", @@ -1053,9 +1060,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.108.0" +version = "1.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a59312a04cf19c962cfee32b64ecfee758f8786407ff6da5b30fff46ae96f201" +checksum = "f4075b8a2c8cda4076a3dcc43b9d6dabd93e0c2502abeaaf7e14aaead9bb312b" dependencies = [ "arc-swap", "aws-credential-types", @@ -1079,9 +1086,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.111.0" +version = "1.112.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e7eb63457a9e547f9986fe3b273f77c43679da4d04f46359fa881c5e19b6e" +checksum = "3f582002918346a3e685be1b391c7bea155073088cea6bd4e4b7663df9e43b6c" dependencies = [ "arc-swap", "aws-credential-types", @@ -1544,6 +1551,30 @@ dependencies = [ "serde", ] +[[package]] +name = "binrw" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ad120d555272286c1017d25165ab8bd74806f13fc85b258484ec7e4ce75458f" +dependencies = [ + "array-init", + "binrw_derive", + "bytemuck", +] + +[[package]] +name = "binrw_derive" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df92e0e9baae4dc82c7bad7715ca40c0a5c71539057bf2ea04a5c29c980410b" +dependencies = [ + "either", + "owo-colors", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -1631,9 +1662,9 @@ dependencies = [ [[package]] name = "blocking" -version = "1.6.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" dependencies = [ "async-channel", "async-task", @@ -1722,6 +1753,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + [[package]] name = "byteorder" version = "1.5.0" @@ -2325,9 +2362,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" dependencies = [ "cfg-if", ] @@ -3934,9 +3971,9 @@ dependencies = [ [[package]] name = "either" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "elliptic-curve" @@ -4470,6 +4507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ "polyval", + "zeroize", ] [[package]] @@ -5052,9 +5090,9 @@ dependencies = [ [[package]] name = "hotpath" -version = "0.23.3" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce755d457a63bdd0c95e4c91511daad1b58b33209543b7f38027b676f387e5e" +checksum = "e2645642a23d4061ec15a4a6e74f851a3145c3125356846cfc7772ff9c6f2737" dependencies = [ "arc-swap", "async-channel", @@ -5086,9 +5124,9 @@ dependencies = [ [[package]] name = "hotpath-macros" -version = "0.23.3" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a903af89a8429cb07790c3818bc15270b394f80af1bc254e5ccf9c7de2961770" +checksum = "89a3d3cdf9b0d4d3d4f6d4a29798f3b9170401ba500eaa58dd8f890f926af0f1" dependencies = [ "proc-macro2", "quote", @@ -5097,15 +5135,15 @@ dependencies = [ [[package]] name = "hotpath-macros-meta" -version = "0.23.3" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc0ab94ffbb2ee77f4a897df02b5a137a10cf24d69bda936e59aff4dd456e61" +checksum = "e84cd2417fa60938241cf1cd6c03e09953f5c821122dc5da9b8f27975d136c5b" [[package]] name = "hotpath-meta" -version = "0.23.3" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053481f6cec8f775a3276c7f6e2f21123111d28261e4edc15ea7421c445964bb" +checksum = "4d2c145b67b1a4e7bcefa918995e212c30a49a85f05cc5962fe1f717878d560b" dependencies = [ "hotpath-macros-meta", ] @@ -5372,9 +5410,9 @@ checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -5761,9 +5799,9 @@ dependencies = [ [[package]] name = "keccak" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd9697dc4a9a62e2da93389f34400b77a28f0287711263cabb203b3ccb9c0e4" +checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -6075,9 +6113,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru" @@ -6267,7 +6305,7 @@ dependencies = [ "hashbrown 0.16.1", "indexmap 2.14.0", "metrics", - "ordered-float 5.3.0", + "ordered-float 5.5.0", "quanta", "radix_trie", "rand 0.9.5", @@ -7215,9 +7253,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "5.3.0" +version = "5.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0" dependencies = [ "num-traits", ] @@ -7252,6 +7290,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "owo-colors" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" + [[package]] name = "p12-keystore" version = "0.2.1" @@ -7794,6 +7838,7 @@ dependencies = [ "cpubits", "cpufeatures 0.3.0", "universal-hash", + "zeroize", ] [[package]] @@ -8264,6 +8309,16 @@ name = "quick-xml" version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ "encoding_rs", "memchr", @@ -8928,9 +8983,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.62.7" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9decb68e4e44e1079700e54f17c8f23806ec53d7e0db73ab1c71d9dabc666812" +checksum = "00cf00190c315093734a8d405225bd8773a219bc86538a9b73bfc51145b33995" dependencies = [ "aes 0.9.2", "aws-lc-rs", @@ -9152,7 +9207,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "proptest", - "quick-xml", + "quick-xml 0.42.0", "rand 0.10.2", "rcgen", "regex", @@ -9418,7 +9473,7 @@ dependencies = [ "path-absolutize", "pin-project-lite", "proptest", - "quick-xml", + "quick-xml 0.42.0", "rand 0.10.2", "ratelimit", "rcgen", @@ -9885,7 +9940,7 @@ dependencies = [ "jiff", "metrics", "percent-encoding", - "quick-xml", + "quick-xml 0.42.0", "rayon", "rustc-hash", "rustfs-config", @@ -10658,9 +10713,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.14" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -10704,7 +10759,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s3s" version = "0.15.0-alpha.1" -source = "git+https://github.com/rustfs/s3s.git?rev=ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a#ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" +source = "git+https://github.com/rustfs/s3s.git?rev=e080e38c56a3b43acbacce55710d765a5ce9003d#e080e38c56a3b43acbacce55710d765a5ce9003d" dependencies = [ "arc-swap", "arrayvec", @@ -10731,7 +10786,7 @@ dependencies = [ "nom 8.0.0", "numeric_cast", "pin-project-lite", - "quick-xml", + "quick-xml 0.41.0", "regex", "serde", "serde_json", @@ -12676,9 +12731,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.24.1" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" +checksum = "f053576934f05a761a402421fbbe3d425d9366f75f978806a037b3ca481abecc" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -13398,9 +13453,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 717a0c6e6..c09da814e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,7 +135,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" } # Async Runtime and Networking async-channel = "2.5.0" -async_zip = { default-features = false, version = "0.0.18" } +async_zip = { default-features = false, version = "0.0.19" } mysql_async = { default-features = false, version = "0.37" } async-compression = { version = "0.4.43" } async-recursion = "1.1.1" @@ -178,7 +178,7 @@ byteorder = "1.5.0" flatbuffers = "25.12.19" form_urlencoded = "1.2.2" prost = "0.14.4" -quick-xml = "0.41.0" +quick-xml = "0.42.0" rmp = { version = "0.8.15" } rmp-serde = { version = "1.3.1" } serde = { version = "1.0.229" } @@ -191,7 +191,7 @@ serde_urlencoded = "0.7.1" # matching stable releases are not available yet, while previous stable lines # have incompatible APIs. Keep them exact-pinned and monitor upstream for stable # releases. -aes-gcm = { version = "=0.11.0" } +aes-gcm = { version = "=0.11.1" } argon2 = { version = "=0.6.0-rc.8" } blake2 = "=0.11.0-rc.6" chacha20poly1305 = { version = "=0.11.0" } @@ -227,11 +227,11 @@ arc-swap = "1.9.2" astral-tokio-tar = "0.6.4" atoi = "3.1.0" atomic_enum = "0.3.0" -aws-config = { version = "1.10.1" } +aws-config = { version = "1.11.0" } aws-credential-types = { version = "1.3.0" } -aws-sdk-kms = { default-features = false, version = "1.115.0" } -aws-sdk-s3 = { default-features = false, version = "1.142.0" } -aws-sdk-sts = { default-features = false, version = "1.111.0" } +aws-sdk-kms = { default-features = false, version = "1.116.0" } +aws-sdk-s3 = { default-features = false, version = "1.143.0" } +aws-sdk-sts = { default-features = false, version = "1.112.0" } aws-smithy-http-client = { default-features = false, version = "1.4.0" } aws-smithy-runtime-api = { version = "1.15.0" } aws-smithy-types = { version = "1.6.2" } @@ -291,7 +291,7 @@ rustify = { version = "0.7", default-features = false } rustix = { version = "1.1.4" } rust-embed = { version = "8.12.0" } rustc-hash = { version = "2.1.3" } -s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" } +s3s = { git = "https://github.com/rustfs/s3s.git", rev = "e080e38c56a3b43acbacce55710d765a5ce9003d" } serial_test = "4.0.1" shadow-rs = { default-features = false, version = "2.0.0" } siphasher = "1.0.3" @@ -314,7 +314,7 @@ tracing-subscriber = { version = "0.3.23" } transform-stream = "0.3.1" url = "2.5.8" urlencoding = "2.1.3" -uuid = { version = "1.24.1" } +uuid = { version = "1.25.0" } vaultrs = { version = "0.8.0" } tar = "0.4.46" walkdir = "2.5.0" @@ -343,7 +343,7 @@ libunftp = { version = "0.23.0" } unftp-core = "0.1.0" suppaftp = { version = "10.0.2" } rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] } -russh = { version = "0.62.7" } +russh = { version = "0.63.0" } russh-sftp = "2.4.0" # WebDAV @@ -352,7 +352,7 @@ dav-server = "0.11.0" # Performance Analysis and Memory Profiling rustfs-mimalloc = { version = "0.5.0" } rustfs-mimalloc-sys = { version = "0.5.0" } -hotpath = { version = "0.23.3", default-features = false } +hotpath = { version = "0.24.0", default-features = false } # Snapshot testing for output format regression detection insta = { version = "1.48" } diff --git a/crates/data-usage/src/data_usage.rs b/crates/data-usage/src/data_usage.rs index fec5930fc..f26410329 100644 --- a/crates/data-usage/src/data_usage.rs +++ b/crates/data-usage/src/data_usage.rs @@ -12,8 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use serde::{Deserialize, Serialize, ser::SerializeMap as _}; +use serde::{ + Deserialize, Serialize, + de::{IgnoredAny, SeqAccess, Visitor}, + ser::SerializeMap as _, +}; use std::{ + borrow::Cow, collections::{HashMap, HashSet}, hash::{DefaultHasher, Hash, Hasher}, time::{Duration, SystemTime}, @@ -48,6 +53,11 @@ pub const DATA_USAGE_OBSERVED_OBJECT_NAME: &str = ".usage.observed.json"; // RUSTFS_COMPAT_TODO(scanner-usage-v2): keep .usage.json readable and removable during rolling upgrades from pre-v2 scanners. Remove after supported direct-upgrade sources all write .usage.v2.json. pub const LEGACY_DATA_USAGE_OBJECT_NAME: &str = ".usage.json"; +/// Fixed bucket for objects whose storage class is not in the scanner's +/// cycle-local tier registry. Keeping this key fixed prevents untrusted or +/// stale tier names from growing persisted per-tier maps without bound. +pub const UNKNOWN_TIER: &str = "UNKNOWN_TIER"; + /// Returns true when `existing_last_update` is ahead of `now` by more than /// [`USAGE_LAST_UPDATE_FUTURE_TOLERANCE`], i.e. the persisted timestamp cannot be /// trusted for staleness comparisons and a fresh snapshot save must be allowed. @@ -78,12 +88,301 @@ impl TierStats { && self.num_objects.checked_add(u.num_objects).is_some() } + /// Add tier counters without allowing a counter to wrap. + pub fn checked_add(&self, u: &TierStats) -> Option { + Some(TierStats { + total_size: self.total_size.checked_add(u.total_size)?, + num_versions: self.num_versions.checked_add(u.num_versions)?, + num_objects: self.num_objects.checked_add(u.num_objects)?, + }) + } + /// True when this tier contributed nothing, i.e. merging it is a no-op. pub fn is_empty(&self) -> bool { self.total_size == 0 && self.num_versions == 0 && self.num_objects == 0 } } +/// Bounded diagnostics for objects whose tier is absent from the cycle +/// registry. Counters are authoritative; diagnostics are only a small, +/// redacted reconciliation aid and may be dropped at the configured caps. +pub const UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP: usize = 64; +pub const UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP: usize = 4096; +pub const UNKNOWN_TIER_DIAGNOSTIC_TTL: Duration = Duration::from_secs(60 * 60); +const UNKNOWN_TIER_DIAGNOSTIC_KEY_BYTES: usize = 256; +const UNKNOWN_TIER_DIAGNOSTIC_INPUT_CAP: usize = UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP * 4; + +#[derive(Clone, Debug, Default, Serialize, PartialEq, Eq)] +pub struct UnknownTierStats { + /// Logical bytes retained in the scanner's normal usage total. + pub unknown_bytes: u64, + /// Physical bytes recorded in the per-tier accounting dimension. + /// + /// Older writers only had `unknown_bytes`; decoding those snapshots keeps + /// this field at zero and the scanner fills it for new observations. + #[serde(default)] + pub unknown_physical_bytes: u64, + pub unknown_objects: u64, + pub unknown_versions: u64, + pub diagnostics_dropped: u64, + #[serde(default)] + pub diagnostics: Vec, + #[serde(default)] + pub diagnostics_at: Option, + /// A saturating update occurred; this snapshot cannot prove conservation. + /// The field is persisted so a restarted scanner cannot mistake a + /// saturated legacy aggregate for exact accounting evidence. + #[serde(default)] + pub counter_overflowed: bool, +} + +#[derive(Default)] +struct BoundedDiagnostics { + entries: Vec, + dropped: u64, +} + +impl<'de> Deserialize<'de> for BoundedDiagnostics { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct BoundedDiagnosticsVisitor; + + impl<'de> Visitor<'de> for BoundedDiagnosticsVisitor { + type Value = BoundedDiagnostics; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a sequence of bounded tier diagnostics") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut bounded = BoundedDiagnostics::default(); + let mut bytes = 0_usize; + let mut inspected = 0_usize; + loop { + if bounded.entries.len() >= UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP + || bytes >= UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP + || inspected >= UNKNOWN_TIER_DIAGNOSTIC_INPUT_CAP + { + if sequence.next_element::()?.is_none() { + break; + } + bounded.dropped = bounded.dropped.saturating_add(1); + continue; + } + let Some(diagnostic) = sequence.next_element::>()? else { + break; + }; + inspected = inspected.saturating_add(1); + if !is_redacted_tier_diagnostic(&diagnostic) + || bytes.saturating_add(diagnostic.len()) > UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP + || bounded.entries.iter().any(|entry| entry == &diagnostic) + { + bounded.dropped = bounded.dropped.saturating_add(1); + continue; + } + bytes = bytes.saturating_add(diagnostic.len()); + bounded.entries.push(diagnostic.into_owned()); + } + Ok(bounded) + } + } + + deserializer.deserialize_seq(BoundedDiagnosticsVisitor) + } +} + +#[derive(Deserialize)] +struct UnknownTierStatsWire { + #[serde(default)] + unknown_bytes: u64, + #[serde(default)] + unknown_physical_bytes: u64, + #[serde(default)] + unknown_objects: u64, + #[serde(default)] + unknown_versions: u64, + #[serde(default)] + diagnostics_dropped: u64, + #[serde(default)] + diagnostics: BoundedDiagnostics, + #[serde(default)] + diagnostics_at: Option, + #[serde(default)] + counter_overflowed: bool, +} + +fn is_redacted_tier_diagnostic(diagnostic: &str) -> bool { + diagnostic + .strip_prefix("tier-hash:") + .is_some_and(|digest| digest.len() == 16 && digest.bytes().all(|byte| byte.is_ascii_hexdigit())) +} + +fn diagnostics_expired(at: SystemTime, now: SystemTime) -> bool { + now.duration_since(at).map_or(true, |age| age > UNKNOWN_TIER_DIAGNOSTIC_TTL) +} + +fn checked_saturating_add(left: u64, right: u64, overflowed: &mut bool) -> u64 { + match left.checked_add(right) { + Some(value) => value, + None => { + *overflowed = true; + u64::MAX + } + } +} + +impl<'de> Deserialize<'de> for UnknownTierStats { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let wire = UnknownTierStatsWire::deserialize(deserializer)?; + let mut stats = Self { + unknown_bytes: wire.unknown_bytes, + unknown_physical_bytes: wire.unknown_physical_bytes, + unknown_objects: wire.unknown_objects, + unknown_versions: wire.unknown_versions, + counter_overflowed: wire.counter_overflowed, + diagnostics_dropped: wire.diagnostics_dropped, + diagnostics: wire.diagnostics.entries, + diagnostics_at: wire.diagnostics_at, + }; + stats.diagnostics_dropped = + checked_saturating_add(stats.diagnostics_dropped, wire.diagnostics.dropped, &mut stats.counter_overflowed); + if stats + .diagnostics_at + .is_some_and(|at| diagnostics_expired(at, SystemTime::now())) + { + stats.diagnostics.clear(); + stats.diagnostics_at = None; + } + Ok(stats) + } +} + +impl UnknownTierStats { + /// Record one observation where the logical and physical dimensions are + /// the same. Kept as a small compatibility helper for callers that only + /// have one size value. + pub fn record(&mut self, tier: &str, bytes: u64, versions: u64, objects: u64) { + self.record_dimensions(tier, bytes, bytes, versions, objects); + } + + /// Record one observation without conflating logical usage with physical + /// tier bytes. Both counters are saturating so malformed metadata cannot + /// wrap an aggregate. + pub fn record_dimensions(&mut self, tier: &str, logical_bytes: u64, physical_bytes: u64, versions: u64, objects: u64) { + self.unknown_bytes = checked_saturating_add(self.unknown_bytes, logical_bytes, &mut self.counter_overflowed); + self.unknown_physical_bytes = + checked_saturating_add(self.unknown_physical_bytes, physical_bytes, &mut self.counter_overflowed); + self.unknown_objects = checked_saturating_add(self.unknown_objects, objects, &mut self.counter_overflowed); + self.unknown_versions = checked_saturating_add(self.unknown_versions, versions, &mut self.counter_overflowed); + + let digest = { + let mut hasher = DefaultHasher::new(); + // Bound hashing work for hostile metadata while retaining enough + // length/prefix entropy to reconcile repeated observations. + hasher.write_u64(u64::try_from(tier.len()).unwrap_or(u64::MAX)); + let bounded = &tier.as_bytes()[..tier.len().min(UNKNOWN_TIER_DIAGNOSTIC_KEY_BYTES)]; + hasher.write(bounded); + hasher.write_u8(u8::from(bounded.iter().any(|byte| byte.is_ascii_control()))); + format!("tier-hash:{:016x}", hasher.finish()) + }; + let now = SystemTime::now(); + if self.diagnostics_at.is_some_and(|at| diagnostics_expired(at, now)) { + self.diagnostics.clear(); + } + self.diagnostics_at = Some(now); + if self.diagnostics.iter().any(|entry| entry == &digest) { + return; + } + let current_bytes: usize = self.diagnostics.iter().map(String::len).sum(); + if self.diagnostics.len() >= UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP + || current_bytes.saturating_add(digest.len()) > UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP + { + self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed); + return; + } + self.diagnostics.push(digest); + } + + pub fn merge(&mut self, other: &Self) { + self.unknown_bytes = checked_saturating_add(self.unknown_bytes, other.unknown_bytes, &mut self.counter_overflowed); + self.unknown_physical_bytes = + checked_saturating_add(self.unknown_physical_bytes, other.unknown_physical_bytes, &mut self.counter_overflowed); + self.unknown_objects = checked_saturating_add(self.unknown_objects, other.unknown_objects, &mut self.counter_overflowed); + self.unknown_versions = + checked_saturating_add(self.unknown_versions, other.unknown_versions, &mut self.counter_overflowed); + self.diagnostics_dropped = + checked_saturating_add(self.diagnostics_dropped, other.diagnostics_dropped, &mut self.counter_overflowed); + self.counter_overflowed |= other.counter_overflowed; + let now = SystemTime::now(); + if self.diagnostics_at.is_some_and(|at| diagnostics_expired(at, now)) { + self.diagnostics.clear(); + self.diagnostics_at = None; + } + if !other.diagnostics_at.is_some_and(|at| diagnostics_expired(at, now)) { + for diagnostic in &other.diagnostics { + if !is_redacted_tier_diagnostic(diagnostic) { + self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed); + continue; + } + if self.diagnostics.iter().any(|entry| entry == diagnostic) { + continue; + } + let current_bytes: usize = self.diagnostics.iter().map(String::len).sum(); + if self.diagnostics.len() >= UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP + || current_bytes.saturating_add(diagnostic.len()) > UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP + { + self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed); + continue; + } + self.diagnostics.push(diagnostic.clone()); + } + } + if !self.diagnostics.is_empty() { + self.diagnostics_at = Some(now); + } + } + + pub fn fits_add(&self, other: &Self) -> bool { + !self.counter_overflowed + && !other.counter_overflowed + && self.unknown_bytes.checked_add(other.unknown_bytes).is_some() + && self + .unknown_physical_bytes + .checked_add(other.unknown_physical_bytes) + .is_some() + && self.unknown_objects.checked_add(other.unknown_objects).is_some() + && self.unknown_versions.checked_add(other.unknown_versions).is_some() + && self.diagnostics_dropped.checked_add(other.diagnostics_dropped).is_some() + } + + pub fn checked_add(&self, other: &Self) -> Option { + if !self.fits_add(other) { + return None; + } + let mut merged = self.clone(); + merged.merge(other); + Some(merged) + } + + pub fn is_empty(&self) -> bool { + !self.counter_overflowed + && self.unknown_bytes == 0 + && self.unknown_physical_bytes == 0 + && self.unknown_objects == 0 + && self.unknown_versions == 0 + && self.diagnostics_dropped == 0 + && self.diagnostics.is_empty() + } +} + #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct AllTierStats { pub tiers: HashMap, @@ -124,6 +423,31 @@ impl AllTierStats { .iter() .all(|(tier, right)| self.tiers.get(tier).is_none_or(|left| left.fits_add(right))) } + + /// Fold keys from an older cache that are no longer present in the + /// current registry into the fixed unknown bucket. Built-in storage + /// classes remain known even when no remote tier is configured. + pub fn fold_unknown_tiers<'a, I>(&mut self, known_tiers: I) + where + I: IntoIterator, + { + let known: HashSet<&str> = known_tiers.into_iter().collect(); + let mut unknown = self.tiers.remove(UNKNOWN_TIER).unwrap_or_default(); + let retired_tiers: Vec = self + .tiers + .keys() + .filter(|tier| tier.as_str() != "STANDARD" && tier.as_str() != "REDUCED_REDUNDANCY" && !known.contains(tier.as_str())) + .cloned() + .collect(); + for tier in retired_tiers { + if let Some(stats) = self.tiers.remove(&tier) { + unknown = unknown.add(&stats); + } + } + if !unknown.is_empty() { + self.tiers.insert(UNKNOWN_TIER.to_string(), unknown); + } + } } /// Bucket target usage info provides replication statistics @@ -211,6 +535,10 @@ pub struct DataUsageInfo { /// tier exists, so an absent value means "not accounted", never "zero". #[serde(default, skip_serializing_if = "Option::is_none")] pub tier_stats: Option, + /// Bounded diagnostics and separate logical/physical counters for objects + /// classified into [`UNKNOWN_TIER`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unknown_tier_stats: Option, /// Total number of buckets in this cluster pub buckets_count: u64, @@ -332,6 +660,48 @@ pub struct DiskUsageStatus { pub snapshot_exists: bool, } +/// Independent conservation evidence for a scanner summary. +/// +/// The totals are maintained while objects are accounted and are deliberately +/// not reconstructed from the tier map at publish time. `*_known` records the +/// portion represented by the corresponding accounting dimension. +#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub struct TierAccountingProof { + pub logical_total: u64, + pub logical_known: u64, + pub physical_total: u64, + pub physical_known: u64, + #[serde(default)] + pub overflowed: bool, +} + +impl TierAccountingProof { + pub fn checked_add(self, other: Self) -> Option { + if self.overflowed || other.overflowed { + return None; + } + Some(Self { + logical_total: self.logical_total.checked_add(other.logical_total)?, + logical_known: self.logical_known.checked_add(other.logical_known)?, + physical_total: self.physical_total.checked_add(other.physical_total)?, + physical_known: self.physical_known.checked_add(other.physical_known)?, + overflowed: false, + }) + } + + pub fn saturating_add(&mut self, other: Self) { + let Some(merged) = (*self).checked_add(other) else { + self.logical_total = self.logical_total.saturating_add(other.logical_total); + self.logical_known = self.logical_known.saturating_add(other.logical_known); + self.physical_total = self.physical_total.saturating_add(other.physical_total); + self.physical_known = self.physical_known.saturating_add(other.physical_known); + self.overflowed = true; + return; + }; + *self = merged; + } +} + /// A bounded reconciliation record for an object whose logical size could not /// be trusted at the scanner boundary. The scanner persists these records in /// its cache; keeping the model here avoids a second, incompatible accounting @@ -393,6 +763,10 @@ pub struct SizeSummary { pub repl_target_stats: HashMap, /// Per-tier accounting, keyed by storage class or remote tier name pub tier_stats: HashMap, + /// Counters and bounded diagnostics for unknown tiers in this summary. + pub unknown_tier_stats: UnknownTierStats, + /// Independent logical/physical conservation evidence. + pub tier_accounting_proof: TierAccountingProof, /// Size-resolution debts observed while scanning this summary. pub size_reconciliation: Vec, /// True when the per-object summary exceeded its bounded debt buffer. @@ -494,6 +868,10 @@ impl<'de> Deserialize<'de> for SizeHistogram { } impl SizeHistogram { + pub fn is_empty(&self) -> bool { + self.0.iter().all(|value| *value == 0) + } + pub fn add(&mut self, size: u64) { let intervals = [ (0, 1024 - 1), // LESS_THAN_1024_B @@ -608,6 +986,10 @@ impl<'de> Deserialize<'de> for VersionsHistogram { } impl VersionsHistogram { + pub fn is_empty(&self) -> bool { + self.0.iter().all(|value| *value == 0) + } + pub fn add(&mut self, count: u64) { let intervals = [ (0, 0), // UNVERSIONED @@ -748,6 +1130,13 @@ pub struct DataUsageEntry { /// observed tier-classified objects. #[serde(default)] pub all_tier_stats: Option, + /// Bounded unknown-tier reconciliation state for this cache entry. + #[serde(default)] + pub unknown_tier_stats: Option, + /// Optional conservation proof. Missing values are legacy/unproven, not + /// zero-valued evidence. + #[serde(default)] + pub tier_accounting_proof: Option, } impl Serialize for DataUsageEntry { @@ -758,7 +1147,9 @@ impl Serialize for DataUsageEntry { // Keep entries map-encoded so older readers can ignore fields appended // by newer scanner versions during rolling upgrades. The derived // (array) encoding made any appended field a decode error for them. - let mut state = serializer.serialize_map(Some(11))?; + let mut state = serializer.serialize_map(Some( + 11 + usize::from(self.unknown_tier_stats.is_some()) + usize::from(self.tier_accounting_proof.is_some()), + ))?; state.serialize_entry("children", &self.children)?; state.serialize_entry("size", &self.size)?; state.serialize_entry("objects", &self.objects)?; @@ -770,11 +1161,35 @@ impl Serialize for DataUsageEntry { state.serialize_entry("compacted", &self.compacted)?; state.serialize_entry("failed_objects", &self.failed_objects)?; state.serialize_entry("all_tier_stats", &self.all_tier_stats)?; + // Keep the legacy no-unknown shape byte-for-byte stable. Once unknown + // accounting exists, append its map field before the optional proof. + if let Some(unknown_tier_stats) = self.unknown_tier_stats.as_ref() { + state.serialize_entry("unknown_tier_stats", unknown_tier_stats)?; + } + if let Some(proof) = self.tier_accounting_proof.as_ref() { + state.serialize_entry("tier_accounting_proof", proof)?; + } state.end() } } impl DataUsageEntry { + fn has_local_usage(&self) -> bool { + self.size != 0 + || self.objects != 0 + || self.versions != 0 + || self.delete_markers != 0 + || self.failed_objects != 0 + || self.obj_sizes.0.iter().any(|value| *value != 0) + || self.obj_versions.0.iter().any(|value| *value != 0) + || self.replication_stats.as_ref().is_some_and(|stats| !stats.is_empty()) + || self + .all_tier_stats + .as_ref() + .is_some_and(|stats| stats.tiers.values().any(|tier| !tier.is_empty())) + || self.unknown_tier_stats.as_ref().is_some_and(|stats| !stats.is_empty()) + } + pub fn add_child(&mut self, hash: &DataUsageHash) { if self.children.contains(&hash.key()) { return; @@ -783,6 +1198,8 @@ impl DataUsageEntry { } pub fn merge(&mut self, other: &DataUsageEntry) { + let self_had_local_usage = self.has_local_usage(); + let other_had_local_usage = other.has_local_usage(); self.objects += other.objects; self.versions += other.versions; self.delete_markers += other.delete_markers; @@ -811,6 +1228,29 @@ impl DataUsageEntry { if let Some(o_tiers) = other.all_tier_stats.as_ref().filter(|tiers| !tiers.is_empty()) { self.all_tier_stats.get_or_insert_with(AllTierStats::new).merge(o_tiers); } + if let Some(other_unknown) = other.unknown_tier_stats.as_ref() { + self.unknown_tier_stats + .get_or_insert_with(UnknownTierStats::default) + .merge(other_unknown); + } + + self.tier_accounting_proof = match (self.tier_accounting_proof, other.tier_accounting_proof) { + (Some(mut left), Some(right)) => { + left.saturating_add(right); + Some(left) + } + (None, Some(right)) if !self_had_local_usage => Some(right), + (Some(left), None) if !other_had_local_usage => Some(left), + (None, None) => None, + _ => None, + }; + // A saturated unknown-tier counter invalidates conservation evidence; + // never let an otherwise valid proof hide that loss of precision. + if self.unknown_tier_stats.as_ref().is_some_and(|stats| stats.counter_overflowed) + && let Some(proof) = self.tier_accounting_proof.as_mut() + { + proof.overflowed = true; + } self.obj_sizes.merge_from(&other.obj_sizes); self.obj_versions.merge_from(&other.obj_versions); @@ -824,6 +1264,15 @@ impl DataUsageEntry { self.all_tier_stats.get_or_insert_with(AllTierStats::new).add_sizes(tiers); } + pub fn add_unknown_tier_stats(&mut self, stats: &UnknownTierStats) { + if stats.is_empty() { + return; + } + self.unknown_tier_stats + .get_or_insert_with(UnknownTierStats::default) + .merge(stats); + } + pub fn checked_merge(&mut self, other: &DataUsageEntry) -> bool { let scalar_counts_fit = self.objects.checked_add(other.objects).is_some() && self.versions.checked_add(other.versions).is_some() @@ -887,8 +1336,21 @@ impl DataUsageEntry { (_, None) | (None, Some(_)) => true, (Some(left), Some(right)) => left.fits_merge(right), }; + let unknown_tier_stats_fit = match (&self.unknown_tier_stats, &other.unknown_tier_stats) { + (None, None) => true, + (Some(left), None) => !left.counter_overflowed, + (None, Some(right)) => !right.counter_overflowed, + (Some(left), Some(right)) => left.fits_add(right), + }; - if !scalar_counts_fit || !histograms_fit || !replication_fits || !tier_stats_fit { + let proof_fit = match (self.tier_accounting_proof, other.tier_accounting_proof) { + (Some(left), Some(right)) => left.checked_add(right).is_some(), + (Some(proof), None) | (None, Some(proof)) => !proof.overflowed, + (None, None) => true, + }; + + if !scalar_counts_fit || !histograms_fit || !replication_fits || !tier_stats_fit || !unknown_tier_stats_fit || !proof_fit + { return false; } self.merge(other); @@ -1407,6 +1869,7 @@ impl DataUsageCache { delete_markers_total_count: flat.delete_markers as u64, objects_total_size: flat.size as u64, tier_stats: flat.all_tier_stats.filter(|tiers| !tiers.is_empty()), + unknown_tier_stats: flat.unknown_tier_stats.filter(|stats| !stats.is_empty()), buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX), buckets_usage, usage_snapshot_complete: self.info.snapshot_complete, @@ -1867,6 +2330,20 @@ impl SizeSummary { self.replica_count = self.replica_count.saturating_add(other.replica_count); self.pending_count = self.pending_count.saturating_add(other.pending_count); self.failed_count = self.failed_count.saturating_add(other.failed_count); + self.unknown_tier_stats.merge(&other.unknown_tier_stats); + self.tier_accounting_proof.saturating_add(other.tier_accounting_proof); + if self.unknown_tier_stats.counter_overflowed { + self.tier_accounting_proof.overflowed = true; + } + + // A disk/bucket aggregate is assembled from many object summaries. + // Keep the per-tier dimension in lockstep with the scalar counters; + // dropping this map here would recreate the original silent-loss bug + // at the cross-disk merge boundary. + for (tier, stats) in &other.tier_stats { + let entry = self.tier_stats.entry(tier.clone()).or_default(); + *entry = entry.add(stats); + } // Merge replication target stats for (target, stats) in &other.repl_target_stats { @@ -2024,6 +2501,59 @@ mod tests { ); } + #[test] + fn retired_tier_stats_fold_into_fixed_unknown_bucket() { + let mut stats = AllTierStats::default(); + stats.tiers.insert( + "RETIRED".to_string(), + TierStats { + total_size: 9, + num_versions: 2, + num_objects: 1, + }, + ); + stats.tiers.insert( + "WARM".to_string(), + TierStats { + total_size: 4, + num_versions: 1, + num_objects: 1, + }, + ); + + stats.fold_unknown_tiers(["WARM"]); + + assert!(!stats.tiers.contains_key("RETIRED")); + assert_eq!(stats.tiers.get("WARM").map(|v| v.total_size), Some(4)); + assert_eq!(stats.tiers.get(UNKNOWN_TIER).map(|v| v.total_size), Some(9)); + } + + #[test] + fn unknown_tier_stats_deserialization_keeps_diagnostics_bounded() { + #[derive(Serialize)] + struct RawUnknownTierStats { + unknown_bytes: u64, + diagnostics: Vec, + diagnostics_dropped: u64, + } + + let mut diagnostics = vec!["tier-hash:0123456789abcdef".to_string(); UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP + 8]; + diagnostics.push("raw-tier-name-that-must-not-be-exposed".to_string()); + diagnostics.push("x".repeat(UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP + 1)); + let encoded = rmp_serde::to_vec_named(&RawUnknownTierStats { + unknown_bytes: 7, + diagnostics, + diagnostics_dropped: 3, + }) + .expect("unknown tier stats should encode"); + let decoded: UnknownTierStats = rmp_serde::from_slice(&encoded).expect("unknown tier stats should decode"); + + assert_eq!(decoded.unknown_bytes, 7); + assert_eq!(decoded.diagnostics.len(), 1); + assert!(decoded.diagnostics_dropped >= 3); + assert!(decoded.diagnostics.iter().all(|entry| is_redacted_tier_diagnostic(entry))); + } + #[test] fn checked_merge_rejects_overflowing_tier_totals() { let mut left = tier_entry( @@ -2047,6 +2577,57 @@ mod tests { assert_eq!(left.all_tier_stats.expect("left is untouched").tiers["WARM"].total_size, u64::MAX); } + #[test] + fn checked_merge_rejects_unknown_counter_overflow_on_either_side() { + let overflowed = DataUsageEntry { + unknown_tier_stats: Some(UnknownTierStats { + counter_overflowed: true, + ..Default::default() + }), + tier_accounting_proof: Some(TierAccountingProof { + logical_total: 1, + logical_known: 1, + ..Default::default() + }), + ..Default::default() + }; + let mut left = DataUsageEntry::default(); + assert!(!left.checked_merge(&overflowed)); + + let mut left = overflowed.clone(); + assert!(!left.checked_merge(&DataUsageEntry::default())); + } + + #[test] + fn checked_merge_rejects_one_sided_overflowed_proof_without_mutation() { + let mut left = DataUsageEntry { + size: 1, + tier_accounting_proof: Some(TierAccountingProof { + logical_total: 1, + logical_known: 1, + overflowed: true, + ..Default::default() + }), + ..Default::default() + }; + let legacy = DataUsageEntry { + size: 2, + ..Default::default() + }; + + assert!(!left.checked_merge(&legacy)); + assert_eq!(left.size, 1); + assert_eq!( + left.tier_accounting_proof, + Some(TierAccountingProof { + logical_total: 1, + logical_known: 1, + overflowed: true, + ..Default::default() + }) + ); + } + /// Entry shape released before per-tier accounting, using the derived /// (array) encoding those writers produced. #[derive(Serialize, Deserialize)] @@ -2086,6 +2667,72 @@ mod tests { assert_eq!(legacy.objects, 0); } + #[test] + fn tier_accounting_proof_is_optional_and_map_encoded() { + let entry = DataUsageEntry { + tier_accounting_proof: Some(TierAccountingProof { + logical_total: 11, + logical_known: 11, + physical_total: 7, + physical_known: 7, + ..Default::default() + }), + ..Default::default() + }; + let encoded = rmp_serde::to_vec(&entry).expect("proof-bearing entry should encode"); + let decoded: DataUsageEntry = rmp_serde::from_slice(&encoded).expect("proof-bearing entry should decode"); + assert_eq!(decoded.tier_accounting_proof, entry.tier_accounting_proof); + + let legacy = LegacyEntry { + children: DataUsageHashMap::default(), + size: 12, + objects: 3, + versions: 4, + delete_markers: 1, + obj_sizes: SizeHistogram::default(), + obj_versions: VersionsHistogram::default(), + replication_stats: None, + compacted: false, + failed_objects: 2, + }; + let legacy_bytes = rmp_serde::to_vec(&legacy).expect("legacy entry should encode"); + let decoded: DataUsageEntry = rmp_serde::from_slice(&legacy_bytes).expect("legacy entry should decode"); + assert!(decoded.tier_accounting_proof.is_none()); + } + + #[test] + fn empty_entry_merge_preserves_accounting_proof() { + let mut entry = DataUsageEntry { + tier_accounting_proof: Some(TierAccountingProof { + logical_total: 7, + logical_known: 7, + physical_total: 7, + physical_known: 7, + ..Default::default() + }), + ..Default::default() + }; + entry.merge(&DataUsageEntry::default()); + assert!(entry.tier_accounting_proof.is_some()); + + let mut empty = DataUsageEntry::default(); + empty.merge(&entry); + assert_eq!(empty.tier_accounting_proof, entry.tier_accounting_proof); + } + + #[test] + fn unknown_counter_overflow_is_not_empty_and_survives_roundtrip() { + let stats = UnknownTierStats { + counter_overflowed: true, + ..Default::default() + }; + assert!(!stats.is_empty()); + let encoded = rmp_serde::to_vec_named(&stats).expect("overflow marker should encode"); + let decoded: UnknownTierStats = rmp_serde::from_slice(&encoded).expect("overflow marker should decode"); + assert!(decoded.counter_overflowed); + assert!(!decoded.is_empty()); + } + #[test] fn legacy_array_encoded_entries_still_load() { let legacy = LegacyEntry { @@ -2903,6 +3550,32 @@ mod tests { ); } + #[test] + fn test_dui_filters_empty_unknown_tier_usage_from_the_flattened_tree() { + let root_hash = hash_path("root"); + let bucket_hash = hash_path("bucket-a"); + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: "root".to_string(), + ..Default::default() + }, + ..Default::default() + }; + cache.replace_hashed(&root_hash, &None, &DataUsageEntry::default()); + + let child = DataUsageEntry { + objects: 1, + unknown_tier_stats: Some(UnknownTierStats::default()), + ..Default::default() + }; + cache.replace_hashed(&bucket_hash, &Some(root_hash), &child); + + let info = cache.dui("root", &["bucket-a".to_string()]); + + assert_eq!(info.objects_total_count, 1); + assert!(info.unknown_tier_stats.is_none()); + } + #[test] fn test_data_usage_entry_merge_preserves_replication_targets() { let mut base = DataUsageEntry { diff --git a/crates/e2e_test/src/protocols/sftp_helpers.rs b/crates/e2e_test/src/protocols/sftp_helpers.rs index 0b1663748..c55723e97 100644 --- a/crates/e2e_test/src/protocols/sftp_helpers.rs +++ b/crates/e2e_test/src/protocols/sftp_helpers.rs @@ -26,7 +26,7 @@ use aws_sdk_s3::config::{Credentials, Region}; use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; use russh::client::{self, Handle}; use russh::keys::ssh_key::LineEnding; -use russh::keys::{Algorithm, PrivateKey, PublicKey}; +use russh::keys::{Algorithm, PrivateKey, PublicKeyOrCertificate}; use russh_sftp::client::SftpSession; use russh_sftp::protocol::OpenFlags; use std::path::Path; @@ -46,7 +46,7 @@ pub struct AcceptAnyServerKey; impl client::Handler for AcceptAnyServerKey { type Error = anyhow::Error; - async fn check_server_key(&mut self, _server_public_key: &PublicKey) -> Result { + async fn check_server_key(&mut self, _server_public_key: &PublicKeyOrCertificate) -> Result { Ok(true) } } diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e55e9b97b..72f87633a 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -458,11 +458,13 @@ pub mod rpc { ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth, - normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_put_file_capability, - sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, - tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, - verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, - verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap, + normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, sign_ns_scanner_capability, + sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof, + tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, + verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, + verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, + verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, + verify_tonic_rpc_signature_with_bootstrap, }; } diff --git a/crates/ecstore/src/cluster/rpc/http_auth.rs b/crates/ecstore/src/cluster/rpc/http_auth.rs index bf05522ed..6be0bab22 100644 --- a/crates/ecstore/src/cluster/rpc/http_auth.rs +++ b/crates/ecstore/src/cluster/rpc/http_auth.rs @@ -94,6 +94,7 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13; const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096; const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432; const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3"; +const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1"; pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService"; static INTERNODE_RPC_SIGNATURE_STRICT: LazyLock = LazyLock::new(|| { get_env_bool( @@ -636,40 +637,79 @@ pub fn verify_put_file_capability(challenge: Uuid, server_epoch: Uuid, version: .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid put_file capability proof")) } -fn update_ns_scanner_capability_mac(mac: &mut HmacSha256, challenge: Uuid, server_epoch: Uuid) { +fn update_ns_scanner_capability_mac( + mac: &mut HmacSha256, + challenge: Uuid, + server_epoch: Uuid, + supports_tier_registry_generation: bool, +) { mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN); mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes()); mac.update(challenge.as_bytes()); mac.update(server_epoch.as_bytes()); + if supports_tier_registry_generation { + // The optional response capability is part of the authenticated + // scope. A proxy cannot turn an old/unsupported peer into a worker + // that receives generation-fenced scanner work. + mac.update(NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN); + } } -fn generate_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result> { +fn generate_ns_scanner_capability_proof( + secret: &str, + challenge: Uuid, + server_epoch: Uuid, + supports_tier_registry_generation: bool, +) -> std::io::Result> { if challenge.is_nil() || server_epoch.is_nil() { return Err(std::io::Error::other("Invalid namespace scanner capability scope")); } let mut mac = ::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?; - update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch); + update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation); Ok(mac.finalize().into_bytes().to_vec()) } -fn verify_ns_scanner_capability_proof(secret: &str, challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> { +fn verify_ns_scanner_capability_proof( + secret: &str, + challenge: Uuid, + server_epoch: Uuid, + proof: &[u8], + supports_tier_registry_generation: bool, +) -> std::io::Result<()> { if challenge.is_nil() || server_epoch.is_nil() { return Err(std::io::Error::other("Invalid namespace scanner capability scope")); } let mut mac = ::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?; - update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch); + update_ns_scanner_capability_mac(&mut mac, challenge, server_epoch, supports_tier_registry_generation); mac.verify_slice(proof) .map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof")) } pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result> { - generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch) + sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false) } pub fn verify_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> { - verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof) + verify_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, proof, false) +} + +pub fn sign_ns_scanner_capability_with_tier_registry_generation( + challenge: Uuid, + server_epoch: Uuid, + supports_tier_registry_generation: bool, +) -> std::io::Result> { + generate_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, supports_tier_registry_generation) +} + +pub fn verify_ns_scanner_capability_with_tier_registry_generation( + challenge: Uuid, + server_epoch: Uuid, + proof: &[u8], + supports_tier_registry_generation: bool, +) -> std::io::Result<()> { + verify_ns_scanner_capability_proof(&get_shared_secret()?, challenge, server_epoch, proof, supports_tier_registry_generation) } #[derive(Clone, Copy)] @@ -1709,13 +1749,28 @@ mod tests { let secret = "test-scanner-capability-secret"; let challenge = Uuid::new_v4(); let server_epoch = Uuid::new_v4(); - let proof = - generate_ns_scanner_capability_proof(secret, challenge, server_epoch).expect("capability proof should be generated"); + let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false) + .expect("capability proof should be generated"); - assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof).is_ok()); - assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof).is_err()); - assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof).is_err()); - assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof).is_err()); + assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_ok()); + assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false).is_err()); + assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false).is_err()); + assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false).is_err()); + } + + #[test] + fn namespace_scanner_capability_proof_binds_tier_registry_generation_support() { + let secret = "test-scanner-capability-secret"; + let challenge = Uuid::new_v4(); + let server_epoch = Uuid::new_v4(); + let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true) + .expect("generation capability proof should be generated"); + + assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true).is_ok()); + assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false).is_err()); + let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false) + .expect("legacy capability proof should be generated"); + assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true).is_err()); } /// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed, diff --git a/crates/ecstore/src/cluster/rpc/internode_data_transport.rs b/crates/ecstore/src/cluster/rpc/internode_data_transport.rs index 6f46fead3..6b695954f 100644 --- a/crates/ecstore/src/cluster/rpc/internode_data_transport.rs +++ b/crates/ecstore/src/cluster/rpc/internode_data_transport.rs @@ -13,17 +13,18 @@ // limitations under the License. use crate::cluster::rpc::{ - build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability, verify_put_file_capability, + build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_tier_registry_generation, + verify_put_file_capability, }; use crate::disk::error::{Error, Result}; use crate::disk::{FileReader, FileWriter}; use crate::storage_api_contracts::internode::{ NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, - NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, - PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, - PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, - WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1, + NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, + NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, + PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, + PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1, }; use async_trait::async_trait; use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE}; @@ -137,6 +138,12 @@ fn put_file_capability_status_is_legacy(status: u16) -> bool { status == 404 } +fn ns_scanner_capability_error_allows_legacy(error: &Error) -> bool { + [400, 404, 405, 426] + .into_iter() + .any(|status| error.is_internode_http_status(status)) +} + #[derive(Debug, Clone, Copy, Eq, PartialEq)] #[allow( dead_code, @@ -220,6 +227,7 @@ pub struct NsScannerStreamRequest { #[derive(Debug, Clone)] pub struct NsScannerCapabilityRequest { pub endpoint: String, + pub supports_tier_registry_generation: bool, } /// Data-plane stream opener used by `RemoteDisk`. @@ -252,6 +260,15 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug { async fn probe_ns_scanner(&self, _request: NsScannerCapabilityRequest) -> Result { Err(Error::MethodNotAllowed) } + async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result { + let server_epoch = self.probe_ns_scanner(request).await?; + Ok(NsScannerCapabilityResponse { + version: NS_SCANNER_PROTOCOL_VERSION, + server_epoch, + proof: Vec::new(), + supports_tier_registry_generation: None, + }) + } // Interface facet nobody calls yet: every transport implements both, but no // caller negotiates on them. Kept for the internode transport split // (backlog#1350); deleting them would delete the seam and six impls. @@ -335,27 +352,44 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport { } async fn probe_ns_scanner(&self, request: NsScannerCapabilityRequest) -> Result { - let challenge = Uuid::new_v4(); - let url = build_ns_scanner_capability_url(&request, challenge); - let mut headers = msgpack_headers(); - build_auth_headers(&url, &Method::GET, &mut headers)?; - let reader = HttpReader::new(url, Method::GET, headers, None).await?; - let mut body = Vec::new(); - reader - .take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX)) - .read_to_end(&mut body) - .await?; - if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE { - return Err(Error::other("invalid remote namespace scanner capability response size")); + Ok(self.probe_ns_scanner_capability(request).await?.server_epoch) + } + + async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result { + if request.supports_tier_registry_generation { + return match self.probe_ns_scanner_capability_once(&request).await { + Ok(response) => Ok(response), + Err(marked_error) if ns_scanner_capability_error_allows_legacy(&marked_error) => { + // A v3 peer may reject the additive query marker, ignore + // it, or return its legacy proof. Retry once without the + // marker and only downgrade after that legacy response is + // authenticated; an unverified epoch is never trusted. + let legacy_request = NsScannerCapabilityRequest { + endpoint: request.endpoint.clone(), + supports_tier_registry_generation: false, + }; + match self.probe_ns_scanner_capability_once(&legacy_request).await { + Ok(mut response) => { + response.supports_tier_registry_generation = None; + Ok(response) + } + Err(legacy_error) if ns_scanner_capability_error_allows_legacy(&legacy_error) => { + // Some old deployments expose only the legacy + // protocol response (or advertise 426). Treat + // the pair as an explicit unsupported result so + // the scanner can use its coordinator fallback. + Err(Error::MethodNotAllowed) + } + Err(_) => Err(marked_error), + } + } + // A server failure, network failure, or authentication error + // is not evidence of an old parser. Do not issue an + // unauthenticated legacy probe or silently downgrade. + Err(marked_error) => Err(marked_error), + }; } - let response: NsScannerCapabilityResponse = - rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?; - if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() { - return Err(Error::other("incompatible remote namespace scanner capability response")); - } - verify_ns_scanner_capability(challenge, response.server_epoch, &response.proof) - .map_err(|err| Error::other(format!("remote namespace scanner capability authentication failed: {err}")))?; - Ok(response.server_epoch) + self.probe_ns_scanner_capability_once(&request).await } fn name(&self) -> &'static str { @@ -368,6 +402,53 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport { } impl TcpHttpInternodeDataTransport { + async fn probe_ns_scanner_capability_once( + &self, + request: &NsScannerCapabilityRequest, + ) -> Result { + let challenge = Uuid::new_v4(); + let url = build_ns_scanner_capability_url(request, challenge); + let mut headers = msgpack_headers(); + build_auth_headers(&url, &Method::GET, &mut headers)?; + let reader = HttpReader::new(url, Method::GET, headers, None).await?; + let mut body = Vec::new(); + reader + .take(u64::try_from(NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE + 1).unwrap_or(u64::MAX)) + .read_to_end(&mut body) + .await?; + if body.is_empty() || body.len() > NS_SCANNER_MAX_CAPABILITY_RESPONSE_SIZE { + return Err(Error::other("invalid remote namespace scanner capability response size")); + } + let mut response: NsScannerCapabilityResponse = + rmp_serde::from_slice(&body).map_err(|_| Error::other("invalid remote namespace scanner capability response"))?; + if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() { + return Err(Error::other("incompatible remote namespace scanner capability response")); + } + if let Err(err) = verify_ns_scanner_capability_with_tier_registry_generation( + challenge, + response.server_epoch, + &response.proof, + request.supports_tier_registry_generation, + ) { + // A permissive older peer can ignore the additive marker and + // return a valid legacy-scope proof with HTTP 200. Accept that + // response only after independently authenticating the legacy + // scope; all other verification failures remain fail-closed. + if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) { + response.supports_tier_registry_generation = None; + return Ok(response); + } + return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}"))); + } + // The proof authenticates the requested capability scope, not the + // optional response field. Derive the client-facing bit from that + // verified scope so an intermediary cannot strip or rewrite the field + // and force a silent downgrade after a successful generation-bound + // handshake. + normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation); + Ok(response) + } + async fn put_file_auth_capability(&self, endpoint: &str) -> Result> { resolve_put_file_auth_capability(endpoint, || async { tokio::time::timeout(PUT_FILE_CAPABILITY_PROBE_TIMEOUT, self.probe_put_file_auth(endpoint)) @@ -649,6 +730,14 @@ fn build_walk_dir_url(request: &WalkDirStreamRequest) -> String { ) } +fn normalize_ns_scanner_capability_response(response: &mut NsScannerCapabilityResponse, requested_generation_support: bool) { + response.supports_tier_registry_generation = requested_generation_support.then_some(true); +} + +fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool { + verify_ns_scanner_capability_with_tier_registry_generation(challenge, response.server_epoch, &response.proof, false).is_ok() +} + fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String { let body_sha256 = hex_simd::encode_to_string(Sha256::digest(&request.body), hex_simd::AsciiCase::Lower); format!( @@ -675,13 +764,18 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String { fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String { format!( - "{}{}?{}={}&{}={}", + "{}{}?{}={}&{}={}{}", request.endpoint, NS_SCANNER_PATH, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, - challenge + challenge, + if request.supports_tier_registry_generation { + format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY) + } else { + String::new() + } ) } @@ -794,6 +888,7 @@ mod tests { let probe_err = transport .probe_ns_scanner(NsScannerCapabilityRequest { endpoint: "http://node1:9000".to_string(), + supports_tier_registry_generation: false, }) .await .expect_err("legacy transport should report namespace scanner as unsupported"); @@ -1387,6 +1482,7 @@ mod tests { let url = build_ns_scanner_capability_url( &NsScannerCapabilityRequest { endpoint: "http://node1:9000".to_string(), + supports_tier_registry_generation: false, }, challenge, ); @@ -1399,6 +1495,85 @@ mod tests { ); } + #[test] + fn ns_scanner_capability_url_marks_generation_support_only_when_requested() { + let challenge = Uuid::new_v4(); + let url = build_ns_scanner_capability_url( + &NsScannerCapabilityRequest { + endpoint: "http://node1:9000".to_string(), + supports_tier_registry_generation: true, + }, + challenge, + ); + + assert!(url.contains(&format!("&{}=true", NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY))); + } + + #[test] + fn ns_scanner_capability_legacy_fallback_requires_explicit_compatibility_status() { + for status in [400, 404, 405, 426] { + let error = Error::from(rustfs_rio::new_test_internode_http_io_error( + rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::from_u16(status).expect("test status")), + )); + assert!( + ns_scanner_capability_error_allows_legacy(&error), + "status {status} should permit legacy retry" + ); + } + + let marked_server_error = Error::from(rustfs_rio::new_test_internode_http_io_error( + rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::INTERNAL_SERVER_ERROR), + )); + let network_error = Error::from(rustfs_rio::new_test_internode_http_io_error( + rustfs_rio::InternodeHttpErrorKind::ConnectionRefused, + )); + let authentication_error = Error::other("remote namespace scanner capability authentication failed"); + assert!(!ns_scanner_capability_error_allows_legacy(&marked_server_error)); + assert!(!ns_scanner_capability_error_allows_legacy(&network_error)); + assert!(!ns_scanner_capability_error_allows_legacy(&authentication_error)); + } + + #[test] + fn authenticated_ns_scanner_capability_ignores_unprotected_response_bit() { + let mut response = NsScannerCapabilityResponse { + version: NS_SCANNER_PROTOCOL_VERSION, + server_epoch: Uuid::new_v4(), + proof: Vec::new(), + supports_tier_registry_generation: None, + }; + + normalize_ns_scanner_capability_response(&mut response, true); + assert_eq!(response.supports_tier_registry_generation, Some(true)); + + response.supports_tier_registry_generation = Some(false); + normalize_ns_scanner_capability_response(&mut response, false); + assert_eq!(response.supports_tier_registry_generation, None); + } + + #[test] + fn ns_scanner_capability_accepts_only_authenticated_legacy_scope_after_marker_mismatch() { + crate::runtime::sources::ensure_test_rpc_secret(); + let challenge = Uuid::new_v4(); + let response = NsScannerCapabilityResponse { + version: NS_SCANNER_PROTOCOL_VERSION, + server_epoch: Uuid::new_v4(), + proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, Uuid::new_v4()) + .expect("placeholder proof should be generated"), + supports_tier_registry_generation: None, + }; + // A proof bound to a different challenge cannot authorize the legacy + // fallback, even though the response has the expected shape. + assert!(!ns_scanner_capability_legacy_proof_is_valid(challenge, &response)); + + let server_epoch = response.server_epoch; + let valid_response = NsScannerCapabilityResponse { + proof: crate::cluster::rpc::sign_ns_scanner_capability(challenge, server_epoch) + .expect("legacy proof should be generated"), + ..response + }; + assert!(ns_scanner_capability_legacy_proof_is_valid(challenge, &valid_response)); + } + #[test] fn transport_config_defaults_to_tcp_http() { let transport = build_internode_data_transport(None).unwrap(); diff --git a/crates/ecstore/src/cluster/rpc/mod.rs b/crates/ecstore/src/cluster/rpc/mod.rs index 9af454ac4..c8e3024eb 100644 --- a/crates/ecstore/src/cluster/rpc/mod.rs +++ b/crates/ecstore/src/cluster/rpc/mod.rs @@ -35,8 +35,9 @@ pub use http_auth::{ TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest, - sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, - tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer, + sign_ns_scanner_capability, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, + sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, + verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap, diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index a47533a81..1505cffb1 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -781,14 +781,16 @@ impl RemoteDisk { if self.health.is_faulty() { return Err(DiskError::FaultyDisk); } - let probe = self.data_transport.probe_ns_scanner(NsScannerCapabilityRequest { + let probe = self.data_transport.probe_ns_scanner_capability(NsScannerCapabilityRequest { endpoint: self.endpoint.grid_host(), + supports_tier_registry_generation: true, }); let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe) .await .map_err(|_| DiskError::other("remote namespace scanner capability probe timed out"))?; match result { - Ok(server_epoch) => Ok(Some(server_epoch)), + Ok(response) if response.supports_tier_registry_generation == Some(true) => Ok(Some(response.server_epoch)), + Ok(_) => Ok(None), // RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3. Err(DiskError::MethodNotAllowed) => Ok(None), Err(err) @@ -4040,10 +4042,21 @@ mod tests { NsScannerProbe(NsScannerCapabilityRequest), } - #[derive(Debug, Clone, Default)] + #[derive(Debug, Clone)] struct RecordingInternodeDataTransport { calls: Arc>>, ns_scanner_probe_status: Arc>>, + ns_scanner_generation_support: Arc>>, + } + + impl Default for RecordingInternodeDataTransport { + fn default() -> Self { + Self { + calls: Arc::default(), + ns_scanner_probe_status: Arc::default(), + ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))), + } + } } #[derive(Clone, Debug)] @@ -4263,6 +4276,15 @@ mod tests { Self { calls: Arc::default(), ns_scanner_probe_status: Arc::new(StdMutex::new(Some(status))), + ns_scanner_generation_support: Arc::new(StdMutex::new(Some(true))), + } + } + + fn with_ns_scanner_generation_support(support: Option) -> Self { + Self { + calls: Arc::default(), + ns_scanner_probe_status: Arc::default(), + ns_scanner_generation_support: Arc::new(StdMutex::new(support)), } } @@ -4945,6 +4967,23 @@ mod tests { Ok(Uuid::from_u128(1)) } + async fn probe_ns_scanner_capability( + &self, + request: NsScannerCapabilityRequest, + ) -> Result { + let server_epoch = self.probe_ns_scanner(request).await?; + let supports_tier_registry_generation = *self + .ns_scanner_generation_support + .lock() + .expect("namespace scanner generation support lock poisoned"); + Ok(crate::storage_api_contracts::internode::NsScannerCapabilityResponse { + version: crate::storage_api_contracts::internode::NS_SCANNER_PROTOCOL_VERSION, + server_epoch, + proof: Vec::new(), + supports_tier_registry_generation, + }) + } + fn name(&self) -> &'static str { "recording" } @@ -6757,6 +6796,22 @@ mod tests { } } + #[tokio::test] + async fn test_remote_disk_namespace_scanner_capability_falls_back_without_generation_support() { + for support in [None, Some(false)] { + let transport = RecordingInternodeDataTransport::with_ns_scanner_generation_support(support); + let remote_disk = new_remote_disk_with_transport(Arc::new(transport)).await; + + assert_eq!( + remote_disk + .ns_scanner_server_epoch() + .await + .expect("missing generation support should be classified as unsupported"), + None + ); + } + } + #[tokio::test] async fn test_remote_disk_namespace_scanner_capability_rejects_legacy_transport() { let remote_disk = new_remote_disk_with_transport(Arc::new(RetryingOpenReadInternodeDataTransport::default())).await; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index e4b8880e9..c40e8a83a 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -71,7 +71,6 @@ use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, Replicatio use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fmt::Display; -use std::future::Future; #[cfg(test)] use std::io::Cursor; use std::io::Write; @@ -956,7 +955,7 @@ where usize::try_from(size).unwrap_or_default() } -fn with_decommission_entry_context(stage: &str, bucket: &str, object: &str, err: E) -> Error { +fn with_decommission_entry_context(stage: &str, bucket: &str, object: &str, err: E) -> Error { Error::other(format!("decommission entry {stage} failed for bucket {bucket} object {object}: {err}")) } @@ -1743,7 +1742,7 @@ fn is_decommission_copy_cleanup_safe_error(err: &Error) -> bool { // A not-found surfacing from inside a data-movement stage is the same // condition once the wrapper is unwrapped (backlog#1827 T2). - crate::data_movement::data_movement_stage_source(err).is_some_and(is_decommission_copy_cleanup_safe_error) + data_movement::data_movement_stage_source(err).is_some_and(is_decommission_copy_cleanup_safe_error) } fn is_decommission_target_capacity_error(err: &Error) -> bool { @@ -1754,7 +1753,7 @@ fn is_decommission_target_capacity_error(err: &Error) -> bool { // A stage failure keeps the error it wrapped, so classify by type rather // than by the rendered message (backlog#1827 T2). The substring fallback // stays for errors that reached here through some other wrapper. - if let Some(source) = crate::data_movement::data_movement_stage_source(err) { + if let Some(source) = data_movement::data_movement_stage_source(err) { return is_decommission_target_capacity_error(source); } @@ -3101,7 +3100,7 @@ fn decommission_remote_tiered_opts( src_pool_idx, data_movement: true, include_part_checksums: true, - http_preconditions: Some(crate::data_movement::data_movement_target_precondition()), + http_preconditions: Some(data_movement::data_movement_target_precondition()), expected_bucket_incarnation_id, ..Default::default() } @@ -3592,7 +3591,6 @@ impl ECStore { } return Err(err); } - drop(operation_guard); if let Some(canceler) = terminal_canceler.as_ref() { self.release_decommission_canceler_slot(idx, canceler).await; @@ -3850,7 +3848,7 @@ impl ECStore { ) -> Result<()> { let index_cancelers = self.reserve_decommission_routines(&rx, indices.as_slice()).await?; if !index_cancelers.is_empty() { - std::mem::drop(spawn_decommission_index_cancelers( + drop(spawn_decommission_index_cancelers( store, rx, index_cancelers, @@ -3877,7 +3875,7 @@ impl ECStore { return Ok(()); } - std::mem::drop(spawn_decommission_index_cancelers( + drop(spawn_decommission_index_cancelers( self.clone(), rx, index_cancelers, @@ -3907,7 +3905,7 @@ impl ECStore { let index_cancelers = self .start_decommission_with_routines(indices, &rx, local_indices.as_slice()) .await?; - std::mem::drop(spawn_decommission_index_cancelers( + drop(spawn_decommission_index_cancelers( store, rx, index_cancelers, @@ -6982,13 +6980,7 @@ mod tests { #[test] fn decommission_classifiers_see_through_a_stage_wrapper() { let wrap = |inner: Error| { - crate::data_movement::data_movement_stage_error_for_test( - "decommission_object", - "put_object", - "bucket-a", - "object-a", - inner, - ) + data_movement::data_movement_stage_error_for_test("decommission_object", "put_object", "bucket-a", "object-a", inner) }; // Capacity: the target pool filling up must still stop the loop. @@ -7120,7 +7112,7 @@ mod tests { let mod_time = OffsetDateTime::now_utc(); let version = rustfs_filemeta::FileInfo { mod_time: Some(mod_time), - metadata: std::collections::HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]), + metadata: HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]), ..Default::default() }; @@ -11359,8 +11351,6 @@ mod pools_tests { async fn test_decommission_supervisor_observes_worker_panic() { let worker = tokio::spawn(async move { panic!("injected decommission worker panic"); - #[allow(unreachable_code)] - Ok(()) }); let err = await_decommission_worker(4, worker) diff --git a/crates/ecstore/src/storage_api_contracts/mod.rs b/crates/ecstore/src/storage_api_contracts/mod.rs index 78a3f9c2d..30d42ef2c 100644 --- a/crates/ecstore/src/storage_api_contracts/mod.rs +++ b/crates/ecstore/src/storage_api_contracts/mod.rs @@ -27,12 +27,12 @@ pub(crate) mod internode { NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, - NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, - PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN, PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, - PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, - PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, - SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, WALK_DIR_BODY_SHA256_QUERY, - WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1, + NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse, + PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN, + PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, + PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, + SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION, + WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1, }; } diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 79d8fc9b7..e69bd67be 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -30,7 +30,8 @@ pub use rustfs_data_usage::{ AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME, DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, DataUsageSnapshotSetState, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeReconciliationEntry, - SizeReconciliationScope, SizeSummary, TierStats, hash_path, prefix_usage_in_cache, + SizeReconciliationScope, SizeSummary, TierAccountingProof, TierStats, UNKNOWN_TIER, UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP, + UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP, UnknownTierStats, hash_path, prefix_usage_in_cache, }; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use tokio::time::{Duration, Instant, sleep, timeout}; @@ -218,25 +219,79 @@ impl ScannerSizeSummaryExt for SizeSummary { self.versions = self.versions.saturating_add(1); } - let size = usize::try_from(size.max(0)).unwrap_or(usize::MAX); + let logical_size = size.max(0); + let size = usize::try_from(logical_size).unwrap_or(usize::MAX); self.total_size = self.total_size.saturating_add(size); + let logical_bytes = u64::try_from(logical_size).unwrap_or(u64::MAX); + let physical_bytes = u64::try_from(oi.size.max(0)).unwrap_or(0); + let mut proof = TierAccountingProof { + logical_total: logical_bytes, + logical_known: 0, + physical_total: physical_bytes, + physical_known: 0, + overflowed: false, + }; if oi.transitioned_object.free_version { + proof.logical_known = logical_bytes; + proof.physical_known = physical_bytes; + self.tier_accounting_proof.saturating_add(proof); return; } - let mut tier = oi.storage_class.clone().unwrap_or_else(|| storageclass::STANDARD.to_string()); - if oi.transitioned_object.status == TRANSITION_COMPLETE { - tier = oi.transitioned_object.tier.clone(); + let tier = if oi.transitioned_object.status == TRANSITION_COMPLETE { + oi.transitioned_object.tier.as_str() + } else { + oi.storage_class.as_deref().unwrap_or(storageclass::STANDARD) + }; + + let builtin_tier = tier == storageclass::STANDARD || tier == storageclass::RRS; + let tier_registry_is_empty = + self.tier_stats.is_empty() || (self.tier_stats.len() == 1 && self.tier_stats.contains_key(UNKNOWN_TIER)); + let known_tier = tier != UNKNOWN_TIER && (builtin_tier || self.tier_stats.contains_key(tier)); + + // With no configured tier, retain the historical empty-map shape for + // ordinary STANDARD/RRS objects. A non-built-in key is still an + // observable unknown and must create only the fixed bucket. + if tier_registry_is_empty && known_tier { + proof.logical_known = logical_bytes; + proof.physical_known = physical_bytes; + self.tier_accounting_proof.saturating_add(proof); + return; } - if let Some(tier_stats) = self.tier_stats.get_mut(&tier) { - *tier_stats = tier_stats.add(&TierStats { - total_size: u64::try_from(oi.size).unwrap_or(0), - num_versions: 1, - num_objects: u64::from(oi.is_latest), - }); + // Configured tiers and the fixed bucket are normally seeded, so the + // hot path can mutate them without allocating a key for every object. + // The fallback inserts only when a legacy/no-config summary sees its + // first unknown key. + let tier_stats = if known_tier { + if let Some(stats) = self.tier_stats.get_mut(tier) { + stats + } else { + self.tier_stats.entry(tier.to_owned()).or_default() + } + } else if let Some(stats) = self.tier_stats.get_mut(UNKNOWN_TIER) { + stats + } else { + self.tier_stats.entry(UNKNOWN_TIER.to_string()).or_default() + }; + *tier_stats = tier_stats.add(&TierStats { + total_size: physical_bytes, + num_versions: 1, + num_objects: u64::from(oi.is_latest), + }); + if known_tier { + proof.logical_known = logical_bytes; + proof.physical_known = physical_bytes; } + if !known_tier { + self.unknown_tier_stats + .record_dimensions(tier, logical_bytes, physical_bytes, 1, u64::from(oi.is_latest)); + if self.unknown_tier_stats.counter_overflowed { + proof.overflowed = true; + } + } + self.tier_accounting_proof.saturating_add(proof); } fn actions_accounting_unknown(&mut self, oi: &ObjectInfo) { @@ -312,6 +367,11 @@ pub struct DataUsageEntryInfo { pub name: String, pub parent: String, pub entry: DataUsageEntry, + /// Registry generation used to classify this root entry. Older remote + /// workers omit it; callers must reject that result when a frozen cycle + /// requires generation fencing. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tier_registry_generation: Option, } #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] @@ -385,6 +445,10 @@ pub struct DataUsageCacheInfo { pub scan_plan_digest: Option, #[serde(default)] pub cache_key_format: u16, + /// Registry generation used for the completed/partial scan. This is + /// process-local audit data; older cache writers omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tier_registry_generation: Option, /// Bounded durable debts for versions whose logical size was not trusted. /// The map key is an identity key, never a user-controlled metric label. #[serde(default)] @@ -410,7 +474,8 @@ impl Serialize for DataUsageCacheInfo { { // Keep this metadata map-encoded so older readers can ignore fields // appended by newer scanner versions during rolling upgrades. - let field_count = 21 + usize::from(!self.size_reconciliation.is_empty()); + let field_count = + 21 + usize::from(self.tier_registry_generation.is_some()) + usize::from(!self.size_reconciliation.is_empty()); let mut state = serializer.serialize_map(Some(field_count))?; state.serialize_entry("name", &self.name)?; state.serialize_entry("next_cycle", &self.next_cycle)?; @@ -428,6 +493,9 @@ impl Serialize for DataUsageCacheInfo { state.serialize_entry("snapshot_complete", &self.snapshot_complete)?; state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?; state.serialize_entry("cache_key_format", &self.cache_key_format)?; + if let Some(generation) = self.tier_registry_generation { + state.serialize_entry("tier_registry_generation", &generation)?; + } if !self.size_reconciliation.is_empty() { state.serialize_entry("size_reconciliation", &self.size_reconciliation)?; } @@ -456,6 +524,58 @@ pub(crate) enum DataUsageCachePrepareOutcome { } impl DataUsageCache { + /// Reconcile tier keys loaded from an older cache against the registry + /// frozen for this scan. New metadata is already routed through + /// `UNKNOWN_TIER`; this pass handles retired keys that predate that rule. + /// Legacy `TierStats` carries physical bytes only, so this migration does + /// not manufacture a logical unknown-byte value from that physical total. + pub(crate) fn fold_retired_tiers(&mut self, tier_names: &[String]) { + let known_tiers = tier_names.iter().map(String::as_str).collect::>(); + for entry in self.cache.values_mut() { + let Some(tiers) = entry.all_tier_stats.as_mut() else { continue }; + let existing_unknown = tiers.tiers.get(UNKNOWN_TIER).cloned().unwrap_or_default(); + let companion_present = entry.unknown_tier_stats.as_ref().is_some_and(|stats| !stats.is_empty()); + let migrate_existing_unknown = !companion_present; + let mut retired = TierStats::default(); + let mut retired_key_found = false; + if migrate_existing_unknown { + retired = retired.add(&existing_unknown); + } + for (tier, stats) in &tiers.tiers { + if tier != UNKNOWN_TIER + && tier != storageclass::STANDARD + && tier != storageclass::RRS + && !known_tiers.contains(tier.as_str()) + { + retired_key_found = true; + retired = retired.add(stats); + } + } + tiers.fold_unknown_tiers(tier_names.iter().map(String::as_str)); + if !retired.is_empty() && !companion_present { + entry.add_unknown_tier_stats(&UnknownTierStats { + // The legacy map stores physical bytes only. Logical + // bytes remain zero until a fresh object scan observes + // them under the current metadata format. + unknown_physical_bytes: retired.total_size, + unknown_objects: retired.num_objects, + unknown_versions: retired.num_versions, + ..Default::default() + }); + // The legacy tier map has no logical-byte dimension, so a + // proof that classified this retired key as known cannot be + // repaired safely. Mark it unvalidated and require a fresh + // scan rather than guessing a logical subtraction. + entry.tier_accounting_proof = None; + } else if retired_key_found { + // A nonempty companion has no provenance tying it to the + // retired map keys. Reject the mixed cache until a fresh scan + // reconciles the dimensions instead of double-counting them. + entry.tier_accounting_proof = None; + } + } + } + /// Prefix-level usage query over this (writer-side) cache; see /// [`prefix_usage_in_cache`] for the semantics /// (rustfs/backlog#1872). @@ -945,6 +1065,7 @@ impl DataUsageCache { delete_markers_total_count: flat.delete_markers as u64, objects_total_size: flat.size as u64, tier_stats: flat.all_tier_stats.filter(|tiers| !tiers.is_empty()), + unknown_tier_stats: flat.unknown_tier_stats.filter(|stats| !stats.is_empty()), buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX), buckets_usage, ..Default::default() diff --git a/crates/scanner/src/data_usage_define/tests.rs b/crates/scanner/src/data_usage_define/tests.rs index 5c4d55f05..4369adb12 100644 --- a/crates/scanner/src/data_usage_define/tests.rs +++ b/crates/scanner/src/data_usage_define/tests.rs @@ -621,7 +621,6 @@ fn size_summary_add_saturates_all_usage_counters() { failed_count: usize::MAX, }, ); - let mut increment = SizeSummary { total_size: 1, versions: 1, @@ -636,6 +635,24 @@ fn size_summary_add_saturates_all_usage_counters() { failed_count: 1, ..Default::default() }; + summary.tier_stats.insert( + UNKNOWN_TIER.to_string(), + TierStats { + total_size: u64::MAX, + num_versions: u64::MAX, + num_objects: u64::MAX, + }, + ); + increment.tier_stats.insert( + UNKNOWN_TIER.to_string(), + TierStats { + total_size: 1, + num_versions: 1, + num_objects: 1, + }, + ); + increment.unknown_tier_stats.unknown_bytes = 1; + increment.unknown_tier_stats.unknown_physical_bytes = 1; increment.repl_target_stats.insert( target.clone(), ReplTargetSizeSummary { @@ -672,6 +689,8 @@ fn size_summary_add_saturates_all_usage_counters() { assert_eq!(target_summary.failed_size, i64::MAX); assert_eq!(target_summary.pending_count, usize::MAX); assert_eq!(target_summary.failed_count, usize::MAX); + assert_eq!(summary.tier_stats[UNKNOWN_TIER].total_size, u64::MAX); + assert_eq!(summary.unknown_tier_stats.unknown_bytes, 1); } #[test] @@ -721,6 +740,249 @@ fn size_summary_actions_accounting_accumulates_tier_stats() { ); } +#[test] +fn unknown_tier_is_bounded_and_accounted() { + let mut summary = SizeSummary::new(); + summary.tier_stats.insert("WARM".to_string(), TierStats::default()); + let object = ObjectInfo { + storage_class: Some("retired-tier".to_string()), + size: 11, + is_latest: true, + ..Default::default() + }; + + summary.actions_accounting(&object, 11, 11); + + assert_eq!(summary.tier_stats.len(), 2); + assert_eq!(summary.tier_stats.get(UNKNOWN_TIER).map(|stats| stats.total_size), Some(11)); + assert_eq!(summary.unknown_tier_stats.unknown_bytes, 11); + assert_eq!(summary.unknown_tier_stats.unknown_physical_bytes, 11); + assert_eq!(summary.unknown_tier_stats.unknown_objects, 1); + assert_eq!(summary.tier_accounting_proof.logical_total, 11); + assert_eq!(summary.tier_accounting_proof.logical_known, 0); + assert_eq!(summary.tier_accounting_proof.physical_total, 11); + assert_eq!(summary.tier_accounting_proof.physical_known, 0); + assert!(summary.unknown_tier_stats.diagnostics.len() <= UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP); + assert!(summary.unknown_tier_stats.diagnostics.iter().map(String::len).sum::() <= UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP); + assert!( + summary + .unknown_tier_stats + .diagnostics + .iter() + .all(|entry| !entry.contains("retired")) + ); +} + +#[test] +fn unknown_tier_is_accounted_when_no_remote_tier_is_configured() { + let mut summary = SizeSummary::new(); + let object = ObjectInfo { + storage_class: Some("retired-tier".to_string()), + size: 3, + is_latest: true, + ..Default::default() + }; + + summary.actions_accounting(&object, 9, 9); + + assert_eq!(summary.tier_stats.len(), 1); + assert_eq!(summary.tier_stats[UNKNOWN_TIER].total_size, 3); + assert_eq!(summary.unknown_tier_stats.unknown_bytes, 9); + assert_eq!(summary.unknown_tier_stats.unknown_physical_bytes, 3); + + let standard = ObjectInfo { + storage_class: Some(storageclass::STANDARD.to_string()), + size: 4, + is_latest: true, + ..Default::default() + }; + summary.actions_accounting(&standard, 4, 4); + assert_eq!(summary.tier_accounting_proof.logical_total, 13); + assert_eq!(summary.tier_accounting_proof.logical_known, 4); + assert_eq!(summary.tier_accounting_proof.physical_total, 3 + 4); + assert_eq!(summary.tier_accounting_proof.physical_known, 4); + assert_eq!(summary.tier_stats.len(), 1, "built-ins preserve the no-tier map shape"); +} + +#[test] +fn million_unique_tier_keys_do_not_grow_stats_map() { + let mut summary = SizeSummary::new(); + summary.tier_stats.insert("WARM".to_string(), TierStats::default()); + for index in 0..1_000_000_u64 { + let object = ObjectInfo { + storage_class: Some(format!("untrusted-tier-{index}")), + size: 1, + ..Default::default() + }; + summary.actions_accounting(&object, 1, 1); + } + + assert_eq!(summary.tier_stats.len(), 2); + assert_eq!(summary.tier_stats[UNKNOWN_TIER].total_size, 1_000_000); + assert_eq!(summary.unknown_tier_stats.unknown_bytes, 1_000_000); + assert!(summary.unknown_tier_stats.diagnostics.len() <= UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP); + assert!(summary.unknown_tier_stats.diagnostics.iter().map(String::len).sum::() <= UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP); +} + +#[test] +fn unknown_tier_never_triggers_transition() { + let mut summary = SizeSummary::new(); + summary.tier_stats.insert("WARM".to_string(), TierStats::default()); + let mut object = ObjectInfo { + storage_class: Some("removed-tier".to_string()), + size: 7, + ..Default::default() + }; + object.transitioned_object.status = TRANSITION_COMPLETE.to_string(); + object.transitioned_object.tier = "removed-tier".to_string(); + + summary.actions_accounting(&object, 7, 7); + + assert_eq!(summary.tier_stats.get("removed-tier"), None); + assert_eq!(summary.tier_stats[UNKNOWN_TIER].total_size, 7); +} + +#[test] +fn removed_tier_survives_restart_as_unknown() { + let mut summary = SizeSummary::new(); + summary.tier_stats.insert("COLD".to_string(), TierStats::default()); + summary.tier_stats.insert( + "RETIRED".to_string(), + TierStats { + total_size: 5, + num_versions: 1, + num_objects: 1, + }, + ); + let object = ObjectInfo { + storage_class: Some("COLD".to_string()), + size: 5, + ..Default::default() + }; + summary.actions_accounting(&object, 5, 5); + let mut entry = DataUsageEntry::default(); + entry.add_tier_sizes(&summary.tier_stats); + entry.add_unknown_tier_stats(&UnknownTierStats { + unknown_bytes: 2, + unknown_physical_bytes: 2, + unknown_objects: 1, + unknown_versions: 1, + ..Default::default() + }); + let encoded = rmp_serde::to_vec(&entry).expect("entry should encode"); + let restored: DataUsageEntry = rmp_serde::from_slice(&encoded).expect("entry should decode"); + assert_eq!(restored.unknown_tier_stats.as_ref().map(|stats| stats.unknown_bytes), Some(2)); + assert_eq!( + restored.all_tier_stats.as_ref().expect("tier stats persisted").tiers["RETIRED"].total_size, + 5 + ); + + let mut cache = DataUsageCache::default(); + cache.replace("bucket", "", restored); + cache.fold_retired_tiers(&["COLD".to_string()]); + let folded = cache.cache.get(&hash_path("bucket").key()).expect("folded cache entry"); + assert_eq!( + folded.all_tier_stats.as_ref().expect("tier stats persisted").tiers[UNKNOWN_TIER].total_size, + 5 + ); + assert_eq!(folded.unknown_tier_stats.as_ref().map(|stats| stats.unknown_bytes), Some(2)); +} + +#[test] +fn retired_tier_fold_is_idempotent_and_rejects_mixed_companion_provenance() { + let mut cache = DataUsageCache::default(); + let mut entry = DataUsageEntry { + all_tier_stats: Some(AllTierStats { + tiers: HashMap::from([( + "RETIRED".to_string(), + TierStats { + total_size: 5, + num_versions: 1, + num_objects: 1, + }, + )]), + }), + ..Default::default() + }; + entry.unknown_tier_stats = Some(UnknownTierStats { + unknown_physical_bytes: 5, + ..Default::default() + }); + entry.tier_accounting_proof = Some(TierAccountingProof { + physical_total: 5, + physical_known: 5, + ..Default::default() + }); + cache.replace("bucket", "", entry); + + cache.fold_retired_tiers(&["COLD".to_string()]); + let first = cache.cache.get(&hash_path("bucket").key()).expect("entry").clone(); + assert_eq!(first.all_tier_stats.as_ref().expect("tiers").tiers[UNKNOWN_TIER].total_size, 5); + assert_eq!(first.unknown_tier_stats.as_ref().expect("companion").unknown_physical_bytes, 5); + assert!(first.tier_accounting_proof.is_none(), "mixed provenance must not publish"); + + cache.fold_retired_tiers(&["COLD".to_string()]); + let second = cache.cache.get(&hash_path("bucket").key()).expect("entry"); + assert_eq!(second.all_tier_stats.as_ref().expect("tiers").tiers[UNKNOWN_TIER].total_size, 5); + assert_eq!(second.unknown_tier_stats.as_ref().expect("companion").unknown_physical_bytes, 5); +} + +#[test] +fn tier_registry_refresh_does_not_mix_cycle_generations() { + let first = crate::TierRegistrySnapshot { + generation: 1, + names: Arc::from(["WARM".to_string()]), + refresh_failed: false, + }; + let second = crate::TierRegistrySnapshot { + generation: 2, + names: Arc::from(["COLD".to_string()]), + refresh_failed: false, + }; + assert_ne!(first.generation, second.generation); + assert_eq!(first.names.as_ref(), ["WARM".to_string()]); + assert_eq!(second.names.as_ref(), ["COLD".to_string()]); + assert!(first.refreshed(Err(())).refresh_failed); + assert!(!second.refreshed(Ok(Arc::from(["HOT".to_string()]))).refresh_failed); + assert_eq!(first.refreshed(Err(())).generation, first.generation); + assert_eq!(first.refreshed(Err(())).names, first.names); +} + +#[test] +fn unknown_tier_counter_uses_checked_arithmetic() { + let max = TierStats { + total_size: u64::MAX, + num_versions: u64::MAX, + num_objects: u64::MAX, + }; + assert!(max.checked_add(&TierStats::default()).is_some()); + assert!( + max.checked_add(&TierStats { + total_size: 1, + ..Default::default() + }) + .is_none() + ); + + let mut unknown = UnknownTierStats { + unknown_bytes: u64::MAX, + ..Default::default() + }; + unknown.record("overflow", 1, 1, 1); + assert_eq!(unknown.unknown_bytes, u64::MAX); + assert_eq!(unknown.unknown_objects, 1); + assert!(unknown.counter_overflowed); + assert!(unknown.checked_add(&UnknownTierStats::default()).is_none()); + assert!( + unknown + .checked_add(&UnknownTierStats { + unknown_bytes: 1, + ..Default::default() + }) + .is_none() + ); +} + #[test] fn size_summary_unknown_accounting_keeps_physical_tier_and_version_only() { let mut summary = SizeSummary::default(); diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 70fa9e530..301a1aba2 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -24,8 +24,11 @@ use bytes::Bytes; use http::HeaderMap; use rustfs_config::server_config::{Config as ServerConfig, get_global_server_config as config_get_global_server_config}; +use sha2::{Digest as _, Sha256}; +use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; +use std::sync::LazyLock; use std::sync::RwLock; use std::time::{Duration, Instant}; use storage_api::owner::{ @@ -96,6 +99,45 @@ static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0); static SCANNER_FOREGROUND_READ_ACTIVITY: AtomicU64 = AtomicU64::new(0); static SCANNER_FOREGROUND_STREAM_READS: AtomicU64 = AtomicU64::new(0); +/// Immutable tier registry captured at the beginning of a folder scan. +/// Generation makes it possible to prove that a result was classified against +/// one registry even when the process-wide TTL cache refreshes concurrently. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TierRegistrySnapshot { + pub(crate) generation: u64, + pub(crate) names: Arc<[String]>, + /// True when the last refresh attempt failed and `names` is therefore a + /// retained last-good snapshot rather than a newly read registry. + pub(crate) refresh_failed: bool, +} + +impl TierRegistrySnapshot { + /// Apply a refresh only when the registry read succeeds. A failed refresh + /// retains the prior generation, preventing a transient config failure + /// from classifying the remainder of a scan against an empty registry. + pub(crate) fn refreshed(&self, names: Result, ()>) -> Self { + match names { + Ok(names) => Self { + generation: self.generation.saturating_add(1), + names, + refresh_failed: false, + }, + Err(()) => Self { + refresh_failed: true, + ..self.clone() + }, + } + } + + pub(crate) fn initial(names: Arc<[String]>) -> Self { + Self { + generation: 1, + names, + refresh_failed: false, + } + } +} + pub fn current_scanner_activity() -> u64 { SCANNER_ACTIVE_WORK_UNITS.load(Ordering::Relaxed) } @@ -373,6 +415,7 @@ pub(crate) fn resolve_scanner_server_config() -> Option { /// How long the scanner caches the runtime tier-name list before re-reading /// the tier configuration manager. const TIER_NAME_CACHE_TTL: Duration = Duration::from_secs(30); +const MAX_TIER_REGISTRY_NAME_BYTES: usize = 256; /// Process-wide TTL cache of runtime tier names. /// @@ -385,24 +428,192 @@ const TIER_NAME_CACHE_TTL: Duration = Duration::from_secs(30); /// `TIER_NAME_CACHE_TTL` later; a removed tier can leave an all-zero /// `TierStats` seed behind for one cache generation, which merges harmlessly /// by key in per-object accounting and disappears on the next refresh. -static TIER_NAME_CACHE: RwLock)>> = RwLock::new(None); +static TIER_NAME_CACHE: RwLock> = RwLock::new(None); +static TIER_REGISTRY_GENERATION: AtomicU64 = AtomicU64::new(0); +static TIER_CYCLE_SNAPSHOTS: LazyLock>> = + LazyLock::new(|| RwLock::new(HashMap::new())); +static TIER_ACTIVE_CYCLES: LazyLock>> = LazyLock::new(|| RwLock::new(HashMap::new())); +static TIER_NAME_REFRESH_LOCK: LazyLock> = LazyLock::new(|| tokio::sync::Mutex::new(())); + +/// Return one immutable registry snapshot for a scanner unit of work. +pub(crate) async fn runtime_tier_registry() -> TierRegistrySnapshot { + { + let cached = TIER_NAME_CACHE.read().unwrap_or_else(|err| err.into_inner()).clone(); + if let Some((refreshed_at, snapshot)) = cached + && refreshed_at.elapsed() < TIER_NAME_CACHE_TTL + { + return snapshot; + } + } + + // Serialize refreshes so a slower read of the old config cannot overwrite + // a newer snapshot published by a concurrent caller. + let _refresh_guard = TIER_NAME_REFRESH_LOCK.lock().await; + { + let cached = TIER_NAME_CACHE.read().unwrap_or_else(|err| err.into_inner()).clone(); + if let Some((refreshed_at, snapshot)) = cached + && refreshed_at.elapsed() < TIER_NAME_CACHE_TTL + { + return snapshot; + } + } + + let previous = TIER_NAME_CACHE + .read() + .unwrap_or_else(|err| err.into_inner()) + .as_ref() + .map(|(_, snapshot)| snapshot.clone()); + let names = ecstore_get_global_tier_config_mgr() + .read() + .await + .list_tiers() + .into_iter() + .map(|tier| tier.name) + .collect::>(); + let snapshot = match validate_tier_registry_names(names) { + Ok(names) => { + let generation = next_tier_registry_generation(); + match previous { + Some(previous) => TierRegistrySnapshot { + generation, + ..previous.refreshed(Ok(names)) + }, + None => TierRegistrySnapshot { + generation, + ..TierRegistrySnapshot::initial(names) + }, + } + } + Err(()) => match previous { + Some(previous) => previous.refreshed(Err(())), + None => TierRegistrySnapshot { + generation: next_tier_registry_generation(), + names: Arc::new([]), + refresh_failed: true, + }, + }, + }; + *TIER_NAME_CACHE.write().unwrap_or_else(|err| err.into_inner()) = Some((Instant::now(), snapshot.clone())); + snapshot +} + +fn next_tier_registry_generation() -> u64 { + TIER_REGISTRY_GENERATION + .fetch_update(Ordering::AcqRel, Ordering::Relaxed, |current| Some(current.saturating_add(1))) + .unwrap_or(u64::MAX) +} + +fn validate_tier_registry_names(mut names: Vec) -> Result, ()> { + if names.iter().any(|name| { + name.is_empty() + || name.len() > MAX_TIER_REGISTRY_NAME_BYTES + || name.bytes().any(|byte| byte.is_ascii_control()) + || name == UNKNOWN_TIER + || name == storageclass::STANDARD + || name == storageclass::RRS + }) { + return Err(()); + } + names.sort_unstable(); + if names.windows(2).any(|pair| pair[0] == pair[1]) { + return Err(()); + } + Ok(names.into()) +} /// Tier names currently registered in the tier configuration, cached for /// `TIER_NAME_CACHE_TTL`. pub(crate) async fn runtime_tier_names() -> Arc<[String]> { + runtime_tier_registry().await.names +} + +/// Return the immutable tier registry for one scanner cycle/leader pair. +/// Different buckets and disks belonging to the same cycle share this entry, +/// so a TTL refresh cannot split one published cycle across generations. +pub(crate) async fn runtime_tier_registry_for_cycle(cycle: u64, leader_epoch: u64) -> TierRegistrySnapshot { + let key = (cycle, leader_epoch); + prune_inactive_tier_cycle_snapshots(cycle, leader_epoch); { - let cached = TIER_NAME_CACHE.read().unwrap_or_else(|err| err.into_inner()).clone(); - if let Some((refreshed_at, names)) = cached - && refreshed_at.elapsed() < TIER_NAME_CACHE_TTL - { - return names; + let cached = TIER_CYCLE_SNAPSHOTS.read().unwrap_or_else(|err| err.into_inner()); + if let Some(snapshot) = cached.get(&key) { + return snapshot.clone(); } } - let tiers = ecstore_get_global_tier_config_mgr().read().await.list_tiers(); - let names: Arc<[String]> = tiers.iter().map(|tier| tier.name.clone()).collect::>().into(); - *TIER_NAME_CACHE.write().unwrap_or_else(|err| err.into_inner()) = Some((Instant::now(), Arc::clone(&names))); - names + let mut snapshot = runtime_tier_registry().await; + // The registry generation describes the configuration snapshot, not the + // scan that consumed it. Keep it stable across cycles so a healthy cache + // can be reused; cycle and leader fencing are carried separately by the + // cache metadata and scan plan. + snapshot.generation = tier_registry_generation(&snapshot.names); + let mut cached = TIER_CYCLE_SNAPSHOTS.write().unwrap_or_else(|err| err.into_inner()); + if let Some(existing) = cached.get(&key) { + return existing.clone(); + } + cached.insert(key, snapshot.clone()); + snapshot +} + +fn prune_inactive_tier_cycle_snapshots(cycle: u64, leader_epoch: u64) { + let active = TIER_ACTIVE_CYCLES.read().unwrap_or_else(|err| err.into_inner()); + let mut snapshots = TIER_CYCLE_SNAPSHOTS.write().unwrap_or_else(|err| err.into_inner()); + snapshots.retain(|(entry_cycle, entry_epoch), _| { + active.contains_key(&(*entry_cycle, *entry_epoch)) + || *entry_epoch > leader_epoch + || (*entry_epoch == leader_epoch && *entry_cycle >= cycle) + }); +} + +pub(crate) struct TierRegistryCycleGuard { + key: (u64, u64), +} + +impl Drop for TierRegistryCycleGuard { + fn drop(&mut self) { + let mut active = TIER_ACTIVE_CYCLES.write().unwrap_or_else(|err| err.into_inner()); + if let Some(count) = active.get_mut(&self.key) { + *count = count.saturating_sub(1); + if *count == 0 { + active.remove(&self.key); + } + } + } +} + +pub(crate) fn begin_tier_registry_cycle(cycle: u64, leader_epoch: u64) -> TierRegistryCycleGuard { + let mut active = TIER_ACTIVE_CYCLES.write().unwrap_or_else(|err| err.into_inner()); + let count = active.entry((cycle, leader_epoch)).or_default(); + *count = count.saturating_add(1); + TierRegistryCycleGuard { + key: (cycle, leader_epoch), + } +} + +fn tier_registry_generation(names: &[String]) -> u64 { + let mut hasher = Sha256::new(); + hasher.update(b"rustfs-tier-registry-v1"); + for name in names { + hasher.update(u64::try_from(name.len()).unwrap_or(u64::MAX).to_le_bytes()); + hasher.update(name.as_bytes()); + } + let digest = hasher.finalize(); + let mut prefix = [0_u8; 8]; + prefix.copy_from_slice(&digest[..8]); + u64::from_le_bytes(prefix) +} + +/// Drop cycle snapshots only after the scanner has finished publishing a +/// cycle. In-flight or retryable cycles must retain their original registry; +/// TTL/capacity eviction could make a later bucket in the same cycle refresh +/// to a different generation. +pub(crate) fn complete_tier_registry_cycle(cycle: u64, leader_epoch: u64) { + let active = TIER_ACTIVE_CYCLES.read().unwrap_or_else(|err| err.into_inner()); + let mut cached = TIER_CYCLE_SNAPSHOTS.write().unwrap_or_else(|err| err.into_inner()); + cached.retain(|(entry_cycle, entry_epoch), _| { + active.contains_key(&(*entry_cycle, *entry_epoch)) + || *entry_epoch > leader_epoch + || (*entry_epoch == leader_epoch && *entry_cycle > cycle) + }); } /// Test-only cache reset; the production cache has no invalidation hook @@ -410,6 +621,9 @@ pub(crate) async fn runtime_tier_names() -> Arc<[String]> { #[cfg(test)] fn reset_tier_name_cache_for_test() { *TIER_NAME_CACHE.write().unwrap_or_else(|err| err.into_inner()) = None; + TIER_ACTIVE_CYCLES.write().unwrap_or_else(|err| err.into_inner()).clear(); + TIER_CYCLE_SNAPSHOTS.write().unwrap_or_else(|err| err.into_inner()).clear(); + TIER_REGISTRY_GENERATION.store(0, Ordering::Relaxed); } pub(crate) async fn enqueue_runtime_free_version(oi: ScannerObjectInfo) { @@ -742,6 +956,68 @@ mod tests { assert!(Arc::ptr_eq(&first, &second)); } + #[tokio::test] + async fn tier_registry_cycle_snapshot_stays_fixed_while_active() { + reset_tier_name_cache_for_test(); + let cycle = 9_000_001; + let leader_epoch = 9_000_002; + let guard = begin_tier_registry_cycle(cycle, leader_epoch); + let first = runtime_tier_registry_for_cycle(cycle, leader_epoch).await; + + // Simulate a TTL refresh observing a different configuration while the + // original cycle is still scanning. The active cycle entry must win. + *TIER_NAME_CACHE.write().unwrap_or_else(|err| err.into_inner()) = Some(( + Instant::now() - TIER_NAME_CACHE_TTL - Duration::from_secs(1), + TierRegistrySnapshot { + generation: u64::MAX, + names: Arc::from(["COLD".to_string()]), + refresh_failed: false, + }, + )); + let second = runtime_tier_registry_for_cycle(cycle, leader_epoch).await; + assert_eq!(second.generation, first.generation); + assert_eq!(second.names, first.names); + + drop(guard); + complete_tier_registry_cycle(cycle, leader_epoch); + reset_tier_name_cache_for_test(); + } + + #[tokio::test] + async fn tier_registry_generation_survives_new_cycle_with_same_names() { + reset_tier_name_cache_for_test(); + let first_cycle = 9_000_011; + let second_cycle = first_cycle + 1; + let leader_epoch = 9_000_012; + + let first_guard = begin_tier_registry_cycle(first_cycle, leader_epoch); + let first = runtime_tier_registry_for_cycle(first_cycle, leader_epoch).await; + drop(first_guard); + complete_tier_registry_cycle(first_cycle, leader_epoch); + + let second_guard = begin_tier_registry_cycle(second_cycle, leader_epoch); + let second = runtime_tier_registry_for_cycle(second_cycle, leader_epoch).await; + assert_eq!(first.names, second.names); + assert_eq!(first.generation, second.generation); + + drop(second_guard); + complete_tier_registry_cycle(second_cycle, leader_epoch); + reset_tier_name_cache_for_test(); + } + + #[test] + fn invalid_tier_registry_names_fail_closed_for_refresh() { + assert!(validate_tier_registry_names(vec!["COLD\n".to_string()]).is_err()); + assert!(validate_tier_registry_names(vec![UNKNOWN_TIER.to_string()]).is_err()); + assert!(validate_tier_registry_names(vec!["COLD".to_string(), "COLD".to_string()]).is_err()); + assert_eq!( + validate_tier_registry_names(vec!["WARM".to_string(), "COLD".to_string()]) + .expect("valid registry names") + .as_ref(), + ["COLD".to_string(), "WARM".to_string()] + ); + } + #[test] fn foreground_read_guard_tracks_stream_lifetime() { reset_foreground_read_activity_for_test(); diff --git a/crates/scanner/src/remote_scanner/stream.rs b/crates/scanner/src/remote_scanner/stream.rs index ef6418b9c..594473196 100644 --- a/crates/scanner/src/remote_scanner/stream.rs +++ b/crates/scanner/src/remote_scanner/stream.rs @@ -16,8 +16,8 @@ use crate::RUSTFS_META_BUCKET; use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig}; use crate::scanner_io::{ - DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, acquire_scanner_cache_locks, cache_root_entry_info, - current_cache_root_or_prepare, scanner_set_disk_inventory, + DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, acquire_scanner_cache_locks, + cache_root_entry_info, current_cache_root_or_prepare_with_generation, scanner_set_disk_inventory, }; use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION; use crate::{ @@ -213,6 +213,7 @@ pub(crate) struct RemoteScannerScanSpec<'a> { pub(crate) session_id: Uuid, pub(crate) session_sequence: u64, pub(crate) scan_plan_digest: DataUsageScanPlanDigest, + pub(crate) tier_registry_generation: u64, pub(crate) skip_healing: bool, pub(crate) scan_mode: HealScanMode, } @@ -223,6 +224,7 @@ struct RemoteScannerResponseExpectation<'a> { source: DataUsageCacheSource, next_cycle: u64, scan_plan_digest: DataUsageScanPlanDigest, + tier_registry_generation: u64, } #[derive(Debug)] @@ -669,6 +671,11 @@ async fn scan_and_persist_local_bucket( scan_mode, .. } = request; + // Keep the worker's cycle snapshot alive through cache reuse, scanning, + // and persistence. Without the guard, a later cycle can prune this key + // while this request is still running and allow a second registry to be + // selected for the same cycle. + let _tier_cycle_guard = crate::begin_tier_registry_cycle(next_cycle, leader_epoch); let store = resolve_scanner_object_store_handle() .ok_or_else(|| RemoteScannerServerError::worker("remote namespace scanner object layer is unavailable"))?; validate_remote_scanner_request_fence_with_store(next_cycle, leader_epoch, store.clone()) @@ -704,7 +711,25 @@ async fn scan_and_persist_local_bucket( let revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| { RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}")) })?; - let scan_state = current_cache_root_or_prepare(&mut cache, &bucket, source, next_cycle, leader_epoch, scan_plan_digest, true); + // Remote workers use the same cycle-frozen registry as `scan_data_folder`. + // Requiring its generation here prevents a cache snapshot classified by an + // older registry from being reused before the folder scan gets a chance to + // refresh it. + let tier_registry_generation = crate::runtime_tier_registry_for_cycle(next_cycle, leader_epoch) + .await + .generation; + let scan_state = current_cache_root_or_prepare_with_generation( + &mut cache, + &bucket, + source, + next_cycle, + leader_epoch, + scan_plan_digest, + DataUsageCacheReuseOptions { + require_source: true, + tier_registry_generation: Some(tier_registry_generation), + }, + ); match scan_state { DataUsageCacheScanState::Current(usage) => { if guard.is_lock_lost() { @@ -869,6 +894,7 @@ pub(crate) async fn scan_remote_bucket( session_id, session_sequence, scan_plan_digest, + tier_registry_generation, skip_healing, scan_mode, } = spec; @@ -957,6 +983,7 @@ pub(crate) async fn scan_remote_bucket( source: expected_source, next_cycle, scan_plan_digest, + tier_registry_generation, }, authenticator, rpc_deadline, @@ -1012,6 +1039,7 @@ where source: expected_source, next_cycle: TEST_NEXT_CYCLE, scan_plan_digest: expected_scan_plan_digest, + tier_registry_generation: 0, }, authenticator, Instant::now() + NS_SCANNER_MAX_RPC_LIFETIME, @@ -1111,6 +1139,11 @@ where "remote namespace scanner returned usage for a different bucket plan", ))); } + if complete.usage.tier_registry_generation != Some(expected.tier_registry_generation) { + return Err(RemoteScannerStreamError::reconciled(StorageError::other( + "remote namespace scanner returned usage for a different tier registry generation", + ))); + } if !complete.usage.entry.children.is_empty() { return Err(RemoteScannerStreamError::reconciled(StorageError::other( "remote namespace scanner returned non-flattened bucket usage", diff --git a/crates/scanner/src/remote_scanner/stream/tests.rs b/crates/scanner/src/remote_scanner/stream/tests.rs index 11d28392e..ba0b71e58 100644 --- a/crates/scanner/src/remote_scanner/stream/tests.rs +++ b/crates/scanner/src/remote_scanner/stream/tests.rs @@ -195,6 +195,7 @@ fn test_usage(bucket: &str, objects: usize) -> DataUsageEntryInfo { name: bucket.to_string(), parent: crate::DATA_USAGE_ROOT.to_string(), entry, + tier_registry_generation: Some(0), } } @@ -751,6 +752,43 @@ async fn complete_terminal_frame_reconciles_progress_and_usage() { assert_eq!(budget.progress(), (3, 2)); } +#[tokio::test] +async fn terminal_usage_from_a_different_tier_generation_is_rejected() { + let request_id = Uuid::new_v4(); + let writer_auth = FrameAuthenticator::for_test(request_id); + let reader_auth = FrameAuthenticator::for_test(request_id); + let (mut writer, reader) = tokio::io::duplex(4096); + tokio::spawn(async move { + let mut usage = test_usage("bucket", 1); + usage.tier_registry_generation = Some(1); + let mut sequence = 0; + write_frame( + &mut writer, + &writer_auth, + &mut sequence, + &RemoteScannerFrame::terminal( + RemoteScannerProgress::default(), + RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete { + source: TEST_SOURCE, + scan_plan_digest: TEST_PLAN_DIGEST, + usage, + pending_maintenance_work: false, + })), + ), + ) + .await + .expect("terminal frame should write"); + }); + + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new(&parent, ScannerCycleBudgetConfig::default()); + let error = consume_remote_scanner_stream(reader, parent, budget, "bucket", TEST_SOURCE, TEST_PLAN_DIGEST, reader_auth) + .await + .expect_err("generation mismatch must fail closed"); + + assert!(error.to_string().contains("tier registry generation")); +} + #[tokio::test] async fn complete_terminal_frame_after_budget_expiry_is_partial() { let request_id = Uuid::new_v4(); diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index bd9727154..b4a3b96e8 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -59,9 +59,9 @@ use tracing::{debug, error, warn}; use crate::{ Disk, DiskError, DiskInfoOptions, Evaluator, Event, LcEventSrc, ListPathRawOptions, ObjectOpts, ReplicationConfig, ReplicationHealObject, ReplicationQueueAdmission, ReplicationStatusType, STORAGE_FORMAT_FILE, ScannerDiskExt as _, - ScannerLifecycleConfigExt as _, ScannerVersioningConfigExt as _, StorageError, apply_expiry_rule, apply_transition_rule, - enqueue_runtime_newer_noncurrent, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object, - path2_bucket_object_with_base_path, queue_replication_heal, scanner_is_erasure, + ScannerLifecycleConfigExt as _, ScannerVersioningConfigExt as _, StorageError, TierRegistrySnapshot, apply_expiry_rule, + apply_transition_rule, enqueue_runtime_newer_noncurrent, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object, + path2_bucket_object_with_base_path, queue_replication_heal, runtime_tier_registry_for_cycle, scanner_is_erasure, scanner_replication_config_for_lifecycle_eval, }; use crate::{ScannerObjectInfo as ObjectInfo, ScannerObjectToDelete as ObjectToDelete}; @@ -638,6 +638,20 @@ fn apply_scanner_size_summary(into: &mut DataUsageEntry, summary: &SizeSummary) } into.add_tier_sizes(&summary.tier_stats); + into.add_unknown_tier_stats(&summary.unknown_tier_stats); + into.tier_accounting_proof = match (into.tier_accounting_proof, Some(summary.tier_accounting_proof)) { + (Some(mut current), Some(next)) => { + current.saturating_add(next); + Some(current) + } + (Some(_), None) => None, + (None, next) => next, + }; + if into.unknown_tier_stats.as_ref().is_some_and(|stats| stats.counter_overflowed) + && let Some(proof) = into.tier_accounting_proof.as_mut() + { + proof.overflowed = true; + } } fn data_usage_root_has_progress(root: &DataUsageEntry) -> bool { @@ -648,6 +662,8 @@ fn data_usage_root_has_progress(root: &DataUsageEntry) -> bool { || root.delete_markers > 0 || root.failed_objects > 0 || root.replication_stats.is_some() + || root.all_tier_stats.as_ref().is_some_and(|stats| !stats.is_empty()) + || root.unknown_tier_stats.as_ref().is_some_and(|stats| !stats.is_empty()) } fn partial_cache_is_useful(root: &DataUsageEntry, pending_heals_changed: bool) -> bool { @@ -682,6 +698,9 @@ pub struct FolderScanner { budget: Arc, skip_heal: Arc, local_disk: Arc, + /// Tier registry frozen for this folder scan. A refresh applies to the + /// next scan and cannot mix generations in one aggregate. + tier_registry: TierRegistrySnapshot, pending_heals_changed: bool, pending_size_reconciliation_keys: HashSet, pending_size_reconciliation_scopes: HashSet, @@ -1450,7 +1469,11 @@ impl FolderScanner { continue; } - let sz = match self.local_disk.get_size(item.clone()).await { + let sz = match self + .local_disk + .get_size_with_tier_names(item.clone(), &self.tier_registry.names) + .await + { Ok(sz) => sz, Err(e) => { let failure_action = classify_get_size_failure(&item, &e); @@ -2326,6 +2349,10 @@ pub async fn scan_data_folder( let failed_object_ttl = rustfs_utils::get_env_u32(ENV_FAILED_OBJECT_TTL_SECS, DEFAULT_FAILED_OBJECT_TTL_SECS) as u64; let failed_objects_max = rustfs_utils::get_env_u32(ENV_FAILED_OBJECTS_MAX, DEFAULT_FAILED_OBJECTS_MAX) as usize; + let tier_registry = runtime_tier_registry_for_cycle(cache.info.next_cycle, cache.info.leader_epoch).await; + let mut cache = cache; + cache.fold_retired_tiers(&tier_registry.names); + cache.info.tier_registry_generation = Some(tier_registry.generation); // Create folder scanner let mut scanner = FolderScanner { @@ -2354,6 +2381,7 @@ pub async fn scan_data_folder( budget: budget.clone(), skip_heal, local_disk, + tier_registry, pending_heals_changed: false, pending_size_reconciliation_keys: HashSet::new(), pending_size_reconciliation_scopes: HashSet::new(), diff --git a/crates/scanner/src/scanner_folder/item_actions.rs b/crates/scanner/src/scanner_folder/item_actions.rs index 635637199..490d80017 100644 --- a/crates/scanner/src/scanner_folder/item_actions.rs +++ b/crates/scanner/src/scanner_folder/item_actions.rs @@ -566,11 +566,62 @@ impl ScannerItem { item.object_path() } + fn effective_tier(oi: &ObjectInfo) -> &str { + if oi.transitioned_object.status == crate::TRANSITION_COMPLETE { + oi.transitioned_object.tier.as_str() + } else { + oi.storage_class.as_deref().unwrap_or(crate::storageclass::STANDARD) + } + } + + fn tier_name_is_known(tier: &str, tier_names: &[String]) -> bool { + !tier.is_empty() + && tier != crate::data_usage_define::UNKNOWN_TIER + && (tier == crate::storageclass::STANDARD + || tier == crate::storageclass::RRS + || tier_names.iter().any(|name| name == tier)) + } + + pub(crate) fn tier_is_known(oi: &ObjectInfo, tier_names: &[String]) -> bool { + Self::tier_name_is_known(Self::effective_tier(oi), tier_names) + } + + fn action_requires_known_tier(action: IlmAction) -> bool { + matches!( + action, + IlmAction::TransitionAction + | IlmAction::TransitionVersionAction + | IlmAction::DeleteAction + | IlmAction::DeleteVersionAction + | IlmAction::DeleteRestoredAction + | IlmAction::DeleteRestoredVersionAction + | IlmAction::DeleteAllVersionsAction + | IlmAction::DelMarkerDeleteAllVersionsAction + ) + } + + fn action_blocked_by_unknown_tier( + action: IlmAction, + oi: &ObjectInfo, + all_versions_known: bool, + tier_names: &[String], + target: &str, + ) -> bool { + if !Self::action_requires_known_tier(action) { + return false; + } + !Self::tier_is_known(oi, tier_names) + || (action.delete_all() && !all_versions_known) + || (matches!(action, IlmAction::TransitionAction | IlmAction::TransitionVersionAction) + && !Self::tier_name_is_known(target, tier_names)) + } + pub async fn apply_actions( &mut self, object_infos: Vec, lock_retention: Option>, versioning_config: VersioningConfiguration, + tier_names: &[String], size_summary: &mut SizeSummary, ) { let object_path = self.object_path(); @@ -694,6 +745,9 @@ impl ScannerItem { let mut noncurrent_unknown: Vec<&ObjectInfo> = Vec::new(); let mut cumulative_size = 0; let mut remaining_versions = object_infos.len(); + let all_versions_known = object_infos + .iter() + .all(|candidate| Self::tier_is_known(candidate, tier_names)); 'eventLoop: { for (i, event) in events.iter().enumerate() { let oi = &object_infos[i]; @@ -799,6 +853,18 @@ impl ScannerItem { let mut size = actual_size; let mut account_now = true; + // A retired/unknown source tier may point at a remote object + // that cannot be safely deleted or transitioned. Lifecycle + // evaluation is still useful for accounting, but all + // side-effecting tier actions fail closed until the registry + // recognizes the source again. + if Self::action_blocked_by_unknown_tier(event.action, oi, all_versions_known, tier_names, &event.storage_class) { + size = self.heal_actions(oi, actual_size, size_summary).await; + size_summary.actions_accounting(oi, size, actual_size); + cumulative_size += size; + continue; + } + match event.action { IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => { debug!( @@ -1303,6 +1369,51 @@ mod tests { assert_eq!(item.object_path(), "object"); } + #[test] + fn unknown_tier_never_triggers_transition() { + let object = ObjectInfo { + storage_class: Some("retired-tier".to_string()), + ..Default::default() + }; + let tier_names = ["WARM".to_string()]; + assert!(!ScannerItem::tier_is_known(&object, &tier_names)); + assert!(ScannerItem::action_requires_known_tier(IlmAction::TransitionAction)); + assert!(ScannerItem::action_requires_known_tier(IlmAction::DeleteVersionAction)); + assert!(ScannerItem::action_blocked_by_unknown_tier( + IlmAction::TransitionAction, + &object, + false, + &tier_names, + "WARM" + )); + assert!(!ScannerItem::action_blocked_by_unknown_tier( + IlmAction::NoneAction, + &object, + false, + &tier_names, + "WARM" + )); + + let known = ObjectInfo { + storage_class: Some(crate::storageclass::STANDARD.to_string()), + ..Default::default() + }; + assert!(ScannerItem::action_blocked_by_unknown_tier( + IlmAction::DeleteAllVersionsAction, + &known, + false, + &tier_names, + "WARM" + )); + assert!(!ScannerItem::action_blocked_by_unknown_tier( + IlmAction::TransitionAction, + &known, + true, + &tier_names, + crate::storageclass::STANDARD + )); + } + #[test] fn size_resolution_rejects_negative_overflow_and_unknown_compression() { let compressed = |actual_size: i64, declared: Option<&str>| { @@ -1681,7 +1792,7 @@ mod tests { ..Default::default() }; let mut summary = SizeSummary::default(); - item.apply_actions(vec![object], None, VersioningConfiguration::default(), &mut summary) + item.apply_actions(vec![object], None, VersioningConfiguration::default(), &[], &mut summary) .await; let bounded_bucket = bounded_reconciliation_field(&item.bucket); diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index 393ff50cc..228fad564 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -337,6 +337,11 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) { budget: ScannerCycleBudget::new(&CancellationToken::new(), Default::default()), skip_heal: Arc::new(AtomicBool::new(false)), local_disk: disk, + tier_registry: crate::TierRegistrySnapshot { + generation: 0, + names: Arc::new([]), + refresh_failed: false, + }, pending_heals_changed: false, pending_size_reconciliation_keys: HashSet::new(), pending_size_reconciliation_scopes: HashSet::new(), diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 0a5ecdf12..bd10ee583 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -58,7 +58,8 @@ use crate::{ BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result, RUSTFS_META_BUCKET, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerConfigObjectDelete as _, ScannerDiskExt as _, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, ScannerVersioningConfigExt as _, SetDisks, StorageError, - enqueue_runtime_free_version, get_lifecycle_config, get_object_lock_config, get_replication_config, runtime_tier_names, + begin_tier_registry_cycle, complete_tier_registry_cycle, enqueue_runtime_free_version, get_lifecycle_config, + get_object_lock_config, get_replication_config, runtime_tier_names, runtime_tier_registry_for_cycle, scanner_publication_admission_for_epoch, scanner_publication_epoch, storageclass, }; @@ -144,6 +145,7 @@ pub struct ScannerBucketScanPlan { all_buckets: Arc>, digest: DataUsageScanPlanDigest, leader_epoch: u64, + tier_registry_generation: u64, /// Epoch captured once for the whole scanner cycle. `None` is retained /// for unfenced test implementations; production plans always carry the /// admission token captured before bucket enumeration. @@ -357,12 +359,20 @@ pub(crate) fn cache_root_entry_info(cache: &DataUsageCache) -> std::result::Resu name: cache.info.name.clone(), parent: DATA_USAGE_ROOT.to_string(), entry, + tier_registry_generation: cache.info.tier_registry_generation, }) } -fn apply_bucket_result_to_cache(cache: &mut DataUsageCache, result: DataUsageEntryInfo, update_time: SystemTime) { +fn apply_bucket_result_to_cache(cache: &mut DataUsageCache, result: DataUsageEntryInfo, update_time: SystemTime) -> bool { + if cache.info.tier_registry_generation != result.tier_registry_generation { + // A result from another registry generation must never be folded into + // this cycle. Leaving it unapplied makes the cycle incomplete and + // forces the caller to re-account it under one frozen registry. + return false; + } cache.replace(&result.name, &result.parent, result.entry); cache.info.last_update = Some(update_time); + true } fn should_publish_completed_snapshot(completed_count: usize, total_count: usize, budget_elapsed: bool, cancelled: bool) -> bool { @@ -514,6 +524,9 @@ pub trait ScannerIODisk: Send + Sync + Debug + 'static { ) -> Result; async fn get_size(&self, item: ScannerItem) -> Result; + + /// Read one object using a registry snapshot captured at scan start. + async fn get_size_with_tier_names(&self, item: ScannerItem, tier_names: &[String]) -> Result; } #[derive(Debug)] @@ -676,7 +689,10 @@ use cache::*; use dirty_usage::*; use guards::*; -pub(crate) use cache::{DataUsageCacheScanState, acquire_scanner_cache_locks, current_cache_root_or_prepare}; +pub(crate) use cache::{ + DataUsageCacheReuseOptions, DataUsageCacheScanState, acquire_scanner_cache_locks, + current_cache_root_or_prepare_with_generation, +}; pub use dirty_usage::{ ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state, diff --git a/crates/scanner/src/scanner_io/cache.rs b/crates/scanner/src/scanner_io/cache.rs index 070a89deb..7b12d7932 100644 --- a/crates/scanner/src/scanner_io/cache.rs +++ b/crates/scanner/src/scanner_io/cache.rs @@ -100,13 +100,14 @@ where let _ = tokio::time::timeout(SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan).await; } -pub(crate) fn current_cache_root_entry( +pub(crate) fn current_cache_root_entry_with_generation( cache: &DataUsageCache, name: &str, source: DataUsageCacheSource, next_cycle: u64, leader_epoch: u64, scan_plan_digest: DataUsageScanPlanDigest, + tier_registry_generation: Option, ) -> std::result::Result, ScannerError> { let metadata_is_current = cache.info.name == name && cache.info.source == Some(source) @@ -115,7 +116,8 @@ pub(crate) fn current_cache_root_entry( && cache.info.last_update.is_some() && cache.info.next_cycle == next_cycle && cache.info.leader_epoch == leader_epoch - && cache.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT; + && cache.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT + && tier_registry_generation.is_none_or(|generation| cache.info.tier_registry_generation == Some(generation)); if !metadata_is_current { return Ok(None); } @@ -131,6 +133,13 @@ pub(crate) enum DataUsageCacheScanState { }, } +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct DataUsageCacheReuseOptions { + pub(crate) require_source: bool, + pub(crate) tier_registry_generation: Option, +} + +#[cfg(test)] pub(crate) fn current_cache_root_or_prepare( cache: &mut DataUsageCache, name: &str, @@ -140,11 +149,51 @@ pub(crate) fn current_cache_root_or_prepare( scan_plan_digest: DataUsageScanPlanDigest, require_source: bool, ) -> DataUsageCacheScanState { - match current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest) { + current_cache_root_or_prepare_with_generation( + cache, + name, + source, + next_cycle, + leader_epoch, + scan_plan_digest, + DataUsageCacheReuseOptions { + require_source, + tier_registry_generation: None, + }, + ) +} + +pub(crate) fn current_cache_root_or_prepare_with_generation( + cache: &mut DataUsageCache, + name: &str, + source: DataUsageCacheSource, + next_cycle: u64, + leader_epoch: u64, + scan_plan_digest: DataUsageScanPlanDigest, + options: DataUsageCacheReuseOptions, +) -> DataUsageCacheScanState { + if options.tier_registry_generation.is_some_and(|generation| { + cache.info.next_cycle <= next_cycle + && cache.info.leader_epoch <= leader_epoch + && cache.info.tier_registry_generation != Some(generation) + }) { + // Make prepare_for_scan take its reset path so an entry classified by + // an older registry cannot be reused under the new cycle generation. + cache.info.scan_plan_digest = None; + } + match current_cache_root_entry_with_generation( + cache, + name, + source, + next_cycle, + leader_epoch, + scan_plan_digest, + options.tier_registry_generation, + ) { Ok(Some(root)) => DataUsageCacheScanState::Current(Box::new(root)), current => DataUsageCacheScanState::Prepared { invalid_current: current.err(), - outcome: cache.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, require_source), + outcome: cache.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, options.require_source), }, } } @@ -159,7 +208,7 @@ pub(super) fn cache_snapshot_is_current( scan_plan_digest: DataUsageScanPlanDigest, ) -> bool { matches!( - current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest), + current_cache_root_entry_with_generation(cache, name, source, next_cycle, leader_epoch, scan_plan_digest, None), Ok(Some(_)) ) } @@ -168,6 +217,7 @@ pub(super) fn completed_data_usage_info( results: &[DataUsageCache], expected_sources: &HashSet, all_buckets: &[String], + tier_registry_names: &[String], bucket_plan_complete: bool, budget_elapsed: bool, cancelled: bool, @@ -183,6 +233,19 @@ pub(super) fn completed_data_usage_info( return None; } + // A generation is comparable across nodes because it is derived from the + // frozen registry names. Cycle and leader fencing remain separate cache + // metadata. Legacy peers omit the generation; an all-legacy result remains + // readable, but mixing legacy and new (or two new generations) would make + // the per-tier accounting ambiguous. + let registry_generation = results.first()?.info.tier_registry_generation; + if results.iter().any(|result| match registry_generation { + Some(generation) => result.info.tier_registry_generation != Some(generation), + None => result.info.tier_registry_generation.is_some(), + }) { + return None; + } + if results.iter().any(|result| result.root().is_none()) { return None; } @@ -200,9 +263,16 @@ pub(super) fn completed_data_usage_info( if !total.checked_merge(&merged) { return None; } + if !tier_accounting_proof_is_publishable(&merged, registry_generation, tier_registry_names) { + return None; + } bucket_entries.insert(bucket.clone(), merged); } + if !tier_accounting_proof_is_publishable(&total, registry_generation, tier_registry_names) { + return None; + } + let merged_last_update = results.iter().filter_map(|result| result.info.last_update).max()?; let buckets_usage = bucket_entries .iter() @@ -220,6 +290,7 @@ pub(super) fn completed_data_usage_info( delete_markers_total_count: u64::try_from(total.delete_markers).ok()?, objects_total_size: u64::try_from(total.size).ok()?, tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()), + unknown_tier_stats: total.unknown_tier_stats.filter(|stats| !stats.is_empty()), buckets_count: u64::try_from(all_buckets.len()).ok()?, bucket_sizes, buckets_usage, @@ -229,6 +300,109 @@ pub(super) fn completed_data_usage_info( Some((data_usage_info, merged_last_update)) } +fn tier_accounting_proof_is_publishable( + entry: &DataUsageEntry, + registry_generation: Option, + tier_registry_names: &[String], +) -> bool { + let has_scalar_usage = entry.size > 0 + || entry.objects > 0 + || entry.versions > 0 + || entry.delete_markers > 0 + || entry.failed_objects > 0 + || !entry.obj_sizes.is_empty() + || !entry.obj_versions.is_empty() + || entry.replication_stats.as_ref().is_some_and(|stats| !stats.is_empty()); + let has_tier_accounted_data = entry + .all_tier_stats + .as_ref() + .is_some_and(|stats| stats.tiers.values().any(|tier| !tier.is_empty())) + || entry.unknown_tier_stats.as_ref().is_some_and(|stats| { + stats.counter_overflowed + || stats.unknown_bytes > 0 + || stats.unknown_physical_bytes > 0 + || stats.unknown_objects > 0 + || stats.unknown_versions > 0 + }); + + let Some(proof) = entry.tier_accounting_proof else { + return !has_scalar_usage && !has_tier_accounted_data; + }; + if proof.overflowed + || entry + .unknown_tier_stats + .as_ref() + .is_some_and(|stats| stats.counter_overflowed) + || u64::try_from(entry.size).ok() != Some(proof.logical_total) + { + return false; + } + let unknown_logical = entry.unknown_tier_stats.as_ref().map_or(0, |stats| stats.unknown_bytes); + let unknown_physical = entry + .unknown_tier_stats + .as_ref() + .map_or(0, |stats| stats.unknown_physical_bytes); + if proof + .logical_known + .checked_add(unknown_logical) + .is_none_or(|total| total != proof.logical_total) + || proof + .physical_known + .checked_add(unknown_physical) + .is_none_or(|total| total != proof.physical_total) + { + return false; + } + if registry_generation.is_some() + && entry.all_tier_stats.as_ref().is_some_and(|stats| { + stats.tiers.keys().any(|tier| { + tier != crate::UNKNOWN_TIER + && tier != crate::storageclass::STANDARD + && tier != crate::storageclass::RRS + && !tier_registry_names.iter().any(|allowed| allowed == tier) + }) + }) + { + return false; + } + + if !has_tier_accounted_data { + return true; + } + + let Some(tiers) = entry.all_tier_stats.as_ref() else { + return false; + }; + let map_unknown_physical = tiers.tiers.get(crate::UNKNOWN_TIER).map_or(0, |stats| stats.total_size); + let companion_unknown_physical = entry + .unknown_tier_stats + .as_ref() + .map_or(0, |stats| stats.unknown_physical_bytes); + if map_unknown_physical != companion_unknown_physical { + return false; + } + + // A no-configuration scan intentionally stores only UNKNOWN_TIER after + // the first unknown object; STANDARD/RRS remain absent to preserve the + // historical empty-map shape. In that shape the scalar proof is the sole + // source of known physical bytes. Configured registries seed at least one + // non-UNKNOWN key, whose map total must match the proof. + let has_known_tier_map = tiers.tiers.keys().any(|tier| tier.as_str() != crate::UNKNOWN_TIER); + if !has_known_tier_map { + return true; + } + let Some(known_tier_physical_total) = tiers + .tiers + .iter() + .filter(|(tier, _)| tier.as_str() != crate::UNKNOWN_TIER) + .map(|(_, stats)| stats) + .try_fold(0_u64, |total, stats| total.checked_add(stats.total_size)) + else { + return false; + }; + proof.physical_known == known_tier_physical_total +} + /// Build a non-authoritative view from the set snapshots that completed this /// cycle plus compatible per-set last-known-good caches. The caller must /// persist this result only on the observational object; a missing set is @@ -238,6 +412,7 @@ pub(super) fn observational_data_usage_info( results: &[DataUsageCache], expected_sources: &HashSet, all_buckets: &[String], + tier_registry_names: &[String], expected_plan_digest: DataUsageScanPlanDigest, scanner_cycle: u64, leader_epoch: u64, @@ -316,6 +491,13 @@ pub(super) fn observational_data_usage_info( if usable.is_empty() { return None; } + let registry_generation = usable.first()?.0.info.tier_registry_generation; + if usable.iter().any(|(result, _)| match registry_generation { + Some(generation) => result.info.tier_registry_generation != Some(generation), + None => result.info.tier_registry_generation.is_some(), + }) { + return None; + } let mut total = DataUsageEntry::default(); let mut bucket_entries = HashMap::with_capacity(all_buckets.len()); @@ -337,6 +519,15 @@ pub(super) fn observational_data_usage_info( } } } + if bucket_entries + .values() + .any(|entry| !tier_accounting_proof_is_publishable(entry, registry_generation, tier_registry_names)) + { + return None; + } + if !tier_accounting_proof_is_publishable(&total, registry_generation, tier_registry_names) { + return None; + } let merged_last_update = merged_last_update?; let buckets_usage = bucket_entries .iter() @@ -352,6 +543,7 @@ pub(super) fn observational_data_usage_info( delete_markers_total_count: u64::try_from(total.delete_markers).ok()?, objects_total_size: u64::try_from(total.size).ok()?, tier_stats: total.all_tier_stats.filter(|tiers| !tiers.is_empty()), + unknown_tier_stats: total.unknown_tier_stats.filter(|stats| !stats.is_empty()), buckets_count: u64::try_from(buckets_usage.len()).ok()?, bucket_sizes: buckets_usage .iter() @@ -463,13 +655,14 @@ pub(super) async fn persist_and_publish_cache_snapshot( return None; } if matches!( - current_cache_root_entry( + current_cache_root_entry_with_generation( &persisted, DATA_USAGE_ROOT, source, cache_snapshot.info.next_cycle, cache_snapshot.info.leader_epoch, scan_plan_digest, + cache_snapshot.info.tier_registry_generation, ), Ok(Some(_)) ) { diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index 5b8246bc8..c99ffacd1 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -31,6 +31,7 @@ impl ScannerIOCache for SetDisks { all_buckets, digest: scan_plan_digest, leader_epoch, + tier_registry_generation, publication_epoch, dirty_usage_buckets, bucket_failures, @@ -70,6 +71,7 @@ impl ScannerIOCache for SetDisks { next_cycle: want_cycle, last_update: Some(now), leader_epoch, + tier_registry_generation: Some(tier_registry_generation), source: Some(source), snapshot_complete: true, scan_plan_digest: Some(scan_plan_digest), @@ -267,7 +269,14 @@ impl ScannerIOCache for SetDisks { record_disk_bucket_scans_active(0, &pool_label, &set_label); let _reset_disk_bucket_scan_gauges = DiskBucketScanGaugeReset::new(pool_label.clone(), set_label.clone()); - let old_lkg = old_cache.info.snapshot_complete.then(|| { + // Fence a stale set aggregate before copying entries into per-bucket work caches. + if old_cache.info.next_cycle <= want_cycle + && old_cache.info.leader_epoch <= leader_epoch + && old_cache.info.tier_registry_generation != Some(tier_registry_generation) + { + old_cache.info.scan_plan_digest = None; + } + let old_lkg = old_cache.info.snapshot_complete.then_some({ ( old_cache.info.next_cycle, old_cache.info.last_update, @@ -333,6 +342,7 @@ impl ScannerIOCache for SetDisks { name: DATA_USAGE_ROOT.to_string(), next_cycle: want_cycle, leader_epoch, + tier_registry_generation: Some(tier_registry_generation), source: Some(source), snapshot_complete: false, scan_plan_digest: Some(scan_plan_digest), @@ -394,8 +404,9 @@ impl ScannerIOCache for SetDisks { }; let mut cache = cache_mutex_clone.lock().await; - apply_bucket_result_to_cache(&mut cache, result, SystemTime::now()); - completed_bucket_count_clone.fetch_add(1, Ordering::Relaxed); + if apply_bucket_result_to_cache(&mut cache, result, SystemTime::now()) { + completed_bucket_count_clone.fetch_add(1, Ordering::Relaxed); + } } } } @@ -527,6 +538,7 @@ impl ScannerIOCache for SetDisks { session_id: remote_session_id, session_sequence: request_sequence, scan_plan_digest: bucket_scan_plan_digest, + tier_registry_generation, skip_healing: healing, scan_mode, }, @@ -742,14 +754,17 @@ impl ScannerIOCache for SetDisks { continue; } }; - let scan_state = current_cache_root_or_prepare( + let scan_state = current_cache_root_or_prepare_with_generation( &mut cache, &bucket.name, source, want_cycle, leader_epoch, bucket_scan_plan_digest, - require_cache_source, + DataUsageCacheReuseOptions { + require_source: require_cache_source, + tier_registry_generation: Some(tier_registry_generation), + }, ); let outcome = match scan_state { DataUsageCacheScanState::Current(root) => { @@ -1237,6 +1252,7 @@ impl ScannerIOCache for SetDisks { incomplete_scope.info.next_cycle = want_cycle; incomplete_scope.info.last_update = None; incomplete_scope.info.leader_epoch = leader_epoch; + incomplete_scope.info.tier_registry_generation = Some(tier_registry_generation); incomplete_scope.info.source = Some(source); incomplete_scope.info.snapshot_complete = false; incomplete_scope.info.scan_plan_digest = Some(scan_plan_digest); diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index 7a74e95f8..56869cd8c 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -48,6 +48,7 @@ impl ScannerIOCycle for ECStore { scan_mode: HealScanMode, ) -> Result { let child_token = ctx.child_token(); + let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch); // Check the local pool metadata before listing buckets. A failed or // canceled decommission remains suspended after its worker exits, so @@ -140,6 +141,8 @@ impl ScannerIOCycle for ECStore { scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before)); let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list)); let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle)); + let tier_registry = runtime_tier_registry_for_cycle(want_cycle, leader_epoch).await; + let tier_registry_generation = tier_registry.generation; if all_buckets.is_empty() { reset_set_scan_gauges(); @@ -172,6 +175,9 @@ impl ScannerIOCycle for ECStore { { return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch)); } + if status == ScannerCycleStatus::Complete { + complete_tier_registry_cycle(want_cycle, leader_epoch); + } let dirty_usage_clear = (status == ScannerCycleStatus::Complete).then(|| dirty_usage_snapshot.buckets.as_ref().clone()); let remote_dirty_usage_acknowledgements = if status == ScannerCycleStatus::Complete { @@ -266,6 +272,7 @@ impl ScannerIOCycle for ECStore { all_buckets: Arc::clone(&all_buckets), digest: scan_plan_digest, leader_epoch, + tier_registry_generation, publication_epoch, dirty_usage_buckets: dirty_usage_snapshot.buckets.clone(), bucket_failures: bucket_failures.clone(), @@ -399,6 +406,7 @@ impl ScannerIOCycle for ECStore { &results, &expected_sources, &all_bucket_names, + &tier_registry.names, bucket_plan_complete, budget_elapsed, ctx.is_cancelled(), @@ -410,6 +418,7 @@ impl ScannerIOCycle for ECStore { &results, &expected_sources, &all_bucket_names, + &tier_registry.names, scan_plan_digest, want_cycle, leader_epoch, @@ -441,6 +450,9 @@ impl ScannerIOCycle for ECStore { &failed_buckets, ); result?; + if cycle_status == ScannerCycleStatus::Complete { + complete_tier_registry_cycle(want_cycle, leader_epoch); + } let remote_dirty_usage_acknowledgements = if cycle_status == ScannerCycleStatus::Complete { crate::scanner::scanner_dirty_usage_acknowledgements(&activity_before) } else { diff --git a/crates/scanner/src/scanner_io/io_disk.rs b/crates/scanner/src/scanner_io/io_disk.rs index b60ac0238..cf209206e 100644 --- a/crates/scanner/src/scanner_io/io_disk.rs +++ b/crates/scanner/src/scanner_io/io_disk.rs @@ -13,29 +13,38 @@ // limitations under the License. /// ScannerIODisk implementation for Disk: get_size and the per-disk bucket scan. use super::*; +use crate::UNKNOWN_TIER; /// /// Seed [`SizeSummary::tier_stats`] from the cached tier-name list. /// -/// Preserves the original seeding semantics: with no tiers configured the map -/// stays completely empty (STANDARD/RRS are not seeded either); otherwise the -/// standard storage classes are seeded alongside every configured tier so -/// per-object accounting always finds its tier key. +/// Preserves the original no-tier shape: with no tiers configured the map +/// stays completely empty (STANDARD/RRS/UNKNOWN are not seeded either). +/// Otherwise the standard storage classes and one fixed unknown bucket are +/// seeded alongside every configured tier so per-object accounting never +/// inserts an untrusted metadata key. pub(super) fn tier_stats_template(tier_names: &[String]) -> HashMap { - let mut tier_stats = HashMap::with_capacity(tier_names.len() + 2); + let mut tier_stats = HashMap::with_capacity(tier_names.len() + 3); for tier_name in tier_names { - tier_stats.insert(tier_name.clone(), TierStats::default()); + if tier_name != UNKNOWN_TIER { + tier_stats.insert(tier_name.clone(), TierStats::default()); + } } if !tier_stats.is_empty() { tier_stats.insert(storageclass::STANDARD.to_string(), TierStats::default()); tier_stats.insert(storageclass::RRS.to_string(), TierStats::default()); + tier_stats.insert(UNKNOWN_TIER.to_string(), TierStats::default()); } tier_stats } #[async_trait::async_trait] impl ScannerIODisk for Disk { - async fn get_size(&self, mut item: ScannerItem) -> Result { + async fn get_size(&self, item: ScannerItem) -> Result { + self.get_size_with_tier_names(item, &runtime_tier_names().await).await + } + + async fn get_size_with_tier_names(&self, mut item: ScannerItem, tier_names: &[String]) -> Result { let done_object = Metrics::time(Metric::ScanObject); if !is_xl_meta_path(&item.path) { @@ -105,12 +114,13 @@ impl ScannerIODisk for Disk { .map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), object_path.as_str(), versioned)) .collect::>(); - let mut size_summary = SizeSummary::default(); - - // Tier names come from the process-wide TTL cache; seeding from them - // replaces the per-object clone of every full TierConfig. - let tier_names = runtime_tier_names().await; - size_summary.tier_stats = tier_stats_template(&tier_names); + // The caller supplies one registry snapshot for the whole folder scan; + // seeding from it prevents a TTL refresh from mixing generations in a + // single result. + let mut size_summary = SizeSummary { + tier_stats: tier_stats_template(tier_names), + ..Default::default() + }; let lock_config = object_lock_config_for_scanner_item(&item).await; @@ -120,12 +130,14 @@ impl ScannerIODisk for Disk { // `object_infos`. global_metrics().record_scanner_versions_scanned(object_infos.len() as u64); - item.apply_actions(object_infos, lock_config, versioning_config, &mut size_summary) + item.apply_actions(object_infos, lock_config, versioning_config, tier_names, &mut size_summary) .await; if !free_version_infos.is_empty() { for oi in free_version_infos { - enqueue_runtime_free_version(oi).await; + if ScannerItem::tier_is_known(&oi, tier_names) { + enqueue_runtime_free_version(oi).await; + } } } diff --git a/crates/scanner/src/scanner_io/publish_gate_tests.rs b/crates/scanner/src/scanner_io/publish_gate_tests.rs index 8d44ff945..2abdf4b40 100644 --- a/crates/scanner/src/scanner_io/publish_gate_tests.rs +++ b/crates/scanner/src/scanner_io/publish_gate_tests.rs @@ -13,7 +13,8 @@ // limitations under the License. use super::*; -use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage}; +use crate::data_usage_define::{UNKNOWN_TIER, UnknownTierStats, hash_path}; +use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage, TierAccountingProof}; const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]); @@ -89,6 +90,11 @@ fn completed_root_cache(bucket: &str, objects: usize, update_secs: u64, source: DataUsageEntry { objects, size: objects.saturating_mul(10), + tier_accounting_proof: Some(TierAccountingProof { + logical_total: u64::try_from(objects.saturating_mul(10)).unwrap_or(u64::MAX), + logical_known: u64::try_from(objects.saturating_mul(10)).unwrap_or(u64::MAX), + ..Default::default() + }), ..Default::default() }, ); @@ -102,7 +108,7 @@ fn completed_data_usage_info_for_test( cancelled: bool, ) -> Option<(DataUsageInfo, SystemTime)> { let expected_sources = results.iter().filter_map(|result| result.info.source).collect::>(); - completed_data_usage_info(results, &expected_sources, all_buckets, true, budget_elapsed, cancelled) + completed_data_usage_info(results, &expected_sources, all_buckets, &[], true, budget_elapsed, cancelled) } fn lkg_root_cache(bucket: &str, objects: usize, source: DataUsageCacheSource) -> DataUsageCache { @@ -130,9 +136,10 @@ fn partial_usage_is_observational_not_authoritative_for_quota() { let expected = HashSet::from([current_source, stalled_source]); assert!( - completed_data_usage_info(&[current.clone(), stalled.clone()], &expected, &all_buckets, true, false, false).is_none() + completed_data_usage_info(&[current.clone(), stalled.clone()], &expected, &all_buckets, &[], true, false, false) + .is_none() ); - let (observed, _) = observational_data_usage_info(&[current, stalled], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3) + let (observed, _) = observational_data_usage_info(&[current, stalled], &expected, &all_buckets, &[], TEST_PLAN_DIGEST, 8, 3) .expect("a completed set should produce an observational view"); assert!(observed.usage_snapshot_partial); assert!(!observed.usage_snapshot_complete); @@ -157,7 +164,7 @@ fn stale_quota_uses_complete_baseline_plus_positive_deltas() { current.info.next_cycle = 8; current.info.leader_epoch = 3; let expected = HashSet::from([source]); - let (observed, _) = observational_data_usage_info(&[current], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3) + let (observed, _) = observational_data_usage_info(&[current], &expected, &all_buckets, &[], TEST_PLAN_DIGEST, 8, 3) .expect("complete set data is a valid observational baseline"); assert_eq!(observed.objects_total_size, 30); assert_eq!(observed.usage_snapshot_set_states[0].complete, true); @@ -170,7 +177,7 @@ fn negative_delta_waits_for_set_reconciliation() { let mut stalled = lkg_root_cache("bucket", 4, source); stalled.info.lkg_scan_plan_digest = Some(DataUsageScanPlanDigest([9; 32])); let expected = HashSet::from([source]); - assert!(observational_data_usage_info(&[stalled], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3).is_none()); + assert!(observational_data_usage_info(&[stalled], &expected, &all_buckets, &[], TEST_PLAN_DIGEST, 8, 3).is_none()); } #[test] @@ -220,7 +227,7 @@ fn old_set_completion_cannot_overwrite_new_aggregate() { old.info.next_cycle = 7; old.info.leader_epoch = 2; let expected = HashSet::from([source]); - assert!(observational_data_usage_info(&[old], &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3).is_none()); + assert!(observational_data_usage_info(&[old], &expected, &all_buckets, &[], TEST_PLAN_DIGEST, 8, 3).is_none()); } #[test] @@ -231,7 +238,7 @@ fn usage_aggregate_survives_restart_and_leader_failover() { lkg.info.lkg_leader_epoch = Some(4); lkg.info.lkg_next_cycle = Some(9); let expected = HashSet::from([source]); - let (observed, _) = observational_data_usage_info(&[lkg], &expected, &all_buckets, TEST_PLAN_DIGEST, 10, 5) + let (observed, _) = observational_data_usage_info(&[lkg], &expected, &all_buckets, &[], TEST_PLAN_DIGEST, 10, 5) .expect("compatible LKG should survive a leader change"); assert_eq!(observed.usage_snapshot_set_states[0].scanner_epoch, Some(4)); assert_eq!(observed.objects_total_size, 50); @@ -250,11 +257,11 @@ fn usage_aggregate_cost_is_linear_in_set_count() { cache.info.leader_epoch = 3; results.push(cache); } - let (observed, _) = observational_data_usage_info(&results, &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3) + let (observed, _) = observational_data_usage_info(&results, &expected, &all_buckets, &[], TEST_PLAN_DIGEST, 8, 3) .expect("all set snapshots should aggregate"); assert_eq!(observed.objects_total_count, 32); let reversed = results.iter().rev().cloned().collect::>(); - let (reversed_observed, _) = observational_data_usage_info(&reversed, &expected, &all_buckets, TEST_PLAN_DIGEST, 8, 3) + let (reversed_observed, _) = observational_data_usage_info(&reversed, &expected, &all_buckets, &[], TEST_PLAN_DIGEST, 8, 3) .expect("reordered set snapshots should aggregate"); assert_eq!(observed.usage_snapshot_set_states, reversed_observed.usage_snapshot_set_states); } @@ -276,11 +283,21 @@ fn completed_data_usage_info_publishes_tier_stats_across_sets() { let mut first_set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); let mut tiered = DataUsageEntry::default(); tiered.add_tier_sizes(&warm(100, 2, 1)); + tiered.tier_accounting_proof = Some(TierAccountingProof { + physical_total: 100, + physical_known: 100, + ..Default::default() + }); first_set.replace("bucket-b", DATA_USAGE_ROOT, tiered); let mut second_set = completed_root_cache("bucket-b", 2, 20, DataUsageCacheSource::new(1, 0)); let mut tiered = DataUsageEntry::default(); tiered.add_tier_sizes(&warm(50, 1, 1)); + tiered.tier_accounting_proof = Some(TierAccountingProof { + physical_total: 50, + physical_known: 50, + ..Default::default() + }); second_set.replace("bucket-a", DATA_USAGE_ROOT, tiered); let (data_usage_info, _) = completed_data_usage_info_for_test(&[first_set, second_set], &all_buckets, false, false) @@ -299,6 +316,269 @@ fn completed_data_usage_info_publishes_tier_stats_across_sets() { ); } +#[test] +fn completed_data_usage_info_rejects_logical_proof_mismatch() { + let all_buckets = vec!["bucket-a".to_string()]; + let mut set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + let entry = set.cache.get_mut(&hash_path("bucket-a").key()).expect("bucket entry"); + entry.add_tier_sizes(&HashMap::from([( + "WARM".to_string(), + TierStats { + total_size: 10, + num_versions: 1, + num_objects: 1, + }, + )])); + entry.tier_accounting_proof = Some(TierAccountingProof { + logical_total: 10, + logical_known: 9, + physical_total: 10, + physical_known: 10, + ..Default::default() + }); + + assert!(completed_data_usage_info_for_test(&[set], &all_buckets, false, false).is_none()); +} + +#[test] +fn completed_data_usage_info_rejects_logical_total_size_mismatch() { + let all_buckets = vec!["bucket-a".to_string()]; + let mut set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + let entry = set.cache.get_mut(&hash_path("bucket-a").key()).expect("bucket entry"); + entry.size = 11; + entry.add_tier_sizes(&HashMap::from([( + "WARM".to_string(), + TierStats { + total_size: 10, + num_versions: 1, + num_objects: 1, + }, + )])); + entry.tier_accounting_proof = Some(TierAccountingProof { + logical_total: 10, + logical_known: 10, + physical_total: 10, + physical_known: 10, + ..Default::default() + }); + + assert!(completed_data_usage_info_for_test(&[set], &all_buckets, false, false).is_none()); +} + +#[test] +fn completed_data_usage_info_rejects_physical_proof_mismatch() { + let all_buckets = vec!["bucket-a".to_string()]; + let mut set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + let entry = set.cache.get_mut(&hash_path("bucket-a").key()).expect("bucket entry"); + entry.add_tier_sizes(&HashMap::from([( + "WARM".to_string(), + TierStats { + total_size: 10, + num_versions: 1, + num_objects: 1, + }, + )])); + entry.tier_accounting_proof = Some(TierAccountingProof { + logical_total: 10, + logical_known: 10, + physical_total: 9, + physical_known: 9, + ..Default::default() + }); + + assert!(completed_data_usage_info_for_test(&[set], &all_buckets, false, false).is_none()); +} + +#[test] +fn completed_data_usage_info_rejects_unknown_physical_double_accounting() { + let all_buckets = vec!["bucket-a".to_string()]; + let mut set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + let entry = set.cache.get_mut(&hash_path("bucket-a").key()).expect("bucket entry"); + entry.add_tier_sizes(&HashMap::from([( + UNKNOWN_TIER.to_string(), + TierStats { + total_size: 10, + num_versions: 1, + num_objects: 1, + }, + )])); + entry.add_unknown_tier_stats(&UnknownTierStats { + unknown_physical_bytes: 9, + ..Default::default() + }); + entry.tier_accounting_proof = Some(TierAccountingProof { + logical_total: 10, + logical_known: 10, + physical_total: 10, + physical_known: 10, + ..Default::default() + }); + + assert!(completed_data_usage_info_for_test(&[set], &all_buckets, false, false).is_none()); +} + +#[test] +fn completed_data_usage_info_rejects_unknown_counter_overflow() { + let all_buckets = vec!["bucket-a".to_string()]; + let mut set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + let entry = set.cache.get_mut(&hash_path("bucket-a").key()).expect("bucket entry"); + entry.add_unknown_tier_stats(&UnknownTierStats { + counter_overflowed: true, + ..Default::default() + }); + + assert!(completed_data_usage_info_for_test(&[set], &all_buckets, false, false).is_none()); +} + +#[test] +fn completed_data_usage_info_accepts_no_tier_standard_empty_map_with_proof() { + let all_buckets = vec!["bucket-a".to_string()]; + let set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + + let (info, _) = completed_data_usage_info_for_test(&[set], &all_buckets, false, false) + .expect("no-tier STANDARD/RRS usage does not require a tier map"); + assert!(info.tier_stats.is_none()); +} + +#[test] +fn completed_data_usage_info_accepts_no_tier_unknown_and_standard_shape() { + let all_buckets = vec!["bucket-a".to_string()]; + let mut set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + let entry = set.cache.get_mut(&hash_path("bucket-a").key()).expect("bucket entry"); + entry.size = 13; + entry.add_tier_sizes(&HashMap::from([( + UNKNOWN_TIER.to_string(), + TierStats { + total_size: 3, + num_versions: 1, + num_objects: 1, + }, + )])); + entry.add_unknown_tier_stats(&UnknownTierStats { + unknown_bytes: 9, + unknown_physical_bytes: 3, + unknown_objects: 1, + unknown_versions: 1, + ..Default::default() + }); + entry.tier_accounting_proof = Some(TierAccountingProof { + logical_total: 13, + logical_known: 4, + physical_total: 7, + physical_known: 4, + ..Default::default() + }); + + let (info, _) = completed_data_usage_info_for_test(&[set], &all_buckets, false, false) + .expect("no-tier STANDARD plus UNKNOWN should remain publishable"); + assert_eq!( + info.tier_stats.expect("unknown bucket should be retained").tiers[UNKNOWN_TIER].total_size, + 3 + ); +} + +#[test] +fn completed_data_usage_info_accepts_unknown_only_with_current_registry_generation() { + let all_buckets = vec!["bucket-a".to_string()]; + let mut set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + set.info.tier_registry_generation = Some(7); + let entry = set.cache.get_mut(&hash_path("bucket-a").key()).expect("bucket entry"); + entry.size = 13; + entry.add_tier_sizes(&HashMap::from([( + UNKNOWN_TIER.to_string(), + TierStats { + total_size: 3, + num_versions: 1, + num_objects: 1, + }, + )])); + entry.add_unknown_tier_stats(&UnknownTierStats { + unknown_bytes: 9, + unknown_physical_bytes: 3, + unknown_objects: 1, + unknown_versions: 1, + ..Default::default() + }); + entry.tier_accounting_proof = Some(TierAccountingProof { + logical_total: 13, + logical_known: 4, + physical_total: 7, + physical_known: 4, + ..Default::default() + }); + + let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0)]); + assert!( + completed_data_usage_info(&[set], &expected_sources, &all_buckets, &["WARM".to_string()], true, false, false,).is_some() + ); +} + +#[test] +fn completed_data_usage_info_rejects_non_registry_tier_in_current_generation() { + let all_buckets = vec!["bucket-a".to_string()]; + let mut set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + set.info.tier_registry_generation = Some(7); + let entry = set.cache.get_mut(&hash_path("bucket-a").key()).expect("bucket entry"); + entry.add_tier_sizes(&HashMap::from([ + ( + "WARM".to_string(), + TierStats { + total_size: 4, + num_versions: 1, + num_objects: 1, + }, + ), + ( + "RETIRED".to_string(), + TierStats { + total_size: 6, + num_versions: 1, + num_objects: 1, + }, + ), + ])); + entry.tier_accounting_proof = Some(TierAccountingProof { + logical_total: 10, + logical_known: 10, + physical_total: 10, + physical_known: 10, + ..Default::default() + }); + + let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0)]); + assert!( + completed_data_usage_info(&[set], &expected_sources, &all_buckets, &["WARM".to_string()], true, false, false,).is_none() + ); +} + +#[test] +fn completed_data_usage_info_rejects_legacy_proof_missing_when_tier_accounted() { + let all_buckets = vec!["bucket-a".to_string()]; + let mut set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + let entry = set.cache.get_mut(&hash_path("bucket-a").key()).expect("bucket entry"); + entry.add_tier_sizes(&HashMap::from([( + "WARM".to_string(), + TierStats { + total_size: 10, + num_versions: 1, + num_objects: 1, + }, + )])); + entry.tier_accounting_proof = None; + + assert!(completed_data_usage_info_for_test(&[set], &all_buckets, false, false).is_none()); +} + +#[test] +fn completed_data_usage_info_rejects_legacy_proof_missing_for_scalar_usage() { + let all_buckets = vec!["bucket-a".to_string()]; + let mut set = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + let entry = set.cache.get_mut(&hash_path("bucket-a").key()).expect("bucket entry"); + entry.tier_accounting_proof = None; + + assert!(completed_data_usage_info_for_test(&[set], &all_buckets, false, false).is_none()); +} + #[test] fn completed_data_usage_info_omits_tier_stats_without_tiered_objects() { let all_buckets = vec!["bucket-a".to_string()]; @@ -310,6 +590,51 @@ fn completed_data_usage_info_omits_tier_stats_without_tiered_objects() { assert!(data_usage_info.tier_stats.is_none()); } +#[test] +fn completed_data_usage_info_rejects_legacy_and_new_tier_generations_mixed() { + let all_buckets = vec!["bucket-a".to_string()]; + let legacy = completed_root_cache("bucket-a", 1, 10, DataUsageCacheSource::new(0, 0)); + let mut current = completed_root_cache("bucket-a", 1, 20, DataUsageCacheSource::new(1, 0)); + current.info.tier_registry_generation = Some(42); + + assert!( + completed_data_usage_info_for_test(&[legacy, current], &all_buckets, false, false).is_none(), + "legacy and generation-tagged sets must not publish a mixed snapshot" + ); +} + +#[test] +fn current_cache_root_with_new_tier_generation_resets_old_cache() { + let source = DataUsageCacheSource::new(0, 0); + let mut cache = completed_root_cache("bucket-a", 1, 10, source); + cache.info.tier_registry_generation = Some(1); + + let state = current_cache_root_or_prepare_with_generation( + &mut cache, + DATA_USAGE_ROOT, + source, + 0, + 0, + TEST_PLAN_DIGEST, + DataUsageCacheReuseOptions { + require_source: false, + tier_registry_generation: Some(2), + }, + ); + + assert!(matches!( + state, + DataUsageCacheScanState::Prepared { + outcome: DataUsageCachePrepareOutcome::Reset, + .. + } + )); + assert!(cache.cache.is_empty(), "old-generation entries must not be reused"); + assert_eq!(cache.info.tier_registry_generation, None); + assert_eq!(cache.info.scan_plan_digest, Some(TEST_PLAN_DIGEST)); + assert!(!cache.info.snapshot_complete); +} + #[test] fn completed_data_usage_info_requires_every_set_before_publish() { let all_buckets = vec!["bucket-a".to_string(), "bucket-b".to_string(), "bucket-empty".to_string()]; @@ -434,6 +759,13 @@ fn completed_data_usage_info_flattens_nested_bucket_entries() { replica_size: 2048, replica_count: 2, }), + tier_accounting_proof: Some(TierAccountingProof { + logical_total: 2048, + logical_known: 2048, + physical_total: 2048, + physical_known: 2048, + ..Default::default() + }), ..Default::default() }; nested.obj_sizes.add(2048); @@ -501,7 +833,8 @@ fn completed_data_usage_info_requires_exact_topology_sources() { let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0), DataUsageCacheSource::new(1, 0)]); assert!( - completed_data_usage_info(&[first_set, unexpected_set], &expected_sources, &all_buckets, true, false, false).is_none() + completed_data_usage_info(&[first_set, unexpected_set], &expected_sources, &all_buckets, &[], true, false, false) + .is_none() ); } @@ -511,7 +844,7 @@ fn completed_data_usage_info_rejects_incomplete_bucket_plan() { let set = completed_root_cache("bucket", 2, 10, DataUsageCacheSource::new(0, 0)); let expected_sources = HashSet::from([DataUsageCacheSource::new(0, 0)]); - assert!(completed_data_usage_info(&[set], &expected_sources, &all_buckets, false, false, false).is_none()); + assert!(completed_data_usage_info(&[set], &expected_sources, &all_buckets, &[], false, false, false).is_none()); } #[test] diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index 7931a44f9..6e6041c24 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -23,7 +23,7 @@ use crate::storage_api::owner::{ use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _}; use crate::{ DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions, - ScannerPutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, + ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path, }; use rustfs_filemeta::FileInfo; @@ -1030,8 +1030,8 @@ fn is_xl_meta_path_accepts_forward_separator() { fn tier_stats_template_seeds_tiers_and_standard_classes() { let template = tier_stats_template(&["WARM".to_string(), "COLD".to_string()]); - assert_eq!(template.len(), 4); - for tier in ["WARM", "COLD", storageclass::STANDARD, storageclass::RRS] { + assert_eq!(template.len(), 5); + for tier in ["WARM", "COLD", storageclass::STANDARD, storageclass::RRS, UNKNOWN_TIER] { assert_eq!(template.get(tier), Some(&TierStats::default()), "missing seed for tier {tier}"); } } @@ -1372,7 +1372,7 @@ fn apply_bucket_result_to_cache_updates_bucket_entry() { ); let update_time = SystemTime::now(); - apply_bucket_result_to_cache( + assert!(apply_bucket_result_to_cache( &mut cache, DataUsageEntryInfo { name: "bucket".to_string(), @@ -1382,12 +1382,51 @@ fn apply_bucket_result_to_cache_updates_bucket_entry() { objects: 2, ..Default::default() }, + tier_registry_generation: None, }, update_time, - ); + )); assert_eq!(cache.info.last_update, Some(update_time)); let entry = cache.find("bucket").expect("bucket entry should remain present"); assert_eq!(entry.size, 10); assert_eq!(entry.objects, 2); } + +#[test] +fn apply_bucket_result_to_cache_rejects_a_different_tier_generation() { + let mut cache = DataUsageCache { + info: DataUsageCacheInfo { + name: DATA_USAGE_ROOT.to_string(), + tier_registry_generation: Some(7), + ..Default::default() + }, + ..Default::default() + }; + cache.replace( + "bucket", + DATA_USAGE_ROOT, + DataUsageEntry { + size: 3, + ..Default::default() + }, + ); + + let applied = apply_bucket_result_to_cache( + &mut cache, + DataUsageEntryInfo { + name: "bucket".to_string(), + parent: DATA_USAGE_ROOT.to_string(), + entry: DataUsageEntry { + size: 11, + ..Default::default() + }, + tier_registry_generation: Some(8), + }, + SystemTime::now(), + ); + + assert!(!applied); + assert_eq!(cache.find("bucket").map(|entry| entry.size), Some(3)); + assert!(cache.info.last_update.is_none()); +} diff --git a/crates/storage-api/src/lib.rs b/crates/storage-api/src/lib.rs index 1114349e1..7768dad62 100644 --- a/crates/storage-api/src/lib.rs +++ b/crates/storage-api/src/lib.rs @@ -46,6 +46,7 @@ pub const NS_SCANNER_SERVER_EPOCH_QUERY: &str = "ns_scanner_server_epoch"; pub const NS_SCANNER_SESSION_ID_QUERY: &str = "ns_scanner_session_id"; pub const NS_SCANNER_SESSION_SEQUENCE_QUERY: &str = "ns_scanner_session_sequence"; pub const NS_SCANNER_PROTOCOL_VERSION_QUERY: &str = "ns_scanner_protocol"; +pub const NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY: &str = "ns_scanner_tier_registry_generation"; pub const NS_SCANNER_PROTOCOL_VERSION: u16 = 3; pub const SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION: u32 = 0; pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 5; @@ -57,6 +58,8 @@ pub struct NsScannerCapabilityResponse { pub version: u16, pub server_epoch: uuid::Uuid, pub proof: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub supports_tier_registry_generation: Option, } pub mod admin; diff --git a/rustfs/src/storage/rpc/http_service.rs b/rustfs/src/storage/rpc/http_service.rs index c2983cc43..052dbaff5 100644 --- a/rustfs/src/storage/rpc/http_service.rs +++ b/rustfs/src/storage/rpc/http_service.rs @@ -19,13 +19,15 @@ use crate::storage::storage_api::rpc_consumer::http_service::{ DEFAULT_READ_BUFFER_SIZE, DeleteOptions, DiskStore, NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_VERSION, PutFileCapabilityResponse, StorageDiskRpcExt as _, WALK_DIR_STREAM_COMPLETION_V1, WalkDirOptions, check_and_record_signed_rpc_nonce, find_local_disk_by_ref, - sign_ns_scanner_capability, sign_put_file_capability, verify_put_file_auth_trailer, verify_rpc_signature, + sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, verify_put_file_auth_trailer, + verify_rpc_signature, }; #[cfg(test)] use crate::storage::storage_api::rpc_consumer::http_service::{ NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, - PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, WALK_DIR_BODY_SHA256_QUERY, + NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, + WALK_DIR_BODY_SHA256_QUERY, }; use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; use crate::storage::storage_api::tonic_rpc_auth_failure_reason; @@ -290,6 +292,8 @@ struct NsScannerQuery { struct NsScannerCapabilityQuery { ns_scanner_protocol: Option, ns_scanner_challenge: Option, + #[serde(rename = "ns_scanner_tier_registry_generation")] + ns_scanner_tier_registry_generation: Option, } fn verify_ns_scanner_body_digest(query: &NsScannerQuery, body: &[u8]) -> bool { @@ -410,7 +414,9 @@ async fn handle_internode_rpc(req: Request) -> Response { (Method::GET, WALK_DIR_PATH) | (Method::HEAD, WALK_DIR_PATH) => handle_walk_dir(req).await, (Method::GET, NS_SCANNER_PATH) => match parse_query::(&req) { Ok(query) if query.ns_scanner_protocol == Some(NS_SCANNER_PROTOCOL_VERSION) => match query.ns_scanner_challenge { - Some(challenge) if !challenge.is_nil() => ns_scanner_capability_response(challenge), + Some(challenge) if !challenge.is_nil() => { + ns_scanner_capability_response(challenge, query.ns_scanner_tier_registry_generation == Some(true)) + } Some(_) | None => response_with_status(StatusCode::BAD_REQUEST, "namespace scanner challenge is invalid"), }, Ok(_) => response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner protocol is unsupported"), @@ -466,31 +472,34 @@ fn record_internode_rpc_error(operation: Option<&'static str>) { } } -fn ns_scanner_capability_response(challenge: uuid::Uuid) -> Response { +fn ns_scanner_capability_response(challenge: uuid::Uuid, include_tier_registry_generation: bool) -> Response { let server_epoch = *NS_SCANNER_SERVER_EPOCH; - let proof = match sign_ns_scanner_capability(challenge, server_epoch) { - Ok(proof) => proof, - Err(err) => { - error!( - event = EVENT_RPC_REQUEST_FAILED, - component = LOG_COMPONENT_INTERNODE_RPC, - subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, - operation = INTERNODE_OPERATION_NS_SCANNER, - result = "failed", - status_code = StatusCode::UPGRADE_REQUIRED.as_u16(), - rpc_path = NS_SCANNER_PATH, - method = %Method::GET, - reason = "capability_authentication_unavailable", - error = %err, - "internode rpc request failed" - ); - return response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner RPC authentication is unavailable"); - } - }; + let proof = + match sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, include_tier_registry_generation) + { + Ok(proof) => proof, + Err(err) => { + error!( + event = EVENT_RPC_REQUEST_FAILED, + component = LOG_COMPONENT_INTERNODE_RPC, + subsystem = LOG_SUBSYSTEM_NAMESPACE_SCANNER, + operation = INTERNODE_OPERATION_NS_SCANNER, + result = "failed", + status_code = StatusCode::UPGRADE_REQUIRED.as_u16(), + rpc_path = NS_SCANNER_PATH, + method = %Method::GET, + reason = "capability_authentication_unavailable", + error = %err, + "internode rpc request failed" + ); + return response_with_status(StatusCode::UPGRADE_REQUIRED, "namespace scanner RPC authentication is unavailable"); + } + }; let body = match rmp_serde::to_vec_named(&NsScannerCapabilityResponse { version: NS_SCANNER_PROTOCOL_VERSION, server_epoch, proof, + supports_tier_registry_generation: include_tier_registry_generation.then_some(true), }) { Ok(body) => body, Err(err) => { @@ -1676,12 +1685,13 @@ mod tests { LOG_SUBSYSTEM_NAMESPACE_SCANNER, LOG_SUBSYSTEM_ROUTING, NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PATH, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, - NS_SCANNER_SESSION_SEQUENCE_QUERY, NsScannerQuery, PUT_FILE_AUTH_STREAM_PATH, PUT_FILE_CAPABILITY_PATH, - PUT_FILE_STREAM_PATH, PutFileQuery, READ_FILE_STREAM_PATH, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_PATH, WalkDirQuery, - append_walk_dir_completion, internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, - ns_scanner_response_body, ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_auth_nonce, - put_file_capability_response, put_file_server_epoch_matches, put_file_stage_error_message, put_file_target_lock, - read_file_body_stream, read_file_stream_buffer_size, remote_scanner_claim_rejection, response_with_disk_error, + NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse, + NsScannerQuery, PUT_FILE_AUTH_STREAM_PATH, PUT_FILE_CAPABILITY_PATH, PUT_FILE_STREAM_PATH, PutFileQuery, + READ_FILE_STREAM_PATH, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_PATH, WalkDirQuery, append_walk_dir_completion, + internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, ns_scanner_response_body, + ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_auth_nonce, put_file_capability_response, + put_file_server_epoch_matches, put_file_stage_error_message, put_file_target_lock, read_file_body_stream, + read_file_stream_buffer_size, remote_scanner_claim_rejection, response_with_disk_error, supports_walk_dir_stream_completion, validate_walk_dir_completion_request, verify_internode_rpc_signature, verify_ns_scanner_body_digest, verify_walk_dir_body_digest, walk_dir_response_body, write_authenticated_put_file, write_body_chunks_to_writer, write_put_file_body_chunks_to_writer, @@ -2114,6 +2124,51 @@ mod tests { ); assert!(serde_urlencoded::from_str::(&query).is_err()); assert!(serde_urlencoded::from_str::("ns_scanner_protocol=1&unexpected=true").is_err()); + let marked = + format!("ns_scanner_protocol=3&ns_scanner_challenge={request_id}&{NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY}=true"); + assert_eq!( + serde_urlencoded::from_str::(&marked) + .expect("generation marker should be accepted") + .ns_scanner_tier_registry_generation, + Some(true) + ); + } + + #[test] + fn namespace_scanner_capability_response_support_is_optional_for_old_peers() { + #[derive(serde::Deserialize)] + #[serde(deny_unknown_fields)] + struct LegacyCapabilityResponse { + version: u16, + server_epoch: uuid::Uuid, + proof: Vec, + } + + let old = rmp_serde::to_vec_named(&NsScannerCapabilityResponse { + version: super::NS_SCANNER_PROTOCOL_VERSION, + server_epoch: uuid::Uuid::new_v4(), + proof: vec![1, 2, 3], + supports_tier_registry_generation: None, + }) + .expect("old response shape should encode"); + let decoded: NsScannerCapabilityResponse = rmp_serde::from_slice(&old).expect("old response should decode"); + assert_eq!(decoded.supports_tier_registry_generation, None); + let legacy_decoded: LegacyCapabilityResponse = + rmp_serde::from_slice(&old).expect("legacy reader should decode old shape"); + assert_eq!(legacy_decoded.version, super::NS_SCANNER_PROTOCOL_VERSION); + assert!(!legacy_decoded.server_epoch.is_nil()); + assert_eq!(legacy_decoded.proof, vec![1, 2, 3]); + + let current = rmp_serde::to_vec_named(&NsScannerCapabilityResponse { + version: super::NS_SCANNER_PROTOCOL_VERSION, + server_epoch: uuid::Uuid::new_v4(), + proof: vec![1, 2, 3], + supports_tier_registry_generation: Some(true), + }) + .expect("new response shape should encode"); + let decoded: NsScannerCapabilityResponse = rmp_serde::from_slice(¤t).expect("new response should decode"); + assert_eq!(decoded.supports_tier_registry_generation, Some(true)); + assert!(rmp_serde::from_slice::(¤t).is_err()); } #[test] diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 512e255ee..e245e1c08 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -224,8 +224,8 @@ pub(crate) mod rpc_consumer { pub(crate) use super::super::storage_contracts::{ NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, - NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, - PUT_FILE_CAPABILITY_QUERY, WALK_DIR_BODY_SHA256_QUERY, + NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, + PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, WALK_DIR_BODY_SHA256_QUERY, }; pub(crate) use super::super::storage_contracts::{ NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_V1, @@ -233,8 +233,8 @@ pub(crate) mod rpc_consumer { }; pub(crate) use super::super::{ DeleteOptions, DiskStore, StorageDiskRpcExt, WalkDirOptions, check_and_record_signed_rpc_nonce, - find_local_disk_by_ref, sign_ns_scanner_capability, sign_put_file_capability, verify_put_file_auth_trailer, - verify_rpc_signature, + find_local_disk_by_ref, sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, + verify_put_file_auth_trailer, verify_rpc_signature, }; } @@ -520,9 +520,10 @@ pub(crate) mod ecstore_rpc { pub(crate) use rustfs_ecstore::api::rpc::{ KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, - check_and_record_signed_rpc_nonce, normalize_tonic_rpc_audience, sign_ns_scanner_capability, sign_put_file_capability, - sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, - tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_rpc_signature, verify_tonic_canonical_body_digest, + check_and_record_signed_rpc_nonce, normalize_tonic_rpc_audience, + sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof, + tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, + verify_put_file_auth_trailer, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_signature_with_bootstrap, }; #[cfg(test)] @@ -1731,8 +1732,16 @@ pub(crate) fn verify_put_file_auth_trailer( ecstore_rpc::verify_put_file_auth_trailer(url, method, nonce, trailer) } -pub(crate) fn sign_ns_scanner_capability(challenge: uuid::Uuid, server_epoch: uuid::Uuid) -> std::io::Result> { - ecstore_rpc::sign_ns_scanner_capability(challenge, server_epoch) +pub(crate) fn sign_ns_scanner_capability_with_tier_registry_generation( + challenge: uuid::Uuid, + server_epoch: uuid::Uuid, + supports_tier_registry_generation: bool, +) -> std::io::Result> { + ecstore_rpc::sign_ns_scanner_capability_with_tier_registry_generation( + challenge, + server_epoch, + supports_tier_registry_generation, + ) } pub(crate) fn sign_put_file_capability( From a8e4b67d9972efc2cf11e145b85b8c5deb2f6185 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 23 Aug 2026 19:31:01 +0800 Subject: [PATCH 21/41] feat(metrics): expose deferred usage freshness (#6449) --- crates/common/src/metrics.rs | 104 ++++++++++++++++ .../ecstore/src/services/metrics_realtime.rs | 24 ++++ crates/madmin/src/metrics.rs | 75 ++++++++++++ crates/scanner/src/scanner.rs | 5 + crates/scanner/src/scanner/tests.rs | 1 + crates/scanner/src/scanner/usage_store.rs | 16 ++- rustfs/src/admin/handlers/cluster_snapshot.rs | 115 ++++++++++++++---- rustfs/src/cluster_snapshot.rs | 16 +++ 8 files changed, 331 insertions(+), 25 deletions(-) diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 813e39ac3..32cb1b18a 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -918,7 +918,15 @@ pub struct Metrics { scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64, scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64, scanner_usage_last_save_unix_secs: AtomicU64, + scanner_usage_last_durable_success_unix_secs: AtomicU64, + scanner_usage_last_publication_unix_secs: AtomicU64, + scanner_usage_last_publication_state: Mutex, + scanner_usage_last_publication_reason: Mutex, scanner_usage_last_save_result: AtomicU8, + scanner_usage_deferred_pending: AtomicBool, + scanner_usage_deferred_total: AtomicU64, + scanner_usage_last_deferred_unix_secs: AtomicU64, + scanner_usage_last_deferred_reason: Mutex, scanner_source_work: Vec, current_scan_cycle_source_work_start: Vec, last_scan_cycle_source_work: Vec, @@ -1216,6 +1224,22 @@ pub struct ScannerUsageFreshnessSnapshot { pub last_usage_save_unix_secs: u64, pub last_usage_save_result: String, pub last_usage_save_result_code: u64, + #[serde(default)] + pub last_durable_success_unix_secs: u64, + #[serde(default)] + pub last_publication_unix_secs: u64, + #[serde(default)] + pub last_publication_state: String, + #[serde(default)] + pub last_publication_reason: String, + #[serde(default)] + pub deferred_pending: bool, + #[serde(default)] + pub deferred_total: u64, + #[serde(default)] + pub last_deferred_unix_secs: u64, + #[serde(default)] + pub last_deferred_reason: String, } #[derive(Clone, Debug, Default, Serialize, Deserialize)] @@ -1945,7 +1969,15 @@ impl Metrics { scanner_dirty_usage_last_cycle_dirty_buckets: AtomicU64::new(0), scanner_dirty_usage_last_cycle_cleared_buckets: AtomicU64::new(0), scanner_usage_last_save_unix_secs: AtomicU64::new(0), + scanner_usage_last_durable_success_unix_secs: AtomicU64::new(0), + scanner_usage_last_publication_unix_secs: AtomicU64::new(0), + scanner_usage_last_publication_state: Mutex::new(String::new()), + scanner_usage_last_publication_reason: Mutex::new(String::new()), scanner_usage_last_save_result: AtomicU8::new(ScannerUsageSaveResult::Unknown as u8), + scanner_usage_deferred_pending: AtomicBool::new(false), + scanner_usage_deferred_total: AtomicU64::new(0), + scanner_usage_last_deferred_unix_secs: AtomicU64::new(0), + scanner_usage_last_deferred_reason: Mutex::new(String::new()), scanner_source_work: ScannerWorkSource::all() .iter() .map(|_| ScannerSourceWorkCounters::default()) @@ -2270,6 +2302,44 @@ impl Metrics { .store(unix_now_secs(), Ordering::Relaxed); } + /// Record an intentional retryable usage publication deferral separately + /// from the last durable save result. + pub fn record_scanner_usage_deferred(&self, reason: impl Into) { + let reason = reason.into(); + self.record_scanner_usage_publication("deferred", reason.clone()); + self.scanner_usage_deferred_pending.store(true, Ordering::Release); + self.scanner_usage_deferred_total.fetch_add(1, Ordering::Relaxed); + self.scanner_usage_last_deferred_unix_secs + .store(unix_now_secs(), Ordering::Relaxed); + let mut last_reason = match self.scanner_usage_last_deferred_reason.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + *last_reason = reason; + } + + pub fn record_scanner_usage_durable_success(&self) { + self.record_scanner_usage_publication("success", ""); + self.scanner_usage_last_durable_success_unix_secs + .store(unix_now_secs(), Ordering::Relaxed); + self.scanner_usage_deferred_pending.store(false, Ordering::Release); + } + + pub fn record_scanner_usage_publication(&self, state: &str, reason: impl Into) { + self.scanner_usage_last_publication_unix_secs + .store(unix_now_secs(), Ordering::Relaxed); + let mut publication_state = match self.scanner_usage_last_publication_state.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + *publication_state = state.to_string(); + let mut publication_reason = match self.scanner_usage_last_publication_reason.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + *publication_reason = reason.into(); + } + pub fn record_scanner_source_work(&self, source: ScannerWorkSource, work: ScannerSourceWorkUpdate) { if let Some(counters) = self.scanner_source_work.get(source.index()) { counters.add(work); @@ -3292,6 +3362,23 @@ impl Metrics { last_usage_save_unix_secs: self.scanner_usage_last_save_unix_secs.load(Ordering::Relaxed), last_usage_save_result: usage_save_result.as_str().to_string(), last_usage_save_result_code: usage_save_result as u8 as u64, + last_durable_success_unix_secs: self.scanner_usage_last_durable_success_unix_secs.load(Ordering::Relaxed), + last_publication_unix_secs: self.scanner_usage_last_publication_unix_secs.load(Ordering::Relaxed), + last_publication_state: match self.scanner_usage_last_publication_state.lock() { + Ok(state) => state.clone(), + Err(poisoned) => poisoned.into_inner().clone(), + }, + last_publication_reason: match self.scanner_usage_last_publication_reason.lock() { + Ok(reason) => reason.clone(), + Err(poisoned) => poisoned.into_inner().clone(), + }, + deferred_pending: self.scanner_usage_deferred_pending.load(Ordering::Acquire), + deferred_total: self.scanner_usage_deferred_total.load(Ordering::Relaxed), + last_deferred_unix_secs: self.scanner_usage_last_deferred_unix_secs.load(Ordering::Relaxed), + last_deferred_reason: match self.scanner_usage_last_deferred_reason.lock() { + Ok(reason) => reason.clone(), + Err(poisoned) => poisoned.into_inner().clone(), + }, }; m.throttle_idle_mode_enabled = self.scanner_throttle_idle_mode_enabled.load(Ordering::Relaxed); m.throttle_sleep_factor = self.scanner_throttle_sleep_factor_micros.load(Ordering::Relaxed) as f64 / 1_000_000.0; @@ -4663,6 +4750,7 @@ mod tests { metrics.record_scanner_dirty_usage_cycle_snapshot(1); metrics.record_scanner_dirty_usage_cycle_clear(1, 1); metrics.record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + metrics.record_scanner_usage_deferred("data_movement"); let report = metrics.report().await; @@ -4674,6 +4762,22 @@ mod tests { assert!(report.usage_freshness.last_usage_save_unix_secs > 0); assert_eq!(report.usage_freshness.last_usage_save_result, "success"); assert_eq!(report.usage_freshness.last_usage_save_result_code, 1); + assert!(report.usage_freshness.deferred_pending); + assert_eq!(report.usage_freshness.deferred_total, 1); + assert!(report.usage_freshness.last_deferred_unix_secs > 0); + assert_eq!(report.usage_freshness.last_deferred_reason, "data_movement"); + + metrics.record_scanner_usage_durable_success(); + let report = metrics.report().await; + assert!(!report.usage_freshness.deferred_pending); + assert_eq!(report.usage_freshness.deferred_total, 1); + assert!(report.usage_freshness.last_durable_success_unix_secs > 0); + assert_eq!(report.usage_freshness.last_publication_state, "success"); + + metrics.record_scanner_usage_publication("no_update", "no_update"); + let report = metrics.report().await; + assert_eq!(report.usage_freshness.last_publication_state, "no_update"); + assert_eq!(report.usage_freshness.last_publication_reason, "no_update"); } #[tokio::test] diff --git a/crates/ecstore/src/services/metrics_realtime.rs b/crates/ecstore/src/services/metrics_realtime.rs index 6e2e5cd7d..290504fcb 100644 --- a/crates/ecstore/src/services/metrics_realtime.rs +++ b/crates/ecstore/src/services/metrics_realtime.rs @@ -220,6 +220,14 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo last_usage_save_unix_secs: metrics.usage_freshness.last_usage_save_unix_secs, last_usage_save_result: metrics.usage_freshness.last_usage_save_result, last_usage_save_result_code: metrics.usage_freshness.last_usage_save_result_code, + last_durable_success_unix_secs: metrics.usage_freshness.last_durable_success_unix_secs, + last_publication_unix_secs: metrics.usage_freshness.last_publication_unix_secs, + last_publication_state: metrics.usage_freshness.last_publication_state, + last_publication_reason: metrics.usage_freshness.last_publication_reason, + deferred_pending: metrics.usage_freshness.deferred_pending, + deferred_total: metrics.usage_freshness.deferred_total, + last_deferred_unix_secs: metrics.usage_freshness.last_deferred_unix_secs, + last_deferred_reason: metrics.usage_freshness.last_deferred_reason, }, maintenance_control: MadminScannerMaintenanceControlSnapshot { primary_control: metrics.maintenance_control.primary_control, @@ -816,6 +824,14 @@ mod test { last_usage_save_unix_secs: 12, last_usage_save_result: "success".to_string(), last_usage_save_result_code: 1, + last_durable_success_unix_secs: 13, + last_publication_unix_secs: 14, + last_publication_state: "published".to_string(), + last_publication_reason: "complete".to_string(), + deferred_pending: true, + deferred_total: 15, + last_deferred_unix_secs: 16, + last_deferred_reason: "data_movement".to_string(), }, ..Default::default() }); @@ -828,6 +844,14 @@ mod test { assert_eq!(scanner.usage_freshness.last_usage_save_unix_secs, 12); assert_eq!(scanner.usage_freshness.last_usage_save_result, "success"); assert_eq!(scanner.usage_freshness.last_usage_save_result_code, 1); + assert_eq!(scanner.usage_freshness.last_durable_success_unix_secs, 13); + assert_eq!(scanner.usage_freshness.last_publication_unix_secs, 14); + assert_eq!(scanner.usage_freshness.last_publication_state, "published"); + assert_eq!(scanner.usage_freshness.last_publication_reason, "complete"); + assert!(scanner.usage_freshness.deferred_pending); + assert_eq!(scanner.usage_freshness.deferred_total, 15); + assert_eq!(scanner.usage_freshness.last_deferred_unix_secs, 16); + assert_eq!(scanner.usage_freshness.last_deferred_reason, "data_movement"); } #[test] diff --git a/crates/madmin/src/metrics.rs b/crates/madmin/src/metrics.rs index 4e55563a5..0ecd457f6 100644 --- a/crates/madmin/src/metrics.rs +++ b/crates/madmin/src/metrics.rs @@ -302,10 +302,28 @@ pub struct ScannerUsageFreshnessSnapshot { pub last_usage_save_result: String, #[serde(rename = "last_usage_save_result_code", default)] pub last_usage_save_result_code: u64, + #[serde(rename = "last_durable_success_unix_secs", default)] + pub last_durable_success_unix_secs: u64, + #[serde(rename = "last_publication_unix_secs", default)] + pub last_publication_unix_secs: u64, + #[serde(rename = "last_publication_state", default)] + pub last_publication_state: String, + #[serde(rename = "last_publication_reason", default)] + pub last_publication_reason: String, + #[serde(rename = "deferred_pending", default)] + pub deferred_pending: bool, + #[serde(rename = "deferred_total", default)] + pub deferred_total: u64, + #[serde(rename = "last_deferred_unix_secs", default)] + pub last_deferred_unix_secs: u64, + #[serde(rename = "last_deferred_reason", default)] + pub last_deferred_reason: String, } impl ScannerUsageFreshnessSnapshot { fn merge(&mut self, other: &Self) { + let self_deferred_state_at = self.last_deferred_unix_secs.max(self.last_durable_success_unix_secs); + let other_deferred_state_at = other.last_deferred_unix_secs.max(other.last_durable_success_unix_secs); self.dirty_pending_buckets = self.dirty_pending_buckets.saturating_add(other.dirty_pending_buckets); self.last_dirty_mark_unix_secs = self.last_dirty_mark_unix_secs.max(other.last_dirty_mark_unix_secs); self.last_dirty_clear_unix_secs = self.last_dirty_clear_unix_secs.max(other.last_dirty_clear_unix_secs); @@ -318,6 +336,22 @@ impl ScannerUsageFreshnessSnapshot { self.last_usage_save_result = other.last_usage_save_result.clone(); self.last_usage_save_result_code = other.last_usage_save_result_code; } + self.last_durable_success_unix_secs = self.last_durable_success_unix_secs.max(other.last_durable_success_unix_secs); + if other.last_publication_unix_secs > self.last_publication_unix_secs { + self.last_publication_unix_secs = other.last_publication_unix_secs; + self.last_publication_state = other.last_publication_state.clone(); + self.last_publication_reason = other.last_publication_reason.clone(); + } + self.deferred_total = self.deferred_total.saturating_add(other.deferred_total); + if other_deferred_state_at > self_deferred_state_at { + self.deferred_pending = other.deferred_pending; + } else if other_deferred_state_at == self_deferred_state_at { + self.deferred_pending |= other.deferred_pending; + } + if other.last_deferred_unix_secs > self.last_deferred_unix_secs { + self.last_deferred_unix_secs = other.last_deferred_unix_secs; + self.last_deferred_reason = other.last_deferred_reason.clone(); + } } } @@ -1566,6 +1600,47 @@ mod tests { assert_eq!(explicit_false.current_cycle_active, Some(false)); } + #[test] + fn usage_freshness_deferred_fields_are_backward_compatible_and_merge() { + let legacy: ScannerUsageFreshnessSnapshot = serde_json::from_value(serde_json::json!({ + "dirty_pending_buckets": 2, + "last_usage_save_result": "success" + })) + .expect("legacy usage freshness should decode"); + assert!(!legacy.deferred_pending); + assert_eq!(legacy.deferred_total, 0); + + let mut merged = ScannerUsageFreshnessSnapshot::default(); + merged.merge(&ScannerUsageFreshnessSnapshot { + deferred_pending: true, + deferred_total: 2, + last_deferred_unix_secs: 20, + last_deferred_reason: "data_movement".to_string(), + ..Default::default() + }); + merged.merge(&ScannerUsageFreshnessSnapshot { + deferred_total: 1, + last_deferred_unix_secs: 10, + last_deferred_reason: "older".to_string(), + ..Default::default() + }); + assert!(merged.deferred_pending); + assert_eq!(merged.deferred_total, 3); + assert_eq!(merged.last_deferred_unix_secs, 20); + assert_eq!(merged.last_deferred_reason, "data_movement"); + + merged.merge(&ScannerUsageFreshnessSnapshot { + deferred_pending: false, + last_durable_success_unix_secs: 30, + last_publication_unix_secs: 30, + last_publication_state: "success".to_string(), + ..Default::default() + }); + assert!(!merged.deferred_pending); + assert_eq!(merged.last_durable_success_unix_secs, 30); + assert_eq!(merged.last_publication_state, "success"); + } + #[test] fn scanner_metrics_merge_prefers_an_active_first_cycle() { let collected_at = Timestamp::now(); diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index c44ea8ae0..362eb79a7 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1375,6 +1375,7 @@ async fn run_data_scanner_cycle_with_budget( Ok(result) => final_data_usage_publication_defer_reason(storeapi.as_ref(), result.status).await, Err(_) => Some(ScannerCycleDeferReason::ActivityBaselineUnavailable), }; + let publication_deferred = publication_defer_reason.is_some(); let publication_epoch = scan_result.as_ref().ok().and_then(ScannerCycleResult::publication_epoch); let budget_elapsed = cycle_budget.budget_elapsed() && !ctx.is_cancelled(); let usage_persist_outcome = match publication_defer_reason { @@ -1539,6 +1540,9 @@ async fn run_data_scanner_cycle_with_budget( state = "deferred", "Scanner cycle deferred before data usage publication" ); + if publication_deferred { + global_metrics().record_scanner_usage_deferred(reason.as_str()); + } emit_scan_cycle_deferred(cycle_start.elapsed()); mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; return ScannerCycleOutcome::Deferred(reason); @@ -1708,6 +1712,7 @@ async fn run_data_scanner_cycle_with_budget( state = "deferred", "Scanner cycle deferred before usage scanning began" ); + global_metrics().record_scanner_usage_deferred(reason.as_str()); emit_scan_cycle_deferred(cycle_start.elapsed()); mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; return ScannerCycleOutcome::Deferred(reason); diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 9e1d7cef8..b59cdc2d8 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -2788,6 +2788,7 @@ async fn test_deferred_usage_save_keeps_last_real_save_metric() { assert_eq!(after.last_usage_save_result, before.last_usage_save_result); assert_eq!(after.last_usage_save_result_code, before.last_usage_save_result_code); assert_eq!(after.last_usage_save_unix_secs, before.last_usage_save_unix_secs); + assert_eq!(after.deferred_total, before.deferred_total.saturating_add(1)); } #[tokio::test] diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs index 9b211c1f5..24e5fd543 100644 --- a/crates/scanner/src/scanner/usage_store.rs +++ b/crates/scanner/src/scanner/usage_store.rs @@ -186,6 +186,7 @@ where state = "publication_blocked_before_reconcile", "Scanner data usage publication deferred by the pool-state fence" ); + global_metrics().record_scanner_usage_deferred(ScannerCycleDeferReason::DataMovement.as_str()); outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); break; } @@ -288,6 +289,7 @@ where state = "reject_incomplete_snapshot", "Scanner refused to persist an incomplete data usage snapshot" ); + global_metrics().record_scanner_usage_publication("failed", "incomplete_snapshot"); global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed); outcome = DataUsagePersistOutcome::Failed; continue; @@ -306,6 +308,7 @@ where error = %e, "Scanner data usage encode failed" ); + global_metrics().record_scanner_usage_publication("failed", "encode_failed"); global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::EncodeFailed); outcome = DataUsagePersistOutcome::Failed; continue; @@ -549,6 +552,7 @@ where replace_bucket_usage_memory_from_info(&data_usage_info).await; } global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + global_metrics().record_scanner_usage_durable_success(); outcome = DataUsagePersistOutcome::AlreadyDurable; } DataUsagePersistOutcome::PriorCycleDurable => { @@ -568,9 +572,17 @@ where invalidate_data_usage_snapshot_cache().await; } global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + global_metrics().record_scanner_usage_durable_success(); outcome = DataUsagePersistOutcome::PriorCycleDurable; } - DataUsagePersistOutcome::Failed | DataUsagePersistOutcome::NoUpdate => { + DataUsagePersistOutcome::NoUpdate => { + global_metrics().record_scanner_usage_publication("no_update", "no_update"); + global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed); + outcome = DataUsagePersistOutcome::NoUpdate; + continue; + } + DataUsagePersistOutcome::Failed => { + global_metrics().record_scanner_usage_publication("failed", "save_failed"); global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Failed); outcome = DataUsagePersistOutcome::Failed; continue; @@ -579,6 +591,7 @@ where // A deferred publication is an intentional retryable state, not a // failed save. Keep the last real save result so admin freshness // reporting does not turn a pool-recovery fence into a false error. + global_metrics().record_scanner_usage_deferred(reason.as_str()); outcome = DataUsagePersistOutcome::Deferred(reason); break 'updates; } @@ -600,6 +613,7 @@ where replace_bucket_usage_memory_from_info(&data_usage_info).await; } global_metrics().record_scanner_usage_save_result(ScannerUsageSaveResult::Success); + global_metrics().record_scanner_usage_durable_success(); outcome = DataUsagePersistOutcome::Saved; } } diff --git a/rustfs/src/admin/handlers/cluster_snapshot.rs b/rustfs/src/admin/handlers/cluster_snapshot.rs index 3694d01a4..1646f1228 100644 --- a/rustfs/src/admin/handlers/cluster_snapshot.rs +++ b/rustfs/src/admin/handlers/cluster_snapshot.rs @@ -37,6 +37,9 @@ use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use serde::Serialize; +use std::time::{SystemTime, UNIX_EPOCH}; + +const USAGE_DEFERRED_STALE_THRESHOLD_SECS: u64 = 300; pub fn register_cluster_snapshot_route(r: &mut S3Router) -> std::io::Result<()> { r.insert( @@ -242,7 +245,16 @@ pub(crate) struct ClusterUsageFreshnessStatus { pub last_usage_save_unix_secs: u64, pub last_usage_save_result: String, pub last_success_unix_secs: Option, + pub last_durable_success_unix_secs: u64, + pub last_publication_unix_secs: u64, + pub last_publication_state: String, + pub last_publication_reason: String, pub last_error: Option, + pub deferred_pending: bool, + pub deferred_total: u64, + pub last_deferred_unix_secs: u64, + pub last_deferred_reason: String, + pub deferred_age_secs: Option, } fn component_status(source: &'static str, status: CapabilityStatus) -> ClusterComponentStatus { @@ -701,32 +713,71 @@ fn summarize_listing_metacache(snapshot: &ClusterReadOnlySnapshot) -> ClusterLis fn summarize_usage_freshness(snapshot: &ClusterReadOnlySnapshot) -> ClusterUsageFreshnessStatus { let freshness = &snapshot.usage_freshness; - let (condition, status) = match freshness.last_usage_save_result.as_str() { - "success" if freshness.dirty_pending_buckets == 0 => ( - "healthy", - CapabilityStatus::supported().with_reason("usage cache was saved successfully and has no pending dirty buckets"), - ), - "success" | "" if freshness.dirty_pending_buckets > 0 => ( + let deferred_age_secs = (freshness.deferred_pending && freshness.last_deferred_unix_secs > 0).then(|| { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| duration.as_secs()); + now.saturating_sub(freshness.last_deferred_unix_secs) + }); + let deferred_stale = deferred_age_secs.is_some_and(|age| age > USAGE_DEFERRED_STALE_THRESHOLD_SECS); + let (condition, status) = if freshness.last_publication_state == "no_update" { + ( + "no_update", + CapabilityStatus::unknown().with_reason("usage cache publication produced no update"), + ) + } else if freshness.deferred_pending && deferred_stale { + ( "stale", - CapabilityStatus::unknown() - .with_reason(format!("usage cache has {} pending dirty buckets", freshness.dirty_pending_buckets)), - ), - "skipped_stale" => ( - "stale", - CapabilityStatus::unknown().with_reason("last usage cache save was skipped because scanner data was stale"), - ), - "failed" => ("degraded", CapabilityStatus::unknown().with_reason("last usage cache save failed")), - "encode_failed" => ( - "degraded", - CapabilityStatus::unknown().with_reason("last usage cache save failed during encoding"), - ), - _ => ( - "unknown", - CapabilityStatus::unknown().with_reason("no usage cache save result has been reported"), - ), + CapabilityStatus::unknown().with_reason(format!( + "usage cache publication has been deferred for {} seconds (threshold: {} seconds)", + deferred_age_secs.unwrap_or_default(), + USAGE_DEFERRED_STALE_THRESHOLD_SECS + )), + ) + } else if freshness.deferred_pending { + ( + "deferred", + CapabilityStatus::unknown().with_reason(if freshness.last_deferred_reason.is_empty() { + "usage cache publication is temporarily deferred" + } else { + freshness.last_deferred_reason.as_str() + }), + ) + } else { + match freshness.last_usage_save_result.as_str() { + "success" if freshness.dirty_pending_buckets == 0 => ( + "healthy", + CapabilityStatus::supported().with_reason("usage cache was saved successfully and has no pending dirty buckets"), + ), + "success" | "" if freshness.dirty_pending_buckets > 0 => ( + "stale", + CapabilityStatus::unknown() + .with_reason(format!("usage cache has {} pending dirty buckets", freshness.dirty_pending_buckets)), + ), + "skipped_stale" => ( + "stale", + CapabilityStatus::unknown().with_reason("last usage cache save was skipped because scanner data was stale"), + ), + "failed" => ("degraded", CapabilityStatus::unknown().with_reason("last usage cache save failed")), + "encode_failed" => ( + "degraded", + CapabilityStatus::unknown().with_reason("last usage cache save failed during encoding"), + ), + _ => ( + "unknown", + CapabilityStatus::unknown().with_reason("no usage cache save result has been reported"), + ), + } + }; + // Mixed-version nodes do not publish the additive durable timestamp yet; + // retain the legacy successful-save timestamp until all peers are upgraded. + let last_success_unix_secs = if freshness.last_durable_success_unix_secs > 0 { + Some(freshness.last_durable_success_unix_secs) + } else if freshness.last_usage_save_result == "success" && freshness.last_usage_save_unix_secs > 0 { + Some(freshness.last_usage_save_unix_secs) + } else { + None }; - let last_success_unix_secs = (freshness.last_usage_save_result == "success" && freshness.last_usage_save_unix_secs > 0) - .then_some(freshness.last_usage_save_unix_secs); let last_error = match freshness.last_usage_save_result.as_str() { "failed" | "skipped_stale" | "encode_failed" => Some(freshness.last_usage_save_result.clone()), _ => None, @@ -744,7 +795,16 @@ fn summarize_usage_freshness(snapshot: &ClusterReadOnlySnapshot) -> ClusterUsage last_usage_save_unix_secs: freshness.last_usage_save_unix_secs, last_usage_save_result: freshness.last_usage_save_result.clone(), last_success_unix_secs, + last_durable_success_unix_secs: freshness.last_durable_success_unix_secs, + last_publication_unix_secs: freshness.last_publication_unix_secs, + last_publication_state: freshness.last_publication_state.clone(), + last_publication_reason: freshness.last_publication_reason.clone(), last_error, + deferred_pending: freshness.deferred_pending, + deferred_total: freshness.deferred_total, + last_deferred_unix_secs: freshness.last_deferred_unix_secs, + last_deferred_reason: freshness.last_deferred_reason.clone(), + deferred_age_secs, } } @@ -1282,6 +1342,7 @@ mod tests { usage_freshness: ClusterUsageFreshnessSnapshot { dirty_pending_buckets: 0, last_usage_save_unix_secs: 456, + last_durable_success_unix_secs: 450, last_usage_save_result: "success".to_string(), last_usage_save_result_code: 1, ..Default::default() @@ -1295,6 +1356,12 @@ mod tests { assert_eq!(component.condition, "healthy"); assert_eq!(component.last_usage_save_unix_secs, 456); assert_eq!(component.last_usage_save_result, "success"); + assert_eq!(component.last_success_unix_secs, Some(450)); + + let mut legacy_snapshot = snapshot.clone(); + legacy_snapshot.usage_freshness.last_durable_success_unix_secs = 0; + let legacy_component = super::summarize_usage_freshness(&legacy_snapshot); + assert_eq!(legacy_component.last_success_unix_secs, Some(456)); } #[test] diff --git a/rustfs/src/cluster_snapshot.rs b/rustfs/src/cluster_snapshot.rs index 8112afaf9..e7535eb5d 100644 --- a/rustfs/src/cluster_snapshot.rs +++ b/rustfs/src/cluster_snapshot.rs @@ -66,6 +66,14 @@ pub struct ClusterUsageFreshnessSnapshot { pub last_usage_save_unix_secs: u64, pub last_usage_save_result: String, pub last_usage_save_result_code: u64, + pub last_durable_success_unix_secs: u64, + pub last_publication_unix_secs: u64, + pub last_publication_state: String, + pub last_publication_reason: String, + pub deferred_pending: bool, + pub deferred_total: u64, + pub last_deferred_unix_secs: u64, + pub last_deferred_reason: String, } impl From<&ScannerMetricsReport> for ClusterUsageFreshnessSnapshot { @@ -79,6 +87,14 @@ impl From<&ScannerMetricsReport> for ClusterUsageFreshnessSnapshot { last_usage_save_unix_secs: report.usage_freshness.last_usage_save_unix_secs, last_usage_save_result: report.usage_freshness.last_usage_save_result.clone(), last_usage_save_result_code: report.usage_freshness.last_usage_save_result_code, + last_durable_success_unix_secs: report.usage_freshness.last_durable_success_unix_secs, + last_publication_unix_secs: report.usage_freshness.last_publication_unix_secs, + last_publication_state: report.usage_freshness.last_publication_state.clone(), + last_publication_reason: report.usage_freshness.last_publication_reason.clone(), + deferred_pending: report.usage_freshness.deferred_pending, + deferred_total: report.usage_freshness.deferred_total, + last_deferred_unix_secs: report.usage_freshness.last_deferred_unix_secs, + last_deferred_reason: report.usage_freshness.last_deferred_reason.clone(), } } } From 3f3b9fd42665abfc420f07a30d0c73886bbb7a4a Mon Sep 17 00:00:00 2001 From: houseme Date: Sun, 23 Aug 2026 19:31:14 +0800 Subject: [PATCH 22/41] perf(ecstore): attribute batch read version wait stages (#6456) --- crates/ecstore/src/core/pools.rs | 1 - .../src/set_disk/core/io_primitives.rs | 23 +++++++++++++++++-- crates/io-metrics/src/internode_metrics.rs | 4 ++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index c40e8a83a..b2d2f79c7 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -3599,7 +3599,6 @@ impl ECStore { if should_save_pool_meta { self.ctx.advance_data_movement_operation_epoch(); } - drop(_movement_guard); if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("decommission_cancel for pool {idx}"); diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index a7310b44f..0c743a4cb 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -71,6 +71,9 @@ use crate::set_disk::shard_source::ShardReadCost; use futures::FutureExt as _; use futures::stream::{FuturesUnordered, StreamExt}; use metrics::counter; +use rustfs_io_metrics::internode_metrics::{ + INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, +}; use std::{ collections::{HashMap, HashSet, VecDeque}, future::Future, @@ -193,6 +196,16 @@ fn record_read_version_coalescer_event(event: &'static str, item_count: usize) { .increment(1); } +fn batch_read_version_stage_timer() -> Option { + rustfs_io_metrics::get_stage_metrics_enabled().then(Instant::now) +} + +fn record_batch_read_version_stage(stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed()); + } +} + async fn read_version_via_coalescer( disk: DiskStore, org_bucket: &str, @@ -242,8 +255,12 @@ async fn read_version_via_coalescer( flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await; } - rx.await - .unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed"))) + let wait_started = batch_read_version_stage_timer(); + let response = rx + .await + .unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed"))); + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, wait_started); + response } async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) { @@ -292,7 +309,9 @@ async fn flush_read_version_coalescer_pending( }; match result { Ok(responses) => { + let map_started = batch_read_version_stage_timer(); let results = map_batch_read_version_responses(&expected_items, responses); + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, map_started); for (tx, result) in senders.into_iter().zip(results) { let _ = tx.send(result); } diff --git a/crates/io-metrics/src/internode_metrics.rs b/crates/io-metrics/src/internode_metrics.rs index 6b1ea382c..24709b82a 100644 --- a/crates/io-metrics/src/internode_metrics.rs +++ b/crates/io-metrics/src/internode_metrics.rs @@ -61,6 +61,8 @@ pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode"; pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip"; pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT: &str = "batch_read_version_coalescer_wait"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP: &str = "batch_read_version_response_map"; const OPERATION_LABEL: &str = "operation"; const BACKEND_LABEL: &str = "backend"; @@ -1431,6 +1433,8 @@ mod tests { ); assert_eq!(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, "read_version_rpc_roundtrip"); assert_eq!(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, "read_version_response_decode"); + assert_eq!(INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT, "batch_read_version_coalescer_wait"); + assert_eq!(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP, "batch_read_version_response_map"); assert_eq!( INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL, "rustfs_system_network_internode_signature_v1_fallback_total" From 38d37121cad8bee1287a71ffd6b135690f04f21a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 20:17:07 +0800 Subject: [PATCH 23/41] test(e2e): activate group management regressions (#6405) --- .config/e2e-full-selection.txt | 4 +- crates/e2e_test/src/group_delete_test.rs | 154 +++++++++++++++++------ docs/testing/e2e-suite-inventory.md | 4 +- 3 files changed, 122 insertions(+), 40 deletions(-) diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index dfad0f0bd..59a5d8ab9 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52 -sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6 +sha256-darwin=f832043fcca8c0b616c5d820a3a652da7544298ef5812a8668a3a9a3e4607b8b +sha256-linux=93b94adb110b86a41d0b7313909e0bf53cb1515e2d08e8f105652b29b249990f diff --git a/crates/e2e_test/src/group_delete_test.rs b/crates/e2e_test/src/group_delete_test.rs index 38d053f99..d6aaec32f 100644 --- a/crates/e2e_test/src/group_delete_test.rs +++ b/crates/e2e_test/src/group_delete_test.rs @@ -14,7 +14,7 @@ //! E2E tests for group management (fixes #2028). -use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging}; +use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging}; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::{Client, Config}; use tracing::info; @@ -83,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo /// Test that deleting a group with members fails, and deleting an empty group succeeds. #[tokio::test(flavor = "multi_thread")] -#[ignore = "requires awscurl and spawns a real RustFS server"] async fn test_delete_group_requires_empty_membership() -> Result<(), Box> { init_logging(); @@ -91,29 +90,58 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), BoxInvalidRequest"), + "deleting a non-empty group must return InvalidRequest, body: {delete_body}" + ); + assert!( + delete_body.contains("group is not empty"), + "deleting a non-empty group returned an unexpected message: {delete_body}" + ); info!("Delete of non-empty group correctly rejected"); // 4. Remove the member from the group @@ -123,17 +151,42 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), BoxNoSuchResource"), + "a deleted group must return NoSuchResource, body: {get_body}" + ); + assert!( + get_body.contains("group 'testgroup' does not exist"), + "a deleted group returned an unexpected message: {get_body}" + ); info!("Confirmed testgroup no longer exists"); Ok(()) @@ -142,7 +195,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box Result<(), Box> { init_logging(); @@ -160,39 +212,56 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box Result<(), Box Result<(), Box> { init_logging(); @@ -221,33 +289,47 @@ async fn test_delete_group_after_deleting_user() -> Result<(), Box Date: Sun, 23 Aug 2026 20:17:24 +0800 Subject: [PATCH 24/41] fix(admin): advertise IAM admin capabilities in runtime capabilities (#6336) --- rustfs/src/admin/handlers/system.rs | 81 +++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/rustfs/src/admin/handlers/system.rs b/rustfs/src/admin/handlers/system.rs index 981baaefb..3b678c069 100644 --- a/rustfs/src/admin/handlers/system.rs +++ b/rustfs/src/admin/handlers/system.rs @@ -70,6 +70,12 @@ const SITE_REPLICATION_EDIT_ROUTE: &str = "/rustfs/admin/v3/site-replication/edi const SITE_REPLICATION_RESYNC_ROUTE: &str = "/rustfs/admin/v3/site-replication/resync/op"; const SITE_REPLICATION_REPAIR_ROUTE: &str = "/rustfs/admin/v3/site-replication/repair"; const SITE_REPLICATION_REPAIR_STATUS_ROUTE: &str = "/rustfs/admin/v3/site-replication/repair/status"; +const IAM_POLICY_ATTACH_ROUTE: &str = "/rustfs/admin/v3/idp/builtin/policy/attach"; +const IAM_POLICY_DETACH_ROUTE: &str = "/rustfs/admin/v3/idp/builtin/policy/detach"; +const IAM_POLICY_ENTITIES_ROUTE: &str = "/rustfs/admin/v3/idp/builtin/policy-entities"; +const IAM_ACCESS_KEYS_BULK_ROUTE: &str = "/rustfs/admin/v3/list-access-keys-bulk"; +const IAM_ACCESS_KEYS_BULK_LDAP_ROUTE: &str = "/rustfs/admin/v3/idp/ldap/list-access-keys-bulk"; +const IAM_ACCESS_KEYS_BULK_OPENID_ROUTE: &str = "/rustfs/admin/v3/idp/openid/list-access-keys-bulk"; macro_rules! log_system_request_rejected { ($operation:expr, $reason:expr) => { @@ -661,9 +667,24 @@ pub struct RuntimeCapabilitiesSummary { pub manual_transition_jobs: CapabilityStatus, } +/// One named admin capability advertised to management clients +/// (rustfs/backlog#1900). `name` is a cross-repo wire contract: the rc +/// client gates commands on these exact strings (see rustfs/cli +/// `IAM_POLICY_DETACH_CAPABILITY` etc.), so entries may be added but +/// existing names must never be renamed or removed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct AdvertisedAdminCapability { + pub name: &'static str, + pub status: CapabilityStatus, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct RuntimeCapabilitiesResponse { pub summary: RuntimeCapabilitiesSummary, + /// Additive field: absent in responses from older servers, so clients + /// must treat a missing list as "no dynamic advertisement" and fall + /// back to their pinned per-version contract. + pub advertised: Vec, pub replication: ReplicationCapabilities, pub manual_transition_jobs: ManualTransitionJobCapabilities, pub diagnostic_probes: DiagnosticProbeCapabilities, @@ -986,6 +1007,7 @@ pub(crate) async fn build_runtime_capabilities_response() Ok(RuntimeCapabilitiesResponse { summary, + advertised: advertised_admin_capabilities(), replication: ReplicationCapabilities::current(), manual_transition_jobs: ManualTransitionJobCapabilities::current(), diagnostic_probes: DiagnosticProbeCapabilities::current(), @@ -1077,6 +1099,23 @@ fn admin_route_capability(method: HttpMethod, path: &str) -> CapabilityStatus { admin_route_capability_from_inventory(method, path, ADMIN_ROUTE_POLICY_SPECS, DEFERRED_ADMIN_ROUTE_POLICIES) } +fn advertised_admin_capabilities() -> Vec { + [ + ("admin.iam.policy-attach", HttpMethod::Post, IAM_POLICY_ATTACH_ROUTE), + ("admin.iam.policy-detach", HttpMethod::Post, IAM_POLICY_DETACH_ROUTE), + ("admin.iam.policy-entities", HttpMethod::Get, IAM_POLICY_ENTITIES_ROUTE), + ("admin.iam.access-keys-bulk", HttpMethod::Get, IAM_ACCESS_KEYS_BULK_ROUTE), + ("admin.iam.access-keys-bulk.ldap", HttpMethod::Get, IAM_ACCESS_KEYS_BULK_LDAP_ROUTE), + ("admin.iam.access-keys-bulk.openid", HttpMethod::Get, IAM_ACCESS_KEYS_BULK_OPENID_ROUTE), + ] + .into_iter() + .map(|(name, method, route)| AdvertisedAdminCapability { + name, + status: admin_route_capability(method, route), + }) + .collect() +} + fn admin_route_capability_from_inventory( method: HttpMethod, path: &str, @@ -1239,6 +1278,48 @@ mod tests { ); } + /// Wire-contract pin (rustfs/backlog#1900): the rc client keys its + /// command gates on these exact capability names, and parses each + /// entry as `{name, status: {state, reason?}}`. Renaming or dropping + /// a name silently disables the corresponding rc command. + #[tokio::test] + async fn runtime_capabilities_response_advertises_iam_capabilities() { + let response = build_runtime_capabilities_response() + .await + .expect("runtime capabilities response should build"); + + let expected_supported = [ + "admin.iam.policy-attach", + "admin.iam.policy-detach", + "admin.iam.policy-entities", + "admin.iam.access-keys-bulk", + "admin.iam.access-keys-bulk.ldap", + "admin.iam.access-keys-bulk.openid", + ]; + for name in expected_supported { + let entry = response + .advertised + .iter() + .find(|capability| capability.name == name) + .unwrap_or_else(|| panic!("{name} must be advertised")); + assert_eq!(entry.status.state, CapabilityState::Supported, "{name} must be supported"); + } + + let mut names: Vec<&str> = response.advertised.iter().map(|capability| capability.name).collect(); + let total = names.len(); + names.sort_unstable(); + names.dedup(); + assert_eq!(names.len(), total, "advertised capability names must be unique"); + + let serialized = serde_json::to_value(&response).expect("response should serialize"); + let advertised = serialized["advertised"].as_array().expect("advertised must be an array"); + let detach = advertised + .iter() + .find(|entry| entry["name"] == "admin.iam.policy-detach") + .expect("serialized detach entry must exist"); + assert_eq!(detach["status"]["state"], "supported"); + } + #[tokio::test] async fn runtime_capabilities_response_reports_missing_topology_before_storage_init() { let response = build_runtime_capabilities_response() From 0e88a27d052a34b8eae58cca389cb28d63120129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 23 Aug 2026 20:17:34 +0800 Subject: [PATCH 25/41] fix(admin): use madmin key names in list-remote-targets response (#6377) --- .../src/bucket/target/bucket_target.rs | 93 +++++++++++- rustfs/src/admin/handlers/replication.rs | 141 +++++++++++++++++- 2 files changed, 228 insertions(+), 6 deletions(-) diff --git a/crates/ecstore/src/bucket/target/bucket_target.rs b/crates/ecstore/src/bucket/target/bucket_target.rs index 55e3d3e8a..0f5e3f62b 100644 --- a/crates/ecstore/src/bucket/target/bucket_target.rs +++ b/crates/ecstore/src/bucket/target/bucket_target.rs @@ -32,6 +32,10 @@ pub struct Credentials { pub access_key: String, #[serde(rename = "secretKey")] pub secret_key: String, + // The aliases accept madmin's JSON tags (MinIO-written bucket-targets + // metadata and mc request bodies) without changing the snake_case + // persisted/peer wire format this struct serializes to. + #[serde(alias = "sessionToken")] pub session_token: Option, pub expiration: Option, } @@ -202,12 +206,14 @@ pub struct BucketTarget { #[serde(default)] pub region: String, - #[serde(alias = "bandwidth", default)] + // madmin-go v3.0.109 tags this `bandwidthlimit`; `bandwidth` is a legacy + // alias kept for inputs written before the madmin tag was verified. + #[serde(alias = "bandwidthlimit", alias = "bandwidth", default)] pub bandwidth_limit: i64, #[serde(rename = "replicationSync", default)] pub replication_sync: bool, - #[serde(default)] + #[serde(alias = "storageclass", default)] pub storage_class: String, #[serde(rename = "skipTlsVerify", default)] pub skip_tls_verify: bool, @@ -220,7 +226,7 @@ pub struct BucketTarget { #[serde(rename = "resetBeforeDate", with = "time::serde::rfc3339::option", default)] pub reset_before_date: Option, - #[serde(default)] + #[serde(alias = "resetID", default)] pub reset_id: String, #[serde(rename = "totalDowntime", with = "duration_seconds", default)] pub total_downtime: Duration, @@ -233,7 +239,7 @@ pub struct BucketTarget { #[serde(default)] pub latency: LatencyStat, - #[serde(default)] + #[serde(alias = "deploymentID", default)] pub deployment_id: String, #[serde(default)] @@ -531,6 +537,85 @@ mod tests { assert_eq!(value["totalDowntime"], 90); } + #[test] + fn bucket_target_persisted_wire_keys_stay_snake_case() { + // bucket-targets.json (persisted via `serde_json::to_vec(&BucketTargets)` + // in the admin set/remove handlers) and the msgpack struct-map form + // (`BucketTargets::marshal_msg`) both come straight from this struct's + // serde field names. madmin naming is applied only in the admin + // response layer (`remote_target_admin_json`); renaming here would + // silently break every existing deployment's persisted metadata. + let targets = BucketTargets { + targets: vec![BucketTarget { + credentials: Some(Credentials { + access_key: "ak".to_string(), + secret_key: "sk".to_string(), + session_token: Some("token".to_string()), + expiration: None, + }), + bandwidth_limit: 5, + storage_class: "STANDARD".to_string(), + reset_id: "reset-1".to_string(), + deployment_id: "deploy-1".to_string(), + ..Default::default() + }], + }; + + let json = serde_json::to_value(&targets).expect("targets should serialize to JSON"); + let msgpack: serde_json::Value = + rmp_serde::from_slice(&targets.marshal_msg().expect("targets should marshal to msgpack")) + .expect("msgpack struct map should decode into a JSON value"); + + for (wire, entry) in [("JSON", &json["targets"][0]), ("msgpack", &msgpack["targets"][0])] { + assert_eq!(entry["bandwidth_limit"], 5, "{wire} key `bandwidth_limit` must stay"); + assert_eq!(entry["storage_class"], "STANDARD", "{wire} key `storage_class` must stay"); + assert_eq!(entry["reset_id"], "reset-1", "{wire} key `reset_id` must stay"); + assert_eq!(entry["deployment_id"], "deploy-1", "{wire} key `deployment_id` must stay"); + assert_eq!(entry["credentials"]["session_token"], "token", "{wire} key `session_token` must stay"); + } + } + + #[test] + fn minio_written_bucket_targets_json_populates_madmin_named_fields() { + // A MinIO-written bucket-targets.json carries madmin's JSON tags + // (`bandwidthlimit`, `storageclass`, `resetID`, `deploymentID`, + // `credentials.sessionToken` — madmin-go v3.0.109 bucket-targets.go). + // On migration these must land in the matching fields instead of + // silently defaulting (backlog#1951). + let targets: BucketTargets = serde_json::from_value(serde_json::json!({ + "targets": [{ + "sourcebucket": "src", + "endpoint": "minio.example:9000", + "credentials": { + "accessKey": "ak", + "secretKey": "sk", + "sessionToken": "minio-session-token" + }, + "targetbucket": "dst", + "type": "replication", + "replicationSync": true, + "bandwidthlimit": 107374182400i64, + "storageclass": "STANDARD", + "resetID": "reset-789", + "deploymentID": "deploy-123" + }] + })) + .expect("MinIO-written bucket-targets.json must deserialize"); + + let target = &targets.targets[0]; + assert_eq!(target.bandwidth_limit, 107374182400); + assert_eq!(target.storage_class, "STANDARD"); + assert_eq!(target.reset_id, "reset-789"); + assert_eq!(target.deployment_id, "deploy-123"); + assert_eq!( + target + .credentials + .as_ref() + .and_then(|credentials| credentials.session_token.as_deref()), + Some("minio-session-token") + ); + } + #[test] fn test_bucket_target_debug_redacts_credentials() { let target = BucketTarget { diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index 229c3c9fd..ea7c990b4 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -376,7 +376,10 @@ impl RemoteTargetRequest { /// Admin-response encoding of a remote target: the persisted bucket-targets /// format keeps `healthCheckDuration`/`totalDowntime` in seconds and the /// `latency` stats in milliseconds, but madmin decodes all of them as Go -/// `time.Duration` (nanoseconds) — re-encode just those fields without +/// `time.Duration` (nanoseconds) — and it looks the fields up under its own +/// JSON tags (`bandwidthlimit`, `storageclass`, `resetID`, `deploymentID`, +/// `credentials.sessionToken` — madmin-go v3.0.109 `bucket-targets.go`), not +/// the persisted snake_case keys. Re-encode just those fields here without /// touching the persistence wire format. fn remote_target_admin_json(target: &BucketTarget) -> Result { fn go_duration_nanos(duration: Duration) -> serde_json::Value { @@ -385,6 +388,12 @@ fn remote_target_admin_json(target: &BucketTarget) -> Result Result, + targetbucket: String, + arn: String, + bandwidthlimit: i64, + #[serde(rename = "replicationSync")] + replication_sync: bool, + storageclass: String, + #[serde(rename = "healthCheckDuration")] + health_check_duration: i64, + #[serde(rename = "resetID")] + reset_id: String, + #[serde(rename = "totalDowntime")] + total_downtime: i64, + #[serde(rename = "deploymentID")] + deployment_id: String, + } + + #[derive(Debug, Default, serde::Deserialize)] + #[serde(default)] + struct MadminCredentials { + #[serde(rename = "accessKey")] + access_key: String, + #[serde(rename = "secretKey")] + secret_key: String, + #[serde(rename = "sessionToken")] + session_token: String, + } + + #[test] + fn list_remote_targets_response_decodes_through_madmin_tags() { + // Regression for the review on backlog#1951: the response must decode + // a nonzero bandwidth limit through madmin's `bandwidthlimit` tag (not + // `bandwidth`, which Go would silently drop as an unknown key). + let target = BucketTarget { + source_bucket: "src".to_string(), + endpoint: "192.168.1.10:9000".to_string(), + target_bucket: "target".to_string(), + arn: "arn:rustfs:replication:us-east-1:dep:target".to_string(), + credentials: Some(TargetCredentials { + access_key: "access".to_string(), + secret_key: String::new(), + session_token: Some("session-token".to_string()), + expiration: None, + }), + bandwidth_limit: 1_073_741_824, + replication_sync: true, + storage_class: "STANDARD".to_string(), + health_check_duration: std::time::Duration::from_secs(60), + reset_id: "reset-123".to_string(), + total_downtime: std::time::Duration::from_secs(90), + deployment_id: "deploy-456".to_string(), + ..Default::default() + }; + + let wire = serde_json::to_string(&super::remote_target_admin_json(&target).expect("admin response should serialize")) + .expect("admin response should encode"); + let decoded: MadminBucketTarget = serde_json::from_str(&wire).expect("madmin-shaped decode must succeed"); + + assert_eq!(decoded.bandwidthlimit, 1_073_741_824, "mc must see the nonzero bandwidth limit"); + assert_eq!(decoded.sourcebucket, "src"); + assert_eq!(decoded.endpoint, "192.168.1.10:9000"); + assert_eq!(decoded.targetbucket, "target"); + assert_eq!(decoded.arn, "arn:rustfs:replication:us-east-1:dep:target"); + assert!(decoded.replication_sync); + assert_eq!(decoded.storageclass, "STANDARD"); + assert_eq!(decoded.health_check_duration, 60_000_000_000); + assert_eq!(decoded.reset_id, "reset-123"); + assert_eq!(decoded.total_downtime, 90_000_000_000); + assert_eq!(decoded.deployment_id, "deploy-456"); + let credentials = decoded.credentials.expect("credentials must decode"); + assert_eq!(credentials.access_key, "access"); + assert_eq!(credentials.secret_key, ""); + assert_eq!(credentials.session_token, "session-token"); + } + #[test] fn remote_target_admin_json_latency_round_trips_through_go_duration() { // Round trip: a madmin reader decodes the latency values as Go From 31933c32f9e925f73c4654e3268f5cb594cdf2b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Sun, 23 Aug 2026 20:17:50 +0800 Subject: [PATCH 26/41] fix(replication): apply receiver-side LWW to inbound metadata categories (#6379) --- crates/e2e_test/src/lib.rs | 5 + .../src/replication_extension_test.rs | 422 ++++++++++++++- .../src/replication_lww_receiver_test.rs | 155 ++++++ .../ecstore/src/bucket/bucket_target_sys.rs | 43 +- .../bucket/replication/replication_pool.rs | 24 + .../replication/replication_resyncer.rs | 140 ++++- .../replication_target_boundary.rs | 32 +- crates/ecstore/src/set_disk/ops/multipart.rs | 177 ++++++ crates/ecstore/src/set_disk/ops/object.rs | 509 ++++++++++++++++++ crates/replication/src/operation.rs | 18 + crates/utils/src/http/headers.rs | 2 + rustfs/src/storage/options.rs | 9 +- 12 files changed, 1523 insertions(+), 13 deletions(-) create mode 100644 crates/e2e_test/src/replication_lww_receiver_test.rs diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index b3c1a175a..28c63b267 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -61,6 +61,11 @@ mod get_codec_streaming_compat_test; #[cfg(test)] mod version_id_regression_test; +// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound +// replication metadata must not overwrite a newer local category state. +#[cfg(test)] +mod replication_lww_receiver_test; + // Data usage regression tests #[cfg(test)] mod data_usage_test; diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index 3d51d139b..4d7b641e4 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -57,7 +57,7 @@ use rustfs_madmin::{ AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus, ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus, }; -use s3s::header::X_AMZ_REPLICATION_STATUS; +use s3s::header::{X_AMZ_REPLICATION_STATUS, X_AMZ_TAGGING}; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::convert::Infallible; @@ -2023,6 +2023,7 @@ async fn forward_replication_proxy_request( client: &reqwest::Client, request_count: &AtomicU64, mut replication_enabled: watch::Receiver, + mut held_tagging: watch::Receiver>, ) -> Response> { let (parts, body) = request.into_parts(); let is_replication = parts @@ -2036,6 +2037,17 @@ async fn forward_replication_proxy_request( return proxy_error_response("replication gate closed"); } } + // Content-keyed hold: park only the replication request whose + // `x-amz-tagging` matches the held value, letting every other delivery + // through, so a test can make one specific (stale) delivery the last + // write the backend sees. + if let Some(tagging) = parts.headers.get(X_AMZ_TAGGING).and_then(|value| value.to_str().ok()) { + while held_tagging.borrow().as_deref() == Some(tagging) { + if held_tagging.changed().await.is_err() { + return proxy_error_response("replication tag hold closed"); + } + } + } } let Some(path_and_query) = parts.uri.path_and_query() else { @@ -2070,12 +2082,26 @@ async fn start_replication_counting_proxy( backend_url: &str, tasks: &mut JoinSet<()>, ) -> Result<(String, Arc, watch::Sender), Box> { + let (proxy_url, request_count, replication_enabled, _held_tagging) = + start_replication_counting_proxy_with_tag_hold(backend_url, tasks).await?; + Ok((proxy_url, request_count, replication_enabled)) +} + +/// [`start_replication_counting_proxy`] plus a content-keyed hold: while the +/// returned `watch::Sender>` holds `Some(tagging)`, replication +/// requests whose `x-amz-tagging` equals `tagging` are parked (and still +/// counted); all other traffic flows. Send `None` to release them. +async fn start_replication_counting_proxy_with_tag_hold( + backend_url: &str, + tasks: &mut JoinSet<()>, +) -> Result<(String, Arc, watch::Sender, watch::Sender>), Box> { let listener = TcpListener::bind("127.0.0.1:0").await?; let proxy_url = format!("http://{}", listener.local_addr()?); let backend_url = backend_url.to_string(); let request_count = Arc::new(AtomicU64::new(0)); let task_request_count = request_count.clone(); let (replication_enabled, task_replication_enabled) = watch::channel(true); + let (held_tagging, task_held_tagging) = watch::channel(None); tasks.spawn(async move { let client = local_http_client(); let mut connections = JoinSet::new(); @@ -2087,12 +2113,14 @@ async fn start_replication_counting_proxy( let client = client.clone(); let request_count = task_request_count.clone(); let replication_enabled = task_replication_enabled.clone(); + let held_tagging = task_held_tagging.clone(); connections.spawn(async move { let service = service_fn(move |request| { let backend_url = backend_url.clone(); let client = client.clone(); let request_count = request_count.clone(); let replication_enabled = replication_enabled.clone(); + let held_tagging = held_tagging.clone(); async move { Ok::<_, Infallible>( forward_replication_proxy_request( @@ -2101,6 +2129,7 @@ async fn start_replication_counting_proxy( &client, &request_count, replication_enabled, + held_tagging, ) .await, ) @@ -2113,7 +2142,7 @@ async fn start_replication_counting_proxy( } } }); - Ok((proxy_url, request_count, replication_enabled)) + Ok((proxy_url, request_count, replication_enabled, held_tagging)) } async fn site_replication_remove( @@ -6949,6 +6978,395 @@ async fn test_site_replication_active_active_converges_without_loops_real_dual_n } } +/// Replication status a site reports for one object version via HEAD +/// (`x-amz-replication-status`), or `None` when the header is absent. +async fn head_replication_status( + client: &Client, + bucket: &str, + key: &str, + version_id: &str, +) -> Result, Box> { + let head = client + .head_object() + .bucket(bucket) + .key(key) + .version_id(version_id) + .send() + .await?; + Ok(head.replication_status().map(|status| status.as_str().to_string())) +} + +/// Poll one site until the version's replication status is one of `expected`. +async fn wait_for_version_replication_status( + client: &Client, + bucket: &str, + key: &str, + version_id: &str, + expected: &[&str], + site: &str, +) -> Result> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + loop { + let last = head_replication_status(client, bucket, key, version_id).await?; + if let Some(status) = last.as_deref() + && expected.contains(&status) + { + return Ok(status.to_string()); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "{site}: {bucket}/{key}?versionId={version_id} replication status {last:?} never reached {expected:?}" + ) + .into()); + } + sleep(Duration::from_millis(200)).await; + } +} + +async fn put_single_tag( + client: &Client, + bucket: &str, + key: &str, + version_id: &str, + tag_key: &str, + tag_value: &str, +) -> Result<(), Box> { + client + .put_object_tagging() + .bucket(bucket) + .key(key) + .version_id(version_id) + .tagging( + aws_sdk_s3::types::Tagging::builder() + .tag_set(aws_sdk_s3::types::Tag::builder().key(tag_key).value(tag_value).build()?) + .build()?, + ) + .send() + .await?; + Ok(()) +} + +async fn get_single_tag( + client: &Client, + bucket: &str, + key: &str, + version_id: &str, + tag_key: &str, +) -> Result, Box> { + let tagging = client + .get_object_tagging() + .bucket(bucket) + .key(key) + .version_id(version_id) + .send() + .await?; + Ok(tagging + .tag_set() + .iter() + .find(|tag| tag.key() == tag_key) + .map(|tag| tag.value().to_string())) +} + +/// Poll one site until the version's `tag_key` equals `expected`. +async fn wait_for_single_tag( + client: &Client, + bucket: &str, + key: &str, + version_id: &str, + tag_key: &str, + expected: &str, + site: &str, +) -> Result<(), Box> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + loop { + let observed = get_single_tag(client, bucket, key, version_id, tag_key).await?; + if observed.as_deref() == Some(expected) { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "{site}: {bucket}/{key}?versionId={version_id} tag {tag_key}={observed:?} never became {expected}" + ) + .into()); + } + sleep(Duration::from_millis(200)).await; + } +} + +/// Tag key the dual-node LWW scenario edits on both sites. +const LWW_TAG_KEY: &str = "owner"; + +/// Assert the version's [`LWW_TAG_KEY`] stays `expected` on both sites for a +/// full quiet window (no late stale delivery flips it back). +async fn assert_tag_stable_on_both_sites( + site_a_client: &Client, + site_b_client: &Client, + bucket: &str, + key: &str, + version_id: &str, + expected: &str, + quiet: Duration, +) -> Result<(), Box> { + let deadline = tokio::time::Instant::now() + quiet; + loop { + let on_a = get_single_tag(site_a_client, bucket, key, version_id, LWW_TAG_KEY).await?; + let on_b = get_single_tag(site_b_client, bucket, key, version_id, LWW_TAG_KEY).await?; + assert_eq!(on_a.as_deref(), Some(expected), "site A tag {LWW_TAG_KEY} regressed from the LWW winner"); + assert_eq!(on_b.as_deref(), Some(expected), "site B tag {LWW_TAG_KEY} regressed from the LWW winner"); + if tokio::time::Instant::now() >= deadline { + return Ok(()); + } + sleep(Duration::from_millis(250)).await; + } +} + +/// Wait until the counting proxy in front of a site has admitted `expected` +/// replication requests in total (requests held by a closed gate still count). +async fn wait_for_proxy_replication_requests( + counter: &AtomicU64, + expected: u64, + site: &str, +) -> Result<(), Box> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + loop { + let observed = counter.load(Ordering::Relaxed); + if observed >= expected { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!("{site} proxy saw {observed} replication requests, expected at least {expected}").into()); + } + sleep(Duration::from_millis(25)).await; + } +} + +/// rustfs/backlog#1953 (audit A4/P1-6): receiver-side LWW for replicated +/// metadata categories, exercised end to end over the real dual-node +/// active-active site-replication control plane — sender, worker, status +/// bookkeeping and persisted failure recovery all participate (the single-server +/// `replication_lww_receiver_test` only injects authorized replication PUTs). +/// +/// Scenario on one versioned object: +/// 1. reciprocal tag edits in real order (A then B) converge both sites on the +/// newer tag and leave the author COMPLETED / the receiver REPLICA; +/// 2. out-of-order delivery: A's edit is held at B's inbound proxy while B +/// authors a newer edit that reaches A first; releasing the stale delivery +/// must NOT roll B back — both sites settle on B's value and stay there +/// through a quiet window, with no FAILED/PENDING status left behind; +/// 3. persisted retry: B is stopped, A's delivery reaches FAILED, A restarts, +/// then B returns and the scanner-replayed edit converges both sites forward. +/// Durable metadata-MRF serialization/reconstruction is covered separately by +/// `metadata_mrf_roundtrip_preserves_tags_and_admitted_targets`. +#[tokio::test] +async fn test_site_replication_tagging_lww_converges_active_active_real_dual_node() -> TestResult { + init_logging(); + + match tokio::time::timeout(Duration::from_secs(420), async { + // The scanner is fast for the final persisted-failure recovery phase. + // Step 2 finishes and proves a quiet stable winner before that phase, + // so a later scanner pass cannot mask its stale-delivery assertion. + let mut site_env = replication_fast_env(); + site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + site_env.extend_from_slice(FAST_SCANNER_ENV); + + let mut site_a_env = RustFSTestEnvironment::new().await?; + site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?; + + let mut site_b_env = RustFSTestEnvironment::new().await?; + site_b_env.start_rustfs_server_with_env(vec![], &site_env).await?; + + let mut proxy_tasks = JoinSet::new(); + let (site_a_proxy, site_a_replication_requests, _site_a_replication_enabled, site_a_held_tagging) = + start_replication_counting_proxy_with_tag_hold(&site_a_env.url, &mut proxy_tasks).await?; + let (site_b_proxy, site_b_replication_requests, _site_b_replication_enabled, site_b_held_tagging) = + start_replication_counting_proxy_with_tag_hold(&site_b_env.url, &mut proxy_tasks).await?; + + let site_a_client = site_a_env.create_s3_client(); + let site_b_client = site_b_env.create_s3_client(); + let bucket = "site-repl-tag-lww"; + let key = "lww.txt"; + + let add_status = site_replication_add( + &site_a_env, + &[ + PeerSite { + name: "lww-site-a".to_string(), + endpoint: site_a_env.url.clone(), + access_key: site_a_env.access_key.clone(), + secret_key: site_a_env.secret_key.clone(), + ..Default::default() + }, + PeerSite { + name: "lww-site-b".to_string(), + endpoint: site_b_env.url.clone(), + access_key: site_b_env.access_key.clone(), + secret_key: site_b_env.secret_key.clone(), + ..Default::default() + }, + ], + ) + .await?; + assert!(add_status.success, "unexpected site add result: {add_status:?}"); + + let site_info = wait_for_site_replication_enabled(&site_a_env, 2).await?; + wait_for_site_replication_enabled(&site_b_env, 2).await?; + + // Route both directions through the counting proxies so inbound + // replication to B can be held (out-of-order delivery) and observed. + for (env_url, proxy_url, label) in [(&site_a_env.url, &site_a_proxy, "A"), (&site_b_env.url, &site_b_proxy, "B")] { + let mut peer = site_info + .sites + .iter() + .find(|peer| peer.endpoint == *env_url) + .ok_or_else(|| format!("site {label} peer missing from replication info"))? + .clone(); + peer.endpoint = proxy_url.clone(); + peer.sync_state = SyncStatus::Enable; + let edit = site_replication_edit(&site_a_env, "", &peer).await?; + assert!(edit.success, "unexpected site {label} endpoint edit: {edit:?}"); + } + for env in [&site_a_env, &site_b_env] { + wait_for_site_replication_info(env, |info| { + info.sites.iter().any(|peer| peer.endpoint == site_a_proxy) + && info.sites.iter().any(|peer| peer.endpoint == site_b_proxy) + }) + .await?; + } + + site_a_client.create_bucket().bucket(bucket).send().await?; + wait_for_bucket_on_target(&site_b_client, bucket).await?; + + let version_id = site_a_client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"tag lww payload")) + .send() + .await? + .version_id() + .ok_or("site A PUT omitted version ID")? + .to_string(); + wait_for_replicated_object(&site_b_client, bucket, key, "tag lww payload").await?; + wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?; + wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?; + wait_for_proxy_replication_requests(&site_b_replication_requests, 1, "site B").await?; + + // --- 1. reciprocal edits in real order: A then B ---------------------- + put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a1").await?; + wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a1", "site B").await?; + wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?; + wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?; + wait_for_proxy_replication_requests(&site_b_replication_requests, 2, "site B").await?; + + put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b1").await?; + wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b1", "site A").await?; + wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED"], "site B").await?; + wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["REPLICA"], "site A").await?; + assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b1", Duration::from_secs(3)) + .await?; + + // --- 2. concurrent edits, stale delivery last ------------------------ + // Both sites edit the same version while each other's delivery is + // parked at the peer's inbound proxy (content-keyed: only the + // `owner=a2` / `owner=b2` replication PUTs wait, everything else + // flows). B's edit is the newer one. Releasing A's stale `a2` first + // makes it the last write B sees while A itself still holds `a2`, so + // nothing A could re-deliver carries the winner: only receiver-side + // LWW on B can keep `b2`. Releasing `b2` afterwards converges A. + site_b_held_tagging.send(Some("owner=a2".to_string()))?; + site_a_held_tagging.send(Some("owner=b2".to_string()))?; + let a2_parked_at = site_b_replication_requests.load(Ordering::Relaxed) + 1; + let b2_parked_at = site_a_replication_requests.load(Ordering::Relaxed) + 1; + put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a2").await?; + wait_for_proxy_replication_requests(&site_b_replication_requests, a2_parked_at, "site B").await?; + sleep(Duration::from_millis(50)).await; + put_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "b2").await?; + wait_for_proxy_replication_requests(&site_a_replication_requests, b2_parked_at, "site A").await?; + assert_eq!( + get_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY) + .await? + .as_deref(), + Some("a2") + ); + assert_eq!( + get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY) + .await? + .as_deref(), + Some("b2") + ); + + // Release the stale a2 delivery onto B: the newer local b2 must + // survive, and the delivery itself must still succeed (A reaches + // COMPLETED instead of looping through MRF with the stale value). + site_b_held_tagging.send(None)?; + wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?; + let stale_deadline = tokio::time::Instant::now() + Duration::from_secs(3); + loop { + assert_eq!( + get_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY) + .await? + .as_deref(), + Some("b2"), + "a stale inbound delivery rolled back site B's newer tag (receiver-side LWW regression)" + ); + if tokio::time::Instant::now() >= stale_deadline { + break; + } + sleep(Duration::from_millis(250)).await; + } + + // Release b2 onto A: the newer edit wins there and both sites settle. + // B's own version may legitimately read REPLICA here: the stale inbound + // a2 write re-labelled it as a replica write (keeping B's tags); what + // must not remain is PENDING/FAILED. + site_a_held_tagging.send(None)?; + wait_for_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "b2", "site A").await?; + wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["COMPLETED", "REPLICA"], "site B") + .await?; + assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "b2", Duration::from_secs(4)) + .await?; + for (client, site) in [(&site_a_client, "site A"), (&site_b_client, "site B")] { + let status = head_replication_status(client, bucket, key, &version_id).await?; + assert!( + matches!(status.as_deref(), Some("COMPLETED" | "REPLICA")), + "{site} must not be left PENDING/FAILED after the concurrent edits: {status:?}" + ); + } + + // --- 3. persisted FAILED state survives a source restart ------------ + site_b_env.stop_server(); + put_single_tag(&site_a_client, bucket, key, &version_id, LWW_TAG_KEY, "a3").await?; + wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["FAILED"], "site A").await?; + site_a_env.restart_server_preserving_data(vec![], &site_env).await?; + wait_for_site_replication_enabled(&site_a_env, 2).await?; + site_b_env.restart_server_preserving_data(vec![], &site_env).await?; + wait_for_site_replication_enabled(&site_b_env, 2).await?; + + wait_for_single_tag(&site_b_client, bucket, key, &version_id, LWW_TAG_KEY, "a3", "site B").await?; + wait_for_version_replication_status(&site_a_client, bucket, key, &version_id, &["COMPLETED"], "site A").await?; + wait_for_version_replication_status(&site_b_client, bucket, key, &version_id, &["REPLICA"], "site B").await?; + assert_tag_stable_on_both_sites(&site_a_client, &site_b_client, bucket, key, &version_id, "a3", Duration::from_secs(3)) + .await?; + + // The object itself never forked: one version on each side. + tokio::time::timeout( + Duration::from_secs(70), + assert_replication_converged(&site_a_client, bucket, &site_b_client, bucket), + ) + .await??; + let state = list_replication_state(&site_a_client, bucket).await?; + assert_eq!(state.len(), 1, "tag edits must not create new object versions: {state:?}"); + assert_eq!(state[0].version_id, version_id); + + proxy_tasks.abort_all(); + Ok(()) + }) + .await + { + Ok(result) => result, + Err(_) => Err("site replication tagging LWW test timed out".into()), + } +} #[tokio::test] async fn test_site_replication_replicates_policy_backed_user_access_real_dual_node() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/replication_lww_receiver_test.rs b/crates/e2e_test/src/replication_lww_receiver_test.rs new file mode 100644 index 000000000..4c856213c --- /dev/null +++ b/crates/e2e_test/src/replication_lww_receiver_test.rs @@ -0,0 +1,155 @@ +#![cfg(test)] +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit +//! A4/P1-6). +//! +//! In an active-active topology both sites' metadata states arrive at the +//! peer as authorized replication PUTs carrying per-category source +//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header +//! family). Before the fix the receiver applied them unconditionally, so a +//! stale delivery overwrote a newer local state and the two sites diverged +//! permanently while both reported COMPLETED. This test drives one live +//! `rustfs` server with simulated inbound replication PUTs for the same +//! object version and asserts the newer tagging state wins regardless of +//! delivery order, while a stale delivery still succeeds at the object level +//! (a failure would loop through MRF re-delivering the stale value). +//! +//! The real dual-site path (sender, worker, status bookkeeping, MRF replay) +//! is covered by +//! `replication_extension_test::test_site_replication_tagging_lww_converges_active_active_real_dual_node`; +//! this file stays as the fast, single-process receiver check. + +use crate::common::{RustFSTestEnvironment, init_logging}; +use aws_sdk_s3::Client; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; + +type TestResult = Result<(), Box>; + +const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request"; +const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id"; +const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime"; +const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp"; + +const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z"; +const T_STALE: &str = "2026-01-01T00:00:01Z"; +const T_LOCAL: &str = "2026-02-01T00:00:00Z"; +const T_NEWER: &str = "2026-03-01T00:00:00Z"; + +/// Simulated inbound authorized replication PUT: same object version, tags and +/// the source-authored tagging timestamp carried in transport headers. +async fn inbound_replication_put( + client: &Client, + bucket: &str, + key: &str, + version_id: &str, + tags: &str, + tagging_timestamp: &str, +) -> TestResult { + let version_id = version_id.to_string(); + let tagging_timestamp = tagging_timestamp.to_string(); + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from_static(b"lww-e2e-body")) + .tagging(tags) + .customize() + .mutate_request(move |req| { + req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true"); + req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone()); + req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME); + req.headers_mut() + .insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone()); + }) + .send() + .await?; + Ok(()) +} + +async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option { + let tagging = client + .get_object_tagging() + .bucket(bucket) + .key(key) + .version_id(version_id) + .send() + .await + .expect("object tagging should be readable"); + tagging + .tag_set() + .iter() + .find(|tag| tag.key() == tag_key) + .map(|tag| tag.value().to_string()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server(vec![]).await?; + let client = env.create_s3_client(); + + let bucket = "replication-lww-receiver"; + let key = "object"; + client.create_bucket().bucket(bucket).send().await?; + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + + // First delivery establishes version V with tags stamped T_LOCAL. + let version_id = uuid::Uuid::new_v4().to_string(); + inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?; + assert_eq!( + tag_value(&client, bucket, key, &version_id, "site").await.as_deref(), + Some("local"), + "the first delivery must establish the tagged version" + ); + + // A stale delivery (older source timestamp) must succeed at the object + // level but must NOT overwrite the newer tags. + inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?; + assert_eq!( + tag_value(&client, bucket, key, &version_id, "site").await.as_deref(), + Some("local"), + "a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)" + ); + + // A newer delivery still converges the version onto the newest state. + inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?; + assert_eq!( + tag_value(&client, bucket, key, &version_id, "site").await.as_deref(), + Some("newer"), + "a newer inbound delivery must overwrite older tags" + ); + + client + .delete_object() + .bucket(bucket) + .key(key) + .version_id(&version_id) + .send() + .await?; + env.delete_test_bucket(bucket).await.ok(); + Ok(()) +} diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 7e5a5df5f..4c4e66a79 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -58,8 +58,8 @@ use rustfs_config::{DEFAULT_TRUST_LEAF_CERT_AS_CA, ENV_TRUST_LEAF_CERT_AS_CA, RU use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url}; use rustfs_utils::http::{ AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, - AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header, - is_rustfs_header, is_standard_header, is_storageclass_header, + AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, + is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header, }; use rustfs_utils::http::{ SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST, @@ -1774,6 +1774,22 @@ impl PutObjectOptions { Self::insert_checked(&mut header, AMZ_BUCKET_REPLICATION_STATUS, self.internal.replication_status.as_str()); } + // MinIO PutObjectOptions.Header parity: object tags travel on the + // `x-amz-tagging` header (form-urlencoded). `replication_put_object_options` + // fills `user_tags` from the source version; without this header the + // whole-object transport delivered a tagless replica, so tag edits + // never reached the peer and the receiver-side LWW comparison + // (rustfs/backlog#1953) had nothing to judge. + if !self.user_tags.is_empty() { + let mut tags: Vec<(&String, &String)> = self.user_tags.iter().collect(); + tags.sort(); + let mut encoded = url::form_urlencoded::Serializer::new(String::new()); + for (key, value) in tags { + encoded.append_pair(key, value); + } + Self::insert_checked(&mut header, AMZ_OBJECT_TAGGING_LOWER, &encoded.finish()); + } + for (k, v) in &self.user_metadata { let Ok(header_value) = HeaderValue::from_str(v) else { warn!("skipping user metadata header with invalid value: {}", k); @@ -3195,6 +3211,29 @@ mod tests { } } + #[test] + fn put_object_headers_carry_user_tags_on_x_amz_tagging() { + // rustfs/backlog#1953: tag edits replicate through the whole-object + // transport, so the source tags must travel on x-amz-tagging. + let mut opts = PutObjectOptions::default(); + opts.user_tags.insert("owner".to_string(), "site a".to_string()); + opts.user_tags.insert("env".to_string(), "prod".to_string()); + + let header = opts.header(); + let tagging = header + .get(AMZ_OBJECT_TAGGING_LOWER) + .expect("user tags must be transported on x-amz-tagging") + .to_str() + .expect("tag header must be ASCII"); + // Deterministic key order; values are form-urlencoded. + assert_eq!(tagging, "env=prod&owner=site+a"); + + assert!( + PutObjectOptions::default().header().get(AMZ_OBJECT_TAGGING_LOWER).is_none(), + "a tagless source must not send an empty x-amz-tagging header" + ); + } + #[test] fn put_object_headers_omit_unset_replication_timestamps() { // UNIX_EPOCH means "never modified on the source"; sending it would diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index 61dbf2f1e..42934318a 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -4194,6 +4194,30 @@ mod tests { assert_eq!(ri.checksum, Some(checksum)); } + #[test] + fn metadata_mrf_roundtrip_preserves_tags_and_admitted_targets() { + let target = "arn:rustfs:replication:target-a"; + let object = ObjectInfo { + bucket: "source".to_string(), + name: "object".to_string(), + version_id: Some(Uuid::new_v4()), + user_tags: Arc::new("owner=a3".to_string()), + ..Default::default() + }; + let live = + replicate_object_info_from_object_info(object.clone(), test_replicate_decision(&[target]), ReplicationType::Metadata); + let persisted = live.to_mrf_entry(); + let encoded = encode_mrf_file(std::slice::from_ref(&persisted)).expect("metadata MRF entry should encode"); + let decoded = decode_mrf_file(&encoded).expect("metadata MRF entry should decode"); + + assert_eq!(decoded[0].op, MrfOpKind::Metadata); + assert_eq!(decoded[0].target_arns, vec![target.to_string()]); + let replayed = admitted_mrf_replicate_object(object, &decoded[0], ReplicationType::Metadata); + assert_eq!(replayed.op_type, ReplicationType::Metadata); + assert_eq!(replayed.user_tags, "owner=a3"); + assert_eq!(replayed.admitted_target_arns(), vec![target.to_string()]); + } + #[tokio::test] async fn mrf_save_admission_waits_for_capacity_instead_of_dropping() { let (tx, mut rx) = mpsc::channel(1); diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index e670a7d6f..587be7ae3 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -76,7 +76,8 @@ use metrics::counter; use rmp_serde; use rustfs_s3_types::EventName; use rustfs_utils::http::{ - AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, has_internal_suffix, insert_str, + AMZ_BUCKET_REPLICATION_STATUS, AMZ_TAGGING_DIRECTIVE, SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, + has_internal_suffix, insert_str, }; use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash}; #[cfg(test)] @@ -174,6 +175,14 @@ fn has_raw_status(err: &SdkError, status: u16) -> bool { err.raw_response().is_some_and(|r| r.status().as_u16() == status) } +fn metadata_requires_existing_target(op_type: ReplicationType, object_info: &ObjectInfo) -> bool { + op_type == ReplicationType::Metadata + && object_info + .user_defined + .get(AMZ_BUCKET_REPLICATION_STATUS) + .is_some_and(|status| status.eq_ignore_ascii_case(ReplicationStatusType::Replica.as_str())) +} + const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total"; /// Targets that already produced a version-identity-drift warning this @@ -3494,6 +3503,7 @@ async fn resolve_replicate_all_action( start_time, ssec_audit_required, } = ctx; + let require_existing_target = metadata_requires_existing_target(roi.op_type, &object_info); let replication_action; match head_object_for_worker(tgt_client.as_ref(), &tgt_client.bucket, object, roi.version_id.map(|v| v.to_string())).await { Ok(oi) => { @@ -3555,7 +3565,13 @@ async fn resolve_replicate_all_action( // Version-ID format mismatch: retry without versionId and compare ETags. match head_object_fallback(tgt_client, object).await { Ok(Some(oi)) => { - replication_action = if replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()) { + let etags_match = replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref()); + if require_existing_target && !etags_match { + rinfo.error = Some("replica metadata target does not contain matching object data".to_string()); + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + return None; + } + replication_action = if etags_match { if ssec_audit_required && !settle_ssec_passthrough_evidence(&oi, tgt_client, bucket, object, rinfo).await { @@ -3568,6 +3584,11 @@ async fn resolve_replicate_all_action( }; } Ok(None) => { + if require_existing_target { + rinfo.error = Some("replica metadata target does not contain this object version".to_string()); + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + return None; + } replication_action = ReplicationAction::All; } Err(e2) => { @@ -3593,7 +3614,12 @@ async fn resolve_replicate_all_action( return None; } } - } else if e.as_service_error().is_some_and(|se| se.is_not_found()) { + } else if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) { + if require_existing_target { + rinfo.error = Some("replica metadata target does not contain this object version".to_string()); + rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs(); + return None; + } replication_action = ReplicationAction::All; } else { rinfo.error = Some(e.to_string()); @@ -3868,6 +3894,7 @@ async fn replicate_object_with_multipart(ctx: MultipartR actual_size, object_info.etag.clone().unwrap_or_default(), object_info.mod_time, + &put_opts.internal, ), ) .await @@ -3921,6 +3948,113 @@ mod tests { }) } + fn spawn_head_status_server(status: u16) -> (String, std::thread::JoinHandle<()>) { + use std::io::{Read, Write}; + + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind"); + let endpoint = format!("http://{}", listener.local_addr().expect("test HTTP listener should have an address")); + let handle = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("test HTTP client should connect"); + let mut request = [0_u8; 8192]; + let bytes_read = stream.read(&mut request).expect("test HTTP request should be read"); + assert!(bytes_read > 0, "test HTTP request should not be empty"); + assert!(request[..bytes_read].starts_with(b"HEAD "), "replication comparison must use HEAD"); + write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .expect("test HTTP response should be written"); + }); + (endpoint, handle) + } + + #[tokio::test] + async fn replica_metadata_missing_target_stops_before_full_put() { + let (endpoint, server) = spawn_head_status_server(404); + let target = test_target_client(endpoint); + let roi = ReplicateObjectInfo { + bucket: "source".to_string(), + name: "object".to_string(), + version_id: Some(Uuid::new_v4()), + op_type: ReplicationType::Metadata, + // Normal metadata writes replace REPLICA with per-target PENDING + // before constructing the worker request. + replication_status: ReplicationStatusType::Pending, + ..Default::default() + }; + let object_info = ObjectInfo { + bucket: roi.bucket.clone(), + name: roi.name.clone(), + version_id: roi.version_id, + etag: Some("source-etag".to_string()), + user_defined: Arc::new(HashMap::from([( + AMZ_BUCKET_REPLICATION_STATUS.to_string(), + ReplicationStatusType::Replica.as_str().to_string(), + )])), + ..Default::default() + }; + let mut rinfo = replicate_all_target_info(&roi, &target); + + let action = resolve_replicate_all_action( + ReplicateAllActionContext { + roi: &roi, + tgt_client: &target, + bucket: &roi.bucket, + object: &roi.name, + start_time: OffsetDateTime::now_utc(), + ssec_audit_required: false, + }, + object_info, + &mut rinfo, + ) + .await; + + assert!(action.is_none(), "missing replica metadata targets must not reach the payload PUT path"); + assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed); + assert_eq!( + rinfo.error.as_deref(), + Some("replica metadata target does not contain this object version") + ); + server.join().expect("test HTTP server should finish"); + } + + #[tokio::test] + async fn source_metadata_missing_target_rebuilds_object() { + let (endpoint, server) = spawn_head_status_server(404); + let target = test_target_client(endpoint); + let roi = ReplicateObjectInfo { + bucket: "source".to_string(), + name: "object".to_string(), + version_id: Some(Uuid::new_v4()), + op_type: ReplicationType::Metadata, + replication_status: ReplicationStatusType::Pending, + ..Default::default() + }; + let object_info = ObjectInfo { + bucket: roi.bucket.clone(), + name: roi.name.clone(), + version_id: roi.version_id, + etag: Some("source-etag".to_string()), + ..Default::default() + }; + let mut rinfo = replicate_all_target_info(&roi, &target); + + let action = resolve_replicate_all_action( + ReplicateAllActionContext { + roi: &roi, + tgt_client: &target, + bucket: &roi.bucket, + object: &roi.name, + start_time: OffsetDateTime::now_utc(), + ssec_audit_required: false, + }, + object_info, + &mut rinfo, + ) + .await; + + assert!(matches!(action, Some((ReplicationAction::All, _)))); + assert!(rinfo.error.is_none()); + server.join().expect("test HTTP server should finish"); + } + async fn register_test_target(target: &Arc) { ReplicationTargetStore::register_test_target(target).await; } diff --git a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs index 4fe89967d..50aa14cd7 100644 --- a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs @@ -472,6 +472,7 @@ pub(crate) fn replication_complete_multipart_options( actual_size: String, source_etag: String, source_mtime: Option, + source_internal: &AdvancedPutOptions, ) -> PutObjectOptions { let mut user_metadata = HashMap::new(); insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size); @@ -484,6 +485,14 @@ pub(crate) fn replication_complete_multipart_options( // mtime must degrade to epoch so header() suppresses the header // instead of asserting the replication time as the object's mtime. source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH), + // Carry the per-category LWW timestamps on the complete request as + // well: the receiver's CompleteMultipartUpload options builder + // parses the same headers, so the multipart transport gets the + // same receiver-side LWW as the single-PUT transport + // (rustfs/backlog#1953). Epoch values keep the headers suppressed. + tagging_timestamp: source_internal.tagging_timestamp, + retention_timestamp: source_internal.retention_timestamp, + legalhold_timestamp: source_internal.legalhold_timestamp, replication_status: ReplicationStatusType::Replica, replication_request: true, ..Default::default() @@ -663,20 +672,39 @@ mod tests { #[test] fn replication_complete_multipart_options_sets_actual_size() { let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp"); + let source_internal = AdvancedPutOptions { + tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"), + retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"), + legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"), + ..Default::default() + }; let options = replication_complete_multipart_options( "1024".to_string(), "0123456789abcdef0123456789abcdef-3".to_string(), Some(source_mtime), + &source_internal, ); assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3"); assert_eq!(options.internal.source_mtime, source_mtime); + // The complete request must carry the same per-category LWW timestamps + // as the initiate request; the receiver reads them from the complete + // headers (rustfs/backlog#1953). + assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp); + assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp); + assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp); + // Absent source mtime must degrade to epoch (header suppressed), not // the AdvancedPutOptions default of now_utc() — that default would // stamp the replication time as the replica's mtime and break the - // multipart HEAD convergence. - let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None); + // multipart HEAD convergence. Unset category timestamps stay epoch so + // header() keeps suppressing them. + let options_no_mtime = + replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default()); assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0); + assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0); + assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0); + assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0); assert_eq!( get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(), diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 5aa59b158..7e67997c0 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -2318,6 +2318,43 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { fi.set_data_moved(); } + // Receiver-side LWW (rustfs/backlog#1953): the multipart replication + // transport carries the category values at CreateMultipartUpload (in + // the staged upload metadata) and the source category timestamps on + // the complete request. Read the destination version under the held + // object write lock and keep any category this site modified more + // recently. Only an absent version has no local state to compare; + // other read failures must leave the upload retryable rather than + // committing inbound metadata without the LWW check. + if crate::set_disk::ops::object::replication_lww_applicable(opts) + && let Some(version_id) = fi.version_id + { + match self + .get_object_info( + bucket, + object, + &ObjectOptions { + version_id: Some(version_id.to_string()), + no_lock: true, + metadata_cache_safe: false, + versioned: opts.versioned, + version_suspended: opts.version_suspended, + ..Default::default() + }, + ) + .await + { + Ok(existing) => { + let stored = crate::set_disk::ops::object::stored_replication_category_metadata(&existing); + crate::set_disk::ops::object::merge_replication_metadata_lww(&mut fi.metadata, &stored, opts); + } + // Version absent: first replication of this version, nothing + // local to compare — the normal path, not a degraded one. + Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} + Err(err) => return Err(err), + } + } + for meta in parts_metadatas.iter_mut() { if meta.has_valid_erasure_geometry() { meta.size = fi.size; @@ -7055,6 +7092,146 @@ mod tests { .await } + /// Receiver-side LWW on the multipart replication transport + /// (rustfs/backlog#1953): a metadata-only replication of a multipart + /// source object rides CreateMultipartUpload (category values in the + /// upload metadata) + CompleteMultipartUpload (category timestamps in + /// the complete options). A stale inbound tagging timestamp must not + /// overwrite a newer locally-tagged destination version. + #[tokio::test] + #[serial] + async fn complete_multipart_upload_stale_replication_tags_keep_local() { + use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING; + use rustfs_utils::http::{SUFFIX_TAGGING_TIMESTAMP, get_str}; + use time::format_description::well_known::Rfc3339; + + const T_OLD: &str = "2026-01-01T00:00:00Z"; + const T_LOCAL: &str = "2026-02-01T00:00:00Z"; + + let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "multipart-replication-lww-bucket"; + let object = "object"; + make_bucket_on_all(&disk_stores, bucket).await; + + // Local destination version with newer tags. + let version_id = Uuid::new_v4(); + let mut local_metadata = HashMap::new(); + local_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()); + rustfs_utils::http::insert_str(&mut local_metadata, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string()); + let mut local_reader = PutObjReader::from_vec(b"local body".to_vec()); + set_disks + .put_object( + bucket, + object, + &mut local_reader, + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + user_defined: local_metadata, + // Explicit-version PUTs require the bucket Object Lock snapshot. + object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new( + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + }, + ) + .await + .expect("local versioned put should commit"); + + // Inbound replication upload carrying older tags for the same version. + let mut inbound_metadata = HashMap::new(); + inbound_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string()); + rustfs_utils::http::insert_str(&mut inbound_metadata, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string()); + let create_opts = ObjectOptions { + versioned: true, + user_defined: inbound_metadata, + ..Default::default() + }; + let (upload_id, parts) = + stage_upload_with_create_opts(&set_disks, bucket, object, &payload(0x5a), &create_opts).await; + rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(version_id)).await; + + let complete_opts = ObjectOptions { + versioned: true, + replication_request: true, + replication_tagging_timestamp: Some(OffsetDateTime::parse(T_OLD, &Rfc3339).expect("test timestamp should parse")), + ..Default::default() + }; + + // Make the destination version unreadable on quorum while the + // staged upload remains intact. The commit barrier lets the old + // fail-open path move past the LWW read; restoring the metadata + // there proves it would otherwise commit the stale tags. + let mut damaged_metadata = Vec::new(); + for temp_dir in temp_dirs.iter().take(3) { + let path = temp_dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE); + let original = tokio::fs::read(&path).await.expect("destination xl.meta should be readable"); + tokio::fs::write(&path, b"not an xl.meta") + .await + .expect("destination xl.meta should be corruptible"); + damaged_metadata.push((path, original)); + } + + let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::BeforeLockLost); + let first_set = set_disks.clone(); + let first_upload_id = upload_id.clone(); + let first_parts = parts.clone(); + let first_opts = complete_opts.clone(); + let mut first_completion = tokio::spawn(async move { + first_set + .complete_multipart_upload(bucket, object, &first_upload_id, first_parts, &first_opts) + .await + }); + + let first_result = tokio::select! { + result = &mut first_completion => result.expect("first completion task should finish"), + () = barrier.wait_until_paused() => { + for (path, original) in &damaged_metadata { + tokio::fs::write(path, original).await.expect("destination xl.meta should be restorable"); + } + barrier.release(); + first_completion.await.expect("released completion task should finish") + } + }; + for (path, original) in &damaged_metadata { + tokio::fs::write(path, original) + .await + .expect("destination xl.meta should be restored"); + } + drop(barrier); + + let first_error = first_result.expect_err("unreadable destination metadata must fail before multipart commit"); + assert!( + !(is_err_object_not_found(&first_error) || is_err_version_not_found(&first_error)), + "corrupt destination metadata must not be treated as an absent version: {first_error}" + ); + + set_disks + .clone() + .complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts) + .await + .expect("replication multipart completion should succeed even when a category keeps local values"); + + let info = set_disks + .get_object_info( + bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + ..Default::default() + }, + ) + .await + .expect("completed version should be readable"); + assert_eq!( + info.user_tags.as_str(), + "site=local", + "older inbound multipart tags must not overwrite newer local tags" + ); + assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL)); + } + #[tokio::test] #[serial] async fn complete_multipart_upload_assigns_completion_version_id() { diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 86f75eb8d..0730277d4 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -1881,6 +1881,110 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn transported } +/// True when an authorized replication write carries at least one per-category +/// source timestamp, i.e. receiver-side LWW has something to judge. +pub(in crate::set_disk) fn replication_lww_applicable(opts: &ObjectOptions) -> bool { + opts.replication_request + && (opts.replication_tagging_timestamp.is_some() + || opts.replication_retention_timestamp.is_some() + || opts.replication_legalhold_timestamp.is_some()) +} + +/// The stored per-category state of a destination version, as compared by +/// [`merge_replication_metadata_lww`]. `ObjectInfo::from_file_info` +/// externalizes tags into `user_tags` (stripping the metadata key), so the +/// tag value is folded back into map form here. +pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &ObjectInfo) -> HashMap { + let mut stored = (*existing.user_defined).clone(); + if !existing.user_tags.is_empty() { + stored.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(), (*existing.user_tags).clone()); + } + stored +} + +/// Receiver-side last-writer-wins for authorized replication writes +/// (rustfs/backlog#1953, audit A4/P1-6). Metadata-only replication reuses the +/// whole-object transports, so in active-active topologies an inbound write +/// carries the source's tags / retention / legal hold verbatim and would +/// otherwise overwrite a category the destination modified more recently — +/// both sites end up permanently diverged while reporting COMPLETED. +/// +/// Judged per category, only when the inbound request carries that category's +/// source timestamp (`ObjectOptions::replication_*_timestamp`): +/// - stored timestamp newer than inbound: the local category values and +/// timestamp are kept; the rest of the write proceeds per the inbound +/// metadata and the object-level result stays successful (failing the write +/// instead would loop through MRF, re-delivering the stale value forever); +/// - otherwise the inbound category wins and its internal timestamp key is +/// pinned to the source-authored time — the PUT path re-stamps the +/// object-lock timestamps with the receiver's clock +/// (`parse_object_lock_retention` / `parse_object_lock_legal_hold` insert +/// `now()` via `eval_metadata`), which would make the replica's clock the +/// LWW authority and wedge later convergence; +/// - no stored timestamp (pre-P1-6 data) or no inbound timestamp: the current +/// overwrite behavior is preserved. +/// +/// Returns whether `inbound` was modified. Callers must hold the object write +/// lock so the stored values compared here are the ones being replaced. +pub(in crate::set_disk) fn merge_replication_metadata_lww( + inbound: &mut HashMap, + existing: &HashMap, + opts: &ObjectOptions, +) -> bool { + use rustfs_utils::http::headers::{ + AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING, + }; + use rustfs_utils::http::metadata_compat::{ + SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str, + remove_str, + }; + use time::format_description::well_known::Rfc3339; + + let categories: [(Option, &str, &[&str]); 3] = [ + (opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP, &[AMZ_OBJECT_TAGGING]), + ( + opts.replication_retention_timestamp, + SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, + &[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER], + ), + ( + opts.replication_legalhold_timestamp, + SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, + &[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER], + ), + ]; + + let mut changed = false; + for (inbound_timestamp, timestamp_suffix, value_keys) in categories { + let Some(inbound_timestamp) = inbound_timestamp else { continue }; + let is_category_value_key = |key: &str| value_keys.iter().any(|value_key| key.eq_ignore_ascii_case(value_key)); + let stored_timestamp = get_str(existing, timestamp_suffix).and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok()); + if stored_timestamp.is_some_and(|stored| stored > inbound_timestamp) { + inbound.retain(|key, _| !is_category_value_key(key)); + remove_str(inbound, timestamp_suffix); + for (key, value) in existing { + if is_category_value_key(key) { + inbound.insert(key.clone(), value.clone()); + } + } + // Restore the winning timestamp via insert_str, not a verbatim key + // copy: a MinIO-written version may carry only the + // x-minio-internal- key, and the dual-key invariant requires every + // write to produce both keys. + if let Some(stored_value) = get_str(existing, timestamp_suffix) { + rustfs_utils::http::insert_str(inbound, timestamp_suffix, stored_value); + } + changed = true; + } else if let Ok(source_authored) = inbound_timestamp.format(&Rfc3339) + && get_str(inbound, timestamp_suffix).as_deref() != Some(source_authored.as_str()) + { + rustfs_utils::http::insert_str(inbound, timestamp_suffix, source_authored); + changed = true; + } + } + changed +} + impl SetDisks { pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts( &self, @@ -2073,6 +2177,14 @@ impl SetDisks { user_defined.insert(key.clone(), value.clone()); } } + if replication_lww_applicable(opts) { + // Object Lock evaluation stamps category timestamps with this + // receiver's clock. Pin them back to the source-authored times + // before the first copy of a version is committed; the existing- + // version branch below may still replace them with newer local + // state. + merge_replication_metadata_lww(&mut user_defined, &HashMap::new(), opts); + } if expected_restore_operation_id.is_some() { rustfs_utils::http::metadata_compat::remove_str(&mut user_defined, SUFFIX_RESTORE_OPERATION_ID); } @@ -2562,6 +2674,22 @@ impl SetDisks { if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() { return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string())); } + // Receiver-side LWW (rustfs/backlog#1953): reuse this + // commit-lock read of the destination version so a + // category (tags / retention / legal hold) modified + // more recently on this site is kept instead of being + // overwritten by the inbound replication metadata. + if replication_lww_applicable(opts) { + let stored = stored_replication_category_metadata(&existing); + let mut merged = parts_metadatas[response_metadata_slot].metadata.clone(); + if merge_replication_metadata_lww(&mut merged, &stored, opts) { + for (pfi, disk) in parts_metadatas.iter_mut().zip(shuffle_disks.iter()) { + if disk.is_some() { + pfi.metadata = merged.clone(); + } + } + } + } } Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {} Err(err) => return Err(err), @@ -8082,6 +8210,387 @@ mod replication_quota_safety_tests { } } +#[cfg(test)] +mod replication_lww_tests { + //! Receiver-side LWW for authorized replication writes (rustfs/backlog#1953, + //! audit A4/P1-6): an inbound replication PUT whose per-category timestamp + //! (tags / retention / legal hold) is older than the destination version's + //! stored timestamp must keep the local category values instead of + //! overwriting them; categories are judged independently and the write + //! itself still succeeds. + + use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; + use super::*; + use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use rustfs_utils::http::headers::{ + AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING, + }; + use rustfs_utils::http::{ + SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str, + insert_str, + }; + use time::format_description::well_known::Rfc3339; + + const T_OLD: &str = "2026-01-01T00:00:00Z"; + const T_LOCAL: &str = "2026-02-01T00:00:00Z"; + const T_NEW: &str = "2026-03-01T00:00:00Z"; + + fn parse_ts(value: &str) -> OffsetDateTime { + OffsetDateTime::parse(value, &Rfc3339).expect("test timestamp should parse") + } + + async fn make_bucket(disks: &[DiskStore], bucket: &str) { + for disk in disks { + disk.make_volume(bucket).await.expect("bucket volume should be created"); + } + } + + async fn put_version(set_disks: &Arc, bucket: &str, object: &str, version_id: &str, opts: &ObjectOptions) { + let mut reader = PutObjReader::from_vec(b"lww-body".to_vec()); + set_disks + .put_object(bucket, object, &mut reader, opts) + .await + .expect("versioned put should commit"); + assert_eq!(opts.version_id.as_deref(), Some(version_id)); + } + + fn versioned_opts(version_id: &str, user_defined: HashMap) -> ObjectOptions { + ObjectOptions { + versioned: true, + version_id: Some(version_id.to_string()), + user_defined, + // Explicit-version PUTs require the bucket Object Lock snapshot. + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new( + crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent, + ))), + ..Default::default() + } + } + + /// Local state: version `version_id` with tags "site=local" stamped `T_LOCAL`. + async fn seed_local_tagged_version(set_disks: &Arc, bucket: &str, object: &str, version_id: &str) { + let mut user_defined = HashMap::new(); + user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()); + insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string()); + put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, user_defined)).await; + } + + fn inbound_tagging_opts(version_id: &str, tags: &str, timestamp: &str) -> ObjectOptions { + let mut user_defined = HashMap::new(); + user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), tags.to_string()); + insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, timestamp.to_string()); + ObjectOptions { + replication_request: true, + replication_tagging_timestamp: Some(parse_ts(timestamp)), + ..versioned_opts(version_id, user_defined) + } + } + + async fn version_info(set_disks: &Arc, bucket: &str, object: &str, version_id: &str) -> ObjectInfo { + set_disks + .get_object_info(bucket, object, &versioned_opts(version_id, HashMap::new())) + .await + .expect("version should be readable") + } + + #[tokio::test] + async fn inbound_stale_tagging_keeps_newer_local_tags() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "lww-tagging-stale"; + let object = "object"; + let version_id = Uuid::new_v4().to_string(); + make_bucket(&disk_stores, bucket).await; + seed_local_tagged_version(&set_disks, bucket, object, &version_id).await; + + put_version( + &set_disks, + bucket, + object, + &version_id, + &inbound_tagging_opts(&version_id, "site=remote", T_OLD), + ) + .await; + + let info = version_info(&set_disks, bucket, object, &version_id).await; + assert_eq!( + info.user_tags.as_str(), + "site=local", + "older inbound tags must not overwrite newer local tags" + ); + assert_eq!( + get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), + Some(T_LOCAL), + "the winning local tagging timestamp must be preserved" + ); + } + + #[tokio::test] + async fn inbound_newer_tagging_overwrites_local_tags() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "lww-tagging-newer"; + let object = "object"; + let version_id = Uuid::new_v4().to_string(); + make_bucket(&disk_stores, bucket).await; + seed_local_tagged_version(&set_disks, bucket, object, &version_id).await; + + put_version( + &set_disks, + bucket, + object, + &version_id, + &inbound_tagging_opts(&version_id, "site=remote", T_NEW), + ) + .await; + + let info = version_info(&set_disks, bucket, object, &version_id).await; + assert_eq!( + info.user_tags.as_str(), + "site=remote", + "newer inbound tags must overwrite older local tags" + ); + assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_NEW)); + } + + #[tokio::test] + async fn inbound_wins_when_local_has_no_tagging_timestamp() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "lww-tagging-no-local-ts"; + let object = "object"; + let version_id = Uuid::new_v4().to_string(); + make_bucket(&disk_stores, bucket).await; + // Pre-P1-6 data: local tags without a stored tagging timestamp. + let mut user_defined = HashMap::new(); + user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()); + put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, user_defined)).await; + + put_version( + &set_disks, + bucket, + object, + &version_id, + &inbound_tagging_opts(&version_id, "site=remote", T_OLD), + ) + .await; + + let info = version_info(&set_disks, bucket, object, &version_id).await; + assert_eq!( + info.user_tags.as_str(), + "site=remote", + "without a local timestamp the inbound category must win (pre-LWW data compatibility)" + ); + } + + #[tokio::test] + async fn categories_are_judged_independently() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "lww-category-independent"; + let object = "object"; + let version_id = Uuid::new_v4().to_string(); + make_bucket(&disk_stores, bucket).await; + + // Local: newer tags (T_LOCAL), older *cleared* retention (T_OLD) — + // timestamp key only, the shape a replicated retention clear stores. + // (An active local retention would already block the overwrite at the + // WORM gate; the LWW-reachable retention states are cleared/expired.) + let mut local = HashMap::new(); + local.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()); + insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string()); + insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string()); + put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await; + + // Inbound: older tags (T_OLD), newer retention (T_NEW). + let mut inbound = HashMap::new(); + inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string()); + insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string()); + inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string()); + inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string()); + insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string()); + let opts = ObjectOptions { + replication_request: true, + replication_tagging_timestamp: Some(parse_ts(T_OLD)), + replication_retention_timestamp: Some(parse_ts(T_NEW)), + ..versioned_opts(&version_id, inbound) + }; + put_version(&set_disks, bucket, object, &version_id, &opts).await; + + let info = version_info(&set_disks, bucket, object, &version_id).await; + assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values"); + assert_eq!( + info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), + Some("COMPLIANCE"), + "the newer retention category must be applied in the same write" + ); + assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), Some(T_NEW)); + } + + #[tokio::test] + async fn inbound_stale_legal_hold_keeps_local_value() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "lww-legalhold-stale"; + let object = "object"; + let version_id = Uuid::new_v4().to_string(); + make_bucket(&disk_stores, bucket).await; + + // Local: legal hold released (OFF) at T_LOCAL. (A local hold that is + // still ON already blocks the overwrite at the WORM gate; the + // LWW-reachable divergence is a stale inbound ON resurrecting a hold + // that was released more recently on this site.) + let mut local = HashMap::new(); + local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string()); + insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string()); + put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await; + + let mut inbound = HashMap::new(); + inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string()); + insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string()); + let opts = ObjectOptions { + replication_request: true, + replication_legalhold_timestamp: Some(parse_ts(T_OLD)), + ..versioned_opts(&version_id, inbound) + }; + put_version(&set_disks, bucket, object, &version_id, &opts).await; + + let info = version_info(&set_disks, bucket, object, &version_id).await; + assert_eq!( + info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str), + Some("OFF"), + "a stale inbound legal hold must not resurrect a hold released more recently" + ); + assert_eq!( + get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(), + Some(T_LOCAL) + ); + } + + /// Dual-key invariant under LWW: a MinIO-written destination version may + /// carry only the x-minio-internal timestamp key; when the local category + /// wins, the restored map must still hold BOTH compatibility keys. + #[test] + fn local_win_restores_both_internal_timestamp_keys_for_minio_only_metadata() { + let mut inbound = HashMap::new(); + inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string()); + insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string()); + let existing = HashMap::from([ + (AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()), + ("X-Minio-Internal-Tagging-Timestamp".to_string(), T_LOCAL.to_string()), + ]); + let opts = ObjectOptions { + replication_request: true, + replication_tagging_timestamp: Some(parse_ts(T_OLD)), + ..Default::default() + }; + + assert!(merge_replication_metadata_lww(&mut inbound, &existing, &opts)); + assert_eq!(inbound.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("site=local")); + assert_eq!( + inbound.get("x-rustfs-internal-tagging-timestamp").map(String::as_str), + Some(T_LOCAL), + "the RustFS twin key must be materialized even when the source version only had the MinIO key" + ); + assert_eq!(inbound.get("x-minio-internal-tagging-timestamp").map(String::as_str), Some(T_LOCAL)); + } + + /// When the inbound category wins, the stored timestamp must be the + /// source-authored one: the PUT path's eval_metadata stamps the + /// object-lock timestamps with the receiver's clock + /// (`parse_object_lock_retention`), which would otherwise make this + /// replica's clock the LWW authority and wedge later convergence. + #[tokio::test] + async fn inbound_win_pins_stored_timestamp_to_source_authored_value() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "lww-retention-ts-pinned"; + let object = "object"; + let version_id = Uuid::new_v4().to_string(); + make_bucket(&disk_stores, bucket).await; + + // Local cleared retention at T_OLD. + let mut local = HashMap::new(); + insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string()); + put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await; + + // Inbound newer retention: the source authored T_LOCAL, but the PUT + // path's eval_metadata stomped the metadata key with receiver-now + // (simulated by T_NEW here). + let mut inbound = HashMap::new(); + inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string()); + inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string()); + insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string()); + let opts = ObjectOptions { + replication_request: true, + replication_retention_timestamp: Some(parse_ts(T_LOCAL)), + ..versioned_opts(&version_id, inbound) + }; + put_version(&set_disks, bucket, object, &version_id, &opts).await; + + let info = version_info(&set_disks, bucket, object, &version_id).await; + assert_eq!( + get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), + Some(T_LOCAL), + "the stored category timestamp must be the source-authored time, not the receiver's clock" + ); + assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE")); + } + + #[tokio::test] + async fn first_inbound_version_pins_source_authored_timestamp() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "lww-first-version-ts-pinned"; + let object = "object"; + let version_id = Uuid::new_v4().to_string(); + make_bucket(&disk_stores, bucket).await; + + let mut inbound = HashMap::new(); + inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string()); + insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string()); + let mut evaluated = inbound.clone(); + insert_str(&mut evaluated, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_NEW.to_string()); + let opts = ObjectOptions { + replication_request: true, + replication_legalhold_timestamp: Some(parse_ts(T_OLD)), + eval_metadata: Some(evaluated), + ..versioned_opts(&version_id, inbound) + }; + + put_version(&set_disks, bucket, object, &version_id, &opts).await; + + let info = version_info(&set_disks, bucket, object, &version_id).await; + assert_eq!( + get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(), + Some(T_OLD), + "the first copy must store the source timestamp, not the receiver evaluation time" + ); + } + + #[tokio::test] + async fn newer_local_tag_deletion_survives_stale_inbound_tags() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "lww-tagging-deleted"; + let object = "object"; + let version_id = Uuid::new_v4().to_string(); + make_bucket(&disk_stores, bucket).await; + // Local DeleteObjectTagging state: no tags, but a newer tagging timestamp. + let mut local = HashMap::new(); + insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string()); + put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await; + + put_version( + &set_disks, + bucket, + object, + &version_id, + &inbound_tagging_opts(&version_id, "site=remote", T_OLD), + ) + .await; + + let info = version_info(&set_disks, bucket, object, &version_id).await; + assert!( + info.user_tags.is_empty(), + "a newer local tag deletion must not be resurrected by older inbound tags" + ); + assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL)); + } +} + #[cfg(test)] mod inline_put_commit_path_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; diff --git a/crates/replication/src/operation.rs b/crates/replication/src/operation.rs index a905ed992..ce566d9f8 100644 --- a/crates/replication/src/operation.rs +++ b/crates/replication/src/operation.rs @@ -88,6 +88,17 @@ impl MustReplicateOptions { return true; } + // A REPLICA version was delivered by a peer and carries no per-target + // internal status of its own; whether its local metadata edits flow + // back is the replication rule's ReplicaModifications decision + // (`ReplicationConfig::replicate` with `replica = true`, MinIO + // mustReplicate parity). Gating it on a COMPLETED target state would + // silently drop every replica-side tag / retention / legal-hold edit in + // an active-active topology (rustfs/backlog#1953). + if self.replication_status() == ReplicationStatusType::Replica { + return true; + } + get_internal_metadata(&self.meta, SUFFIX_REPLICATION_STATUS) .as_deref() .and_then(|statuses| { @@ -381,6 +392,13 @@ mod tests { assert!(options.metadata_target_is_eligible(arn)); assert!(!options.metadata_target_is_eligible("arn:rustfs:replication:missing")); + + // A replica-side metadata edit (active-active) has no per-target + // internal status; eligibility is left to the ReplicaModifications rule. + let replica = MustReplicateOptions::new(&HashMap::new(), String::new(), ReplicationType::Metadata, false) + .with_replication_status(ReplicationStatusType::Replica); + assert!(replica.metadata_target_is_eligible(arn)); + assert!(replica.metadata_target_is_eligible("arn:rustfs:replication:missing")); assert!( MustReplicateOptions::new(&HashMap::new(), String::new(), ReplicationType::Object, false) .metadata_target_is_eligible(arn) diff --git a/crates/utils/src/http/headers.rs b/crates/utils/src/http/headers.rs index b983218c0..380c59705 100644 --- a/crates/utils/src/http/headers.rs +++ b/crates/utils/src/http/headers.rs @@ -47,6 +47,8 @@ pub const AMZ_DELETE_MARKER: &str = "x-amz-delete-marker"; // S3 object tagging pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging"; +/// Lowercase wire form of [`AMZ_OBJECT_TAGGING`] for `HeaderMap` insertion. +pub const AMZ_OBJECT_TAGGING_LOWER: &str = "x-amz-tagging"; pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count"; pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive"; diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 701dbd38b..097cc30e0 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -554,10 +554,11 @@ fn apply_replication_timestamps_from_headers(headers: &HeaderMap, o // Persist into the internal metadata keys so a later outbound replication // pass (replication_target_boundary) reads the source's modification - // times instead of falling back to mod_time. - // TODO(P1-6): receiver-side LWW is still missing — when the stored - // per-category timestamp is newer than the inbound one, the existing - // tags/retention/legal-hold should win instead of being overwritten. + // times instead of falling back to mod_time. Receiver-side LWW happens at + // the set layer under the object write lock + // (ecstore set_disk::ops::object::merge_replication_metadata_lww, + // rustfs/backlog#1953): a category whose stored timestamp is newer than + // the inbound one keeps the local values. for (timestamp, suffix) in [ (opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP), (opts.replication_retention_timestamp, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP), From 6f6dd19cc702644cff674a7a326ffd1dde91f7bc Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 20:18:04 +0800 Subject: [PATCH 27/41] ci(coverage): add security ratchet calibration (#6388) --- .config/coverage-baselines.toml | 20 +++ .config/make/tests.mak | 1 + .github/workflows/coverage.yml | 41 +++-- docs/testing/README.md | 15 +- scripts/check_security_coverage.py | 264 +++++++++++++++++++++++++++++ scripts/coverage_per_crate.py | 41 +++-- 6 files changed, 352 insertions(+), 30 deletions(-) create mode 100644 .config/coverage-baselines.toml create mode 100644 scripts/check_security_coverage.py diff --git a/.config/coverage-baselines.toml b/.config/coverage-baselines.toml new file mode 100644 index 000000000..77409d9d2 --- /dev/null +++ b/.config/coverage-baselines.toml @@ -0,0 +1,20 @@ +# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173. +# Update counts only with a linked coverage run and a reviewed explanation. +phase = "report-only" +allowed_drop_percentage_points = 1.0 + +[crates."crates/iam"] +covered = 5149 +count = 8131 + +[crates."crates/kms"] +covered = 2950 +count = 4200 + +[crates."crates/policy"] +covered = 4636 +count = 5464 + +[crates."crates/crypto"] +covered = 469 +count = 494 diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 626df9b84..993f27347 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -36,6 +36,7 @@ script-tests: ## Run shell script tests ./scripts/test_manual_transition_runbooks.sh ./scripts/check_embedded_secrets.sh --self-test python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_security_coverage.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test python3 ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ece00c2eb..8a843fdd5 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,14 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Weekly workspace line-coverage baseline (backlog#1153 infra-5). +# Workspace line-coverage baseline and security-crate calibration +# (backlog#1153 infra-5/infra-6). # -# NON-BLOCKING by design: this workflow only runs on schedule and manual -# dispatch, so it never attaches a status to a PR and must never be made a -# required check. It exists to give coverage a visible baseline and trend -# (per-crate table in the job summary, lcov artifact kept 90 days) — the -# per-crate ratchet for the security-critical crates builds on it later -# (backlog#1153 infra-6, report-only first per the ci-11 ladder). +# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and +# trend, while relevant pull requests run a report-only security-crate +# comparison. Neither job is a required check during calibration. # # Measurement scope matches the PR test gate (ci.yml "Run tests"): # `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are @@ -31,6 +29,17 @@ name: coverage on: + pull_request: + branches: [main] + paths: + - "crates/iam/**" + - "crates/kms/**" + - "crates/policy/**" + - "crates/crypto/**" + - ".config/coverage-baselines.toml" + - "scripts/coverage_per_crate.py" + - "scripts/check_security_coverage.py" + - ".github/workflows/coverage.yml" workflow_dispatch: schedule: # 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00), @@ -39,6 +48,10 @@ on: # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes. - cron: "43 7 * * 0" +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name != 'schedule' }} + # Only alert-on-failure needs more than read access; it declares its own # job-level `issues: write`. permissions: @@ -46,12 +59,14 @@ permissions: jobs: coverage: - name: Workspace coverage (weekly) + name: Workspace line coverage runs-on: sm-standard-4 # The instrumented build cannot reuse the regular CI cache (different - # RUSTFLAGS), so a cold week rebuilds the workspace before running the - # full suite; give it double the test job's 60-minute budget. - timeout-minutes: 120 + # RUSTFLAGS), so a cold run rebuilds the workspace before running the + # full suite. Two later exact-head runs exhausted 150 minutes before the + # report steps, so allow one additional 90-minute cold-run margin while + # keeping the calibration job bounded. + timeout-minutes: 240 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" # Match the PR gate's nextest semantics (ci.yml runs `--profile ci`): @@ -91,7 +106,9 @@ jobs: cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json - name: Write per-crate summary - run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY" + run: | + python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY" + python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY" - name: Upload coverage artifact if: always() diff --git a/docs/testing/README.md b/docs/testing/README.md index e38c0d1a2..702c2c4c9 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -158,10 +158,11 @@ added by backlog#1153 infra-4. ## Coverage -Line coverage is measured **weekly, not per-PR**, and is non-blocking: it -exists for visibility and trend, never as a required check. Per-crate ratchets -for the security-critical crates (iam / kms / policy / crypto) build on this -baseline later (backlog#1153 infra-6, report-only first). +Workspace line coverage is measured weekly. Pull requests that touch iam, kms, +policy, or crypto also run a non-required, report-only comparison against +`.config/coverage-baselines.toml`. During calibration, a regression is recorded +in the job summary without failing the job; missing or malformed coverage +evidence still fails closed (backlog#1153 infra-6). - **CI**: `.github/workflows/coverage.yml` runs every Sunday and on manual dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the @@ -174,6 +175,12 @@ baseline later (backlog#1153 infra-6, report-only first). plus the full suite). It prints the same per-crate table via `scripts/coverage_per_crate.py` and writes `target/llvm-cov/lcov.info` and `coverage.json`. +- **Security-critical ratchet**: relevant pull requests compare iam / kms / + policy / crypto line coverage with the versioned baseline. Drops greater than + the configured one-percentage-point calibration threshold are marked + `REGRESSION (report-only)`. The weekly summary runs the same comparison so + calibration continues even when no relevant pull request is open. Baseline + changes require a linked coverage run and a reviewed explanation. - **Trend comparison**: each run's job summary is the weekly per-crate snapshot — open two runs from the Actions history (workflow "coverage") and compare their tables. For line-level diffs, download the two runs' diff --git a/scripts/check_security_coverage.py b/scripts/check_security_coverage.py new file mode 100644 index 000000000..710421589 --- /dev/null +++ b/scripts/check_security_coverage.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare security-critical crate line coverage with the report-only baseline.""" + +import argparse +import json +import math +import os +import sys +import tempfile +import tomllib +from pathlib import Path + +from coverage_per_crate import fmt_pct, load_coverage + + +SECURITY_CRATES = ("crates/iam", "crates/kms", "crates/policy", "crates/crypto") + + +def load_baselines(path: str) -> tuple[float, dict[str, tuple[int, int]]]: + with open(path, "rb") as fh: + config = tomllib.load(fh) + + if config.get("phase") != "report-only": + raise ValueError("coverage baseline phase must be report-only") + + raw_allowed_drop = config["allowed_drop_percentage_points"] + if isinstance(raw_allowed_drop, bool) or not isinstance(raw_allowed_drop, (int, float)): + raise ValueError("allowed_drop_percentage_points must be a number") + allowed_drop = float(raw_allowed_drop) + if not math.isfinite(allowed_drop) or allowed_drop < 0: + raise ValueError("allowed_drop_percentage_points must be finite and non-negative") + + baselines: dict[str, tuple[int, int]] = {} + for crate, values in config["crates"].items(): + covered = values["covered"] + count = values["count"] + if type(covered) is not int or type(count) is not int: + raise ValueError(f"invalid baseline for {crate}: covered and count must be integers") + if covered < 0 or count <= 0 or covered > count: + raise ValueError(f"invalid baseline for {crate}: {covered}/{count}") + baselines[crate] = (covered, count) + missing = [crate for crate in SECURITY_CRATES if crate not in baselines] + unexpected = sorted(set(baselines).difference(SECURITY_CRATES)) + if missing or unexpected: + raise ValueError(f"coverage baseline crate set mismatch: missing={missing}, unexpected={unexpected}") + return allowed_drop, baselines + + +def compare( + current: dict[str, list[int]], + baselines: dict[str, tuple[int, int]], + allowed_drop: float, +) -> list[tuple[str, int, int, int, int, float, bool]]: + rows = [] + for crate, (baseline_covered, baseline_count) in baselines.items(): + if crate not in current: + raise ValueError(f"coverage report is missing {crate}") + covered, count = current[crate] + if type(covered) is not int or type(count) is not int: + raise ValueError(f"invalid coverage for {crate}: covered and count must be integers") + if covered < 0 or count <= 0 or covered > count: + raise ValueError(f"invalid coverage for {crate}: {covered}/{count}") + current_pct = 100.0 * covered / count + baseline_pct = 100.0 * baseline_covered / baseline_count + delta = current_pct - baseline_pct + rows.append((crate, covered, count, baseline_covered, baseline_count, delta, delta < -allowed_drop)) + return rows + + +def print_report(rows: list[tuple[str, int, int, int, int, float, bool]], allowed_drop: float) -> None: + print("## Security-critical coverage ratchet (report-only)") + print() + print(f"Calibration threshold: a drop greater than {allowed_drop:.2f} percentage points is reported as a regression.") + print() + print("| Crate | Current | Baseline | Delta | Status |") + print("|---|---:|---:|---:|---|") + for crate, covered, count, baseline_covered, baseline_count, delta, regressed in rows: + status = "REGRESSION (report-only)" if regressed else "OK" + print( + f"| `{crate}` | {fmt_pct(covered, count)} ({covered}/{count}) " + f"| {fmt_pct(baseline_covered, baseline_count)} ({baseline_covered}/{baseline_count}) " + f"| {delta:+.2f} pp | {status} |" + ) + print() + print("This calibration phase records regressions without failing the job; malformed or incomplete evidence still fails closed.") + + +def self_test() -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + coverage = root / "coverage.json" + baseline = root / "baseline.toml" + coverage_data = { + "data": [ + { + "files": [ + { + "filename": str(root / "crates/iam/src/lib.rs"), + "summary": {"lines": {"covered": 80, "count": 100}}, + }, + { + "filename": str(root / "crates/kms/src/lib.rs"), + "summary": {"lines": {"covered": 90, "count": 100}}, + }, + { + "filename": str(root / "crates/policy/src/lib.rs"), + "summary": {"lines": {"covered": 90, "count": 100}}, + }, + { + "filename": str(root / "crates/crypto/src/lib.rs"), + "summary": {"lines": {"covered": 90, "count": 100}}, + }, + ], + "totals": {"lines": {"covered": 350, "count": 400}}, + } + ] + } + coverage.write_text(json.dumps(coverage_data), encoding="utf-8") + baseline_text = """phase = "report-only" +allowed_drop_percentage_points = 1.0 +[crates."crates/iam"] +covered = 90 +count = 100 +[crates."crates/kms"] +covered = 85 +count = 100 +[crates."crates/policy"] +covered = 90 +count = 100 +[crates."crates/crypto"] +covered = 90 +count = 100 +""" + baseline.write_text(baseline_text, encoding="utf-8") + current, _ = load_coverage(str(coverage), str(root)) + allowed_drop, baselines = load_baselines(str(baseline)) + rows = compare(current, baselines, allowed_drop) + assert [row[-1] for row in rows] == [True, False, False, False] + try: + compare({"crates/iam": current["crates/iam"]}, baselines, allowed_drop) + except ValueError as error: + assert str(error) == "coverage report is missing crates/kms" + else: + raise AssertionError("missing crate must fail closed") + try: + compare({**current, "crates/iam": [101, 100]}, baselines, allowed_drop) + except ValueError as error: + assert str(error) == "invalid coverage for crates/iam: 101/100" + else: + raise AssertionError("invalid coverage must fail closed") + for invalid_threshold in ("true", '"1.0"', "nan", "inf", "-inf"): + baseline.write_text( + baseline_text.replace("allowed_drop_percentage_points = 1.0", f"allowed_drop_percentage_points = {invalid_threshold}"), + encoding="utf-8", + ) + try: + load_baselines(str(baseline)) + except ValueError: + pass + else: + raise AssertionError(f"non-finite threshold {invalid_threshold} must fail closed") + for field, invalid_values in ( + ("covered", ("true", '"90"', "90.0", "90.5")), + ("count", ("true", '"100"', "100.0", "100.5")), + ): + for invalid_value in invalid_values: + baseline.write_text( + baseline_text.replace(f"{field} = {90 if field == 'covered' else 100}", f"{field} = {invalid_value}", 1), + encoding="utf-8", + ) + try: + load_baselines(str(baseline)) + except ValueError: + pass + else: + raise AssertionError(f"non-integer baseline {field} {invalid_value} must fail closed") + for covered, count in ( + (True, 100), + (80, True), + (80.0, 100), + (80, 100.0), + (float("nan"), 100), + (80, float("inf")), + ): + try: + compare({**current, "crates/iam": [covered, count]}, baselines, allowed_drop) + except ValueError: + pass + else: + raise AssertionError(f"invalid aggregate coverage {covered}/{count} must fail closed") + lines = coverage_data["data"][0]["files"][0]["summary"]["lines"] + for field, invalid_values in ( + ("covered", (True, "80", 80.0, 80.5, float("nan"), float("inf"), float("-inf"))), + ("count", (True, "100", 100.0, 100.5, float("nan"), float("inf"), float("-inf"))), + ): + original = lines[field] + for invalid_value in invalid_values: + lines[field] = invalid_value + coverage.write_text(json.dumps(coverage_data), encoding="utf-8") + try: + load_coverage(str(coverage), str(root)) + except ValueError: + pass + else: + raise AssertionError(f"invalid raw coverage {field} {invalid_value} must fail closed") + lines[field] = original + baseline.write_text( + baseline_text.replace( + '[crates."crates/crypto"]\ncovered = 90\ncount = 100\n', + "", + ), + encoding="utf-8", + ) + try: + load_baselines(str(baseline)) + except ValueError: + pass + else: + raise AssertionError("missing security-crate baseline must fail closed") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("coverage_json", nargs="?") + parser.add_argument("--baseline", default=".config/coverage-baselines.toml") + parser.add_argument("--repo-root", default=os.getcwd()) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + + if args.self_test: + self_test() + print("security coverage self-test passed") + return 0 + if not args.coverage_json: + parser.error("coverage_json is required unless --self-test is used") + + try: + current, _ = load_coverage(args.coverage_json, os.path.abspath(args.repo_root)) + allowed_drop, baselines = load_baselines(args.baseline) + rows = compare(current, baselines, allowed_drop) + except (OSError, ValueError, KeyError, IndexError, json.JSONDecodeError, tomllib.TOMLDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + print_report(rows, allowed_drop) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/coverage_per_crate.py b/scripts/coverage_per_crate.py index 6c6622022..4a0a37b3e 100755 --- a/scripts/coverage_per_crate.py +++ b/scripts/coverage_per_crate.py @@ -47,6 +47,31 @@ def fmt_pct(covered: int, count: int) -> str: return f"{100.0 * covered / count:.2f}%" if count else "—" +def _line_counts(lines: dict[str, int], source: str) -> tuple[int, int]: + covered = lines["covered"] + count = lines["count"] + if type(covered) is not int or type(count) is not int or covered < 0 or count < 0 or covered > count: + raise ValueError(f"invalid line coverage for {source}: {covered}/{count}") + return covered, count + + +def load_coverage(path: str, root: str) -> tuple[dict[str, list[int]], dict[str, int]]: + with open(path, encoding="utf-8") as fh: + export = json.load(fh) + + data = export["data"][0] + files = data["files"] + total_covered, total_count = _line_counts(data["totals"]["lines"], "totals") + + crates: dict[str, list[int]] = {} + for f in files: + covered, count = _line_counts(f["summary"]["lines"], f["filename"]) + acc = crates.setdefault(crate_label(f["filename"], root), [0, 0]) + acc[0] += covered + acc[1] += count + return crates, {"covered": total_covered, "count": total_count} + + def main() -> int: if len(sys.argv) < 2 or len(sys.argv) > 3: print(__doc__.strip(), file=sys.stderr) @@ -54,24 +79,12 @@ def main() -> int: path = sys.argv[1] root = os.path.abspath(sys.argv[2] if len(sys.argv) == 3 else os.getcwd()) - with open(path, encoding="utf-8") as fh: - export = json.load(fh) - try: - data = export["data"][0] - files = data["files"] - totals = data["totals"]["lines"] - except (KeyError, IndexError) as exc: + crates, totals = load_coverage(path, root) + except (KeyError, IndexError, ValueError) as exc: print(f"error: unexpected llvm-cov JSON shape ({exc})", file=sys.stderr) return 1 - crates: dict[str, list[int]] = {} - for f in files: - lines = f["summary"]["lines"] - acc = crates.setdefault(crate_label(f["filename"], root), [0, 0]) - acc[0] += lines["covered"] - acc[1] += lines["count"] - rows = sorted( crates.items(), key=lambda kv: (100.0 * kv[1][0] / kv[1][1]) if kv[1][1] else 101.0, From 5bb9ffffcd9efb52316538530cbe5e683994cb79 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 20:18:41 +0800 Subject: [PATCH 28/41] test(e2e): enforce external client prerequisites (#6402) --- .config/nextest.toml | 4 +- .github/workflows/ci.yml | 27 ++++++++++ .github/workflows/e2e-replication-nightly.yml | 8 +-- crates/e2e_test/README.md | 16 +++--- .../e2e_test/src/bucket_policy_check_test.rs | 4 -- crates/e2e_test/src/common.rs | 35 ++++++++++--- .../src/existing_object_tag_policy_test.rs | 24 +-------- crates/e2e_test/src/kms/common.rs | 17 +------ crates/e2e_test/src/kms/kms_local_test.rs | 6 +-- crates/e2e_test/src/kms/kms_vault_test.rs | 19 +------ .../src/mc_mirror_small_bucket_test.rs | 12 +---- crates/e2e_test/src/multipart_auth_test.rs | 4 -- crates/e2e_test/src/quota_test.rs | 51 ------------------- .../src/replication_extension_test.rs | 10 +--- crates/e2e_test/src/security_boundary_test.rs | 13 ++--- 15 files changed, 78 insertions(+), 172 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index edf5b5df7..841368bef 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -335,8 +335,8 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" } # # Wired by .github/workflows/e2e-replication-nightly.yml (schedule + # workflow_dispatch), which builds the rustfs binary once, installs awscurl so -# the STS dual-node test actually exercises its path (it skips gracefully with -# a visible log line when awscurl is absent), and routes scheduled failures +# the STS dual-node test actually exercises its path (the test fails when +# awscurl is absent), and routes scheduled failures # through .github/actions/schedule-failure-issue (ci-8). Explicit division of # labor with e2e-full: these tests run only in the consolidated nightly # workflow, not in the merge/main lane. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae74ec40a..a45cbe65c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -681,6 +681,19 @@ jobs: cache-save-if: 'false' install-build-packaging-tools: 'false' + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + + - name: Install awscurl + run: | + python3 -m pip install --user --upgrade pip "awscurl==0.44" + echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV" + + - name: Verify awscurl + run: test -x "$AWSCURL_PATH" + # Download after the cache restore so the freshly built binary from the # build job always wins over anything restored into target/debug. - name: Download debug binary @@ -803,6 +816,20 @@ jobs: - name: Verify awscurl run: test -x "$AWSCURL_PATH" + - name: Install mc + env: + MC_VERSION: RELEASE.2025-08-13T08-35-41Z + MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891 + run: | + MC_BINARY="mc.linux-amd64.${MC_VERSION}" + curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}" + echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status + chmod +x "$RUNNER_TEMP/mc" + echo "$RUNNER_TEMP" >> "$GITHUB_PATH" + + - name: Verify mc + run: mc --version + - name: Install Vault run: | VAULT_VERSION="1.17.6" diff --git a/.github/workflows/e2e-replication-nightly.yml b/.github/workflows/e2e-replication-nightly.yml index 145ad317e..24c4aee3b 100644 --- a/.github/workflows/e2e-replication-nightly.yml +++ b/.github/workflows/e2e-replication-nightly.yml @@ -75,11 +75,7 @@ jobs: cache-save-if: ${{ github.ref == 'refs/heads/main' }} install-build-packaging-tools: 'false' - # awscurl lets the STS dual-node test actually exercise its path. Without - # it the test skips gracefully with a visible log line - # (`awscurl_available()` in crates/e2e_test/src/common.rs), so the lane - # still passes — installing it just upgrades that one test from skip to - # real coverage. + # The STS dual-node test requires awscurl and fails if it is unavailable. - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: @@ -87,7 +83,7 @@ jobs: - name: Install awscurl run: | - python3 -m pip install --user --upgrade pip awscurl + python3 -m pip install --user --upgrade pip "awscurl==0.44" echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV" - name: Verify awscurl diff --git a/crates/e2e_test/README.md b/crates/e2e_test/README.md index 24567ac6f..62b609e27 100644 --- a/crates/e2e_test/README.md +++ b/crates/e2e_test/README.md @@ -123,7 +123,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See | `find_available_port` | Random free port (isolation primitive) | | `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` | | `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with | -| `awscurl_available` + `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl` (skip gracefully when absent) | +| `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures | | `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` | | `local_http_client` / `init_logging` | Loopback HTTP client; idempotent tracing init | | `RustFSTestClusterEnvironment` (`new`/`start`/`start_node`/`stop_node`/`create_all_clients`) | Multi-node harness | @@ -189,7 +189,7 @@ cargo nextest run --profile e2e-smoke -p e2e_test cargo nextest run --profile e2e-full -p e2e_test # Cluster fault nightly lane cargo nextest run --profile e2e-nightly -p e2e_test -# Replication nightly lane; install awscurl so STS paths do not skip +# Replication nightly lane; awscurl is required for STS paths cargo nextest run --profile e2e-repl-nightly -p e2e_test # Fixed-port protocol nightly lane RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \ @@ -221,9 +221,8 @@ The `s3s-e2e` CI job selects a random `RUSTFS_TEST_PORT` (see the `e2e-tests` job) to dodge this; local single-node tests already use random ports, so a lingering orphan is usually the cause of a spurious bind failure. -**`awscurl` not found.** `awscurl`-dependent tests skip gracefully with a -visible log line (`awscurl_available()`); install `awscurl` to actually run -them. +**`awscurl` not found.** `awscurl`-dependent tests fail closed with a process +spawn error. Install the pinned CI version before running their profiles. ## Related @@ -258,10 +257,9 @@ A test module may join the smoke filter only if every test in it is: 2. **Single-node** — spawns its own server via `RustFSTestEnvironment`/`start_rustfs_server` on a random port with an isolated temp dir. No `RustFSTestClusterEnvironment`, no fixed ports. -3. **Dependency-free** — no pre-started server at `localhost:9000`, no Vault, - no fixed protocol ports. Tools that may be absent on the runner (e.g. - `awscurl`) are acceptable only when the test skips gracefully with a - visible log line (see `bucket_policy_check_test.rs`). +3. **Hermetic dependencies** — no pre-started server at `localhost:9000`, no + Vault, and no fixed protocol ports. Any required CLI must be pinned and + installed by the workflow; a missing CLI must fail the test. 4. **Not `#[ignore]`** — ignored tests are activation work (backlog#1149 ci-13 / backlog#1148 ilm-3), not smoke candidates. diff --git a/crates/e2e_test/src/bucket_policy_check_test.rs b/crates/e2e_test/src/bucket_policy_check_test.rs index e71e5fe97..0b9345f06 100644 --- a/crates/e2e_test/src/bucket_policy_check_test.rs +++ b/crates/e2e_test/src/bucket_policy_check_test.rs @@ -52,10 +52,6 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: #[tokio::test] async fn test_bucket_policy_authenticated_user() -> Result<(), Box> { init_logging(); - if !crate::common::awscurl_available() { - info!("Skipping test_bucket_policy_authenticated_user because awscurl is not available"); - return Ok(()); - } info!("Starting test_bucket_policy_authenticated_user..."); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index 7fad7ea76..fda121840 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -494,15 +494,20 @@ fn awscurl_binary_path() -> PathBuf { .unwrap_or_else(|| PathBuf::from("awscurl")) } -pub fn awscurl_available() -> bool { - let path = awscurl_binary_path(); - if path.components().count() > 1 || path.is_absolute() { - return path.is_file(); +fn verify_awscurl_path(path: &Path) -> std::io::Result<()> { + let output = Command::new(path).arg("--help").output()?; + if output.status.success() { + return Ok(()); } - std::env::var_os("PATH") - .map(|paths| std::env::split_paths(&paths).any(|dir| dir.join(&path).is_file())) - .unwrap_or(false) + Err(std::io::Error::other(format!( + "awscurl prerequisite check failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))) +} + +pub fn require_awscurl() -> std::io::Result<()> { + verify_awscurl_path(&awscurl_binary_path()) } // Global initialization @@ -1747,6 +1752,22 @@ mod tests { assert_eq!(normalize_rustfs_build_features(" , "), None); } + #[test] + fn missing_awscurl_is_a_prerequisite_failure() { + let missing = std::env::temp_dir().join(format!("missing-awscurl-{}", Uuid::new_v4())); + + let error = verify_awscurl_path(&missing).expect_err("a missing awscurl binary must fail the test prerequisite"); + + assert_eq!(error.kind(), ErrorKind::NotFound); + } + + #[test] + fn available_awscurl_client_passes_prerequisite_check() { + let executable = std::env::current_exe().expect("the test executable should have a path"); + + verify_awscurl_path(&executable).expect("an available client with a working help command should pass"); + } + #[test] fn capture_log_path_uses_temp_directory_basename() { assert_eq!( diff --git a/crates/e2e_test/src/existing_object_tag_policy_test.rs b/crates/e2e_test/src/existing_object_tag_policy_test.rs index 17c34f166..7668773ed 100644 --- a/crates/e2e_test/src/existing_object_tag_policy_test.rs +++ b/crates/e2e_test/src/existing_object_tag_policy_test.rs @@ -16,9 +16,7 @@ //! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit //! `Content-Type: application/x-www-form-urlencoded` on `POST /`. -use crate::common::{ - RustFSTestEnvironment, awscurl_available, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging, -}; +use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging}; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging}; @@ -175,11 +173,6 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) { #[tokio::test] async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box> { init_logging(); - if !awscurl_available() { - info!("Skipping test_e2e_iam_policy_existing_object_tag_get_object: awscurl not available"); - return Ok(()); - } - let suffix = Uuid::new_v4(); let user = format!("e2eiamtag-{suffix}"); let user_secret = "longSecretKeyForTest123!"; @@ -233,11 +226,6 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box< #[tokio::test] async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box> { init_logging(); - if !awscurl_available() { - info!("Skipping test_e2e_bucket_policy_existing_object_tag_get_object: awscurl not available"); - return Ok(()); - } - let suffix = Uuid::new_v4(); let user = format!("e2ebptag-{suffix}"); let user_secret = "longSecretKeyForTest456!"; @@ -294,11 +282,6 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B #[tokio::test] async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box> { init_logging(); - if !awscurl_available() { - info!("Skipping test_e2e_sts_assume_role_session_policy_existing_object_tag: awscurl not available"); - return Ok(()); - } - let suffix = Uuid::new_v4(); let parent = format!("e2e-sts-par-{suffix}"); let parent_secret = "longSecretKeyForParentSts99!"; @@ -370,11 +353,6 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result #[tokio::test] async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box> { init_logging(); - if !awscurl_available() { - info!("Skipping test_e2e_sts_session_policy_delete_objects_object_prefix_only: awscurl not available"); - return Ok(()); - } - let suffix = Uuid::new_v4(); let parent = format!("e2e-sts-del-par-{suffix}"); let parent_secret = "longSecretKeyForParentDelete99!"; diff --git a/crates/e2e_test/src/kms/common.rs b/crates/e2e_test/src/kms/common.rs index 07b21db4d..3a6c2d364 100644 --- a/crates/e2e_test/src/kms/common.rs +++ b/crates/e2e_test/src/kms/common.rs @@ -22,9 +22,7 @@ //! - KMS backend configuration (Local and Vault) //! - SSE encryption testing utilities -use crate::common::{ - RustFSTestEnvironment, awscurl_available, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client, -}; +use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client}; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::ServerSideEncryption; @@ -59,15 +57,6 @@ pub fn init_logging() { // Additional KMS-specific logging configuration can be added here if needed } -pub fn skip_if_kms_admin_tool_unavailable(test_name: &str) -> bool { - if awscurl_available() { - return false; - } - - info!("Skipping {} because awscurl is not available in PATH", test_name); - true -} - pub fn sse_customer_key_md5_base64(key: &str) -> String { let mut hasher = Md5::new(); hasher.update(key.as_bytes()); @@ -490,10 +479,6 @@ pub async fn test_kms_key_management( access_key: &str, secret_key: &str, ) -> Result<(), Box> { - if skip_if_kms_admin_tool_unavailable("test_kms_key_management") { - return Ok(()); - } - info!("Testing KMS key management APIs"); // Test CreateKey diff --git a/crates/e2e_test/src/kms/kms_local_test.rs b/crates/e2e_test/src/kms/kms_local_test.rs index 522b28478..4a8550b32 100644 --- a/crates/e2e_test/src/kms/kms_local_test.rs +++ b/crates/e2e_test/src/kms/kms_local_test.rs @@ -20,8 +20,7 @@ //! - Complete encryption/decryption lifecycle use super::common::{ - LocalKMSTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64, - test_kms_key_management, test_sse_c_encryption, + LocalKMSTestEnvironment, get_kms_status, sse_customer_key_md5_base64, test_kms_key_management, test_sse_c_encryption, }; use crate::common::{TEST_BUCKET, init_logging}; use tracing::{error, info}; @@ -29,9 +28,6 @@ use tracing::{error, info}; #[tokio::test] async fn test_local_kms_end_to_end() -> Result<(), Box> { init_logging(); - if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") { - return Ok(()); - } info!("Starting Local KMS End-to-End Test"); // Create LocalKMS test environment diff --git a/crates/e2e_test/src/kms/kms_vault_test.rs b/crates/e2e_test/src/kms/kms_vault_test.rs index 0dd19f703..c73515d95 100644 --- a/crates/e2e_test/src/kms/kms_vault_test.rs +++ b/crates/e2e_test/src/kms/kms_vault_test.rs @@ -22,8 +22,8 @@ use crate::common::{TEST_BUCKET, init_logging}; use tracing::{error, info}; use super::common::{ - VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, skip_if_kms_admin_tool_unavailable, sse_customer_key_md5_base64, - start_kms, test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption, + VAULT_KEY_NAME, VaultTestEnvironment, get_kms_status, sse_customer_key_md5_base64, start_kms, + test_all_multipart_encryption_types, test_error_scenarios, test_kms_key_management, test_sse_c_encryption, test_sse_kms_encryption, test_sse_s3_encryption, }; @@ -62,9 +62,6 @@ impl VaultKmsTestContext { #[tokio::test] async fn test_vault_kms_end_to_end() -> Result<(), Box> { init_logging(); - if skip_if_kms_admin_tool_unavailable("test_vault_kms_end_to_end") { - return Ok(()); - } info!("Starting Vault KMS End-to-End Test with default key {}", VAULT_KEY_NAME); let context = VaultKmsTestContext::new().await?; @@ -117,9 +114,6 @@ async fn test_vault_kms_end_to_end() -> Result<(), Box Result<(), Box> { init_logging(); - if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_isolation") { - return Ok(()); - } info!("Starting Vault KMS SSE-C key isolation test"); let context = VaultKmsTestContext::new().await?; @@ -203,9 +197,6 @@ async fn test_vault_kms_key_isolation() -> Result<(), Box Result<(), Box> { init_logging(); - if skip_if_kms_admin_tool_unavailable("test_vault_kms_large_file") { - return Ok(()); - } info!("Starting Vault KMS large file SSE-S3 test"); let context = VaultKmsTestContext::new().await?; @@ -267,9 +258,6 @@ async fn test_vault_kms_large_file() -> Result<(), Box Result<(), Box> { init_logging(); - if skip_if_kms_admin_tool_unavailable("test_vault_kms_multipart_upload") { - return Ok(()); - } info!("Starting Vault KMS multipart upload encryption suite"); let context = VaultKmsTestContext::new().await?; @@ -297,9 +285,6 @@ async fn test_vault_kms_multipart_upload() -> Result<(), Box Result<(), Box> { init_logging(); - if skip_if_kms_admin_tool_unavailable("test_vault_kms_key_operations") { - return Ok(()); - } info!("Starting Vault KMS key operations test (CRUD)"); let context = VaultKmsTestContext::new().await?; diff --git a/crates/e2e_test/src/mc_mirror_small_bucket_test.rs b/crates/e2e_test/src/mc_mirror_small_bucket_test.rs index 2a9c1a507..213da60c0 100644 --- a/crates/e2e_test/src/mc_mirror_small_bucket_test.rs +++ b/crates/e2e_test/src/mc_mirror_small_bucket_test.rs @@ -41,13 +41,6 @@ async fn create_issue_3107_fixture(root: &Path) -> TestResult { Ok(()) } -fn mc_available() -> bool { - Command::new("mc") - .arg("--version") - .output() - .is_ok_and(|output| output.status.success()) -} - fn run_mc(args: &[&str]) -> TestResult { let output = Command::new("mc").args(args).output()?; if !output.status.success() { @@ -75,10 +68,7 @@ fn count_files(root: &Path) -> usize { async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult { crate::common::init_logging(); info!("Starting issue #3107 mc mirror regression test"); - if !mc_available() { - info!("Skipping issue #3107 mc mirror regression test because mc is not installed"); - return Ok(()); - } + run_mc(&["--version"])?; let mut env = RustFSTestEnvironment::new().await?; env.start_rustfs_server(vec![]).await?; diff --git a/crates/e2e_test/src/multipart_auth_test.rs b/crates/e2e_test/src/multipart_auth_test.rs index 247cb46d7..334c55339 100644 --- a/crates/e2e_test/src/multipart_auth_test.rs +++ b/crates/e2e_test/src/multipart_auth_test.rs @@ -4278,10 +4278,6 @@ async fn test_signed_put_object_extract_preserves_pax_metadata_and_version_id() async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retention_conditions() -> Result<(), Box> { init_logging(); - if !crate::common::awscurl_available() { - return Ok(()); - } - let mut env = RustFSTestEnvironment::new().await?; env.start_rustfs_server(vec![]).await?; diff --git a/crates/e2e_test/src/quota_test.rs b/crates/e2e_test/src/quota_test.rs index adff548ad..85d54b3f8 100644 --- a/crates/e2e_test/src/quota_test.rs +++ b/crates/e2e_test/src/quota_test.rs @@ -18,15 +18,6 @@ use http::{Method, StatusCode}; use tokio::time::{Duration, sleep, timeout}; use tracing::{debug, info}; -fn skip_without_awscurl() -> bool { - if crate::common::awscurl_available() { - return false; - } - - info!("Skipping quota test because awscurl is not available"); - true -} - /// Test environment setup for quota tests pub struct QuotaTestEnv { pub env: RustFSTestEnvironment, @@ -276,9 +267,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_basic_operations() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; // Create test bucket @@ -320,9 +308,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_admission_aws_chunked_declared_encoding() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -371,9 +356,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_update_and_clear() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -406,9 +388,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_delete_operations() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -442,9 +421,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_usage_tracking() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -480,9 +456,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_statistics() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -513,9 +486,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_check_api() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -553,9 +523,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_multiple_buckets() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; // Create two buckets in the same environment @@ -593,9 +560,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_error_handling() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -628,9 +592,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_http_endpoints() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -689,9 +650,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_normal_user_permissions() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -744,9 +702,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_copy_operations() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -789,9 +744,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_batch_delete() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; @@ -847,9 +799,6 @@ mod integration_tests { #[tokio::test] async fn test_quota_multipart_upload() -> Result<(), Box> { init_logging(); - if skip_without_awscurl() { - return Ok(()); - } let env = QuotaTestEnv::new().await?; env.create_bucket().await?; diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index 4d7b641e4..206bd50e2 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -13,9 +13,8 @@ // limitations under the License. use crate::common::{ - RustFSTestEnvironment, admin_create_user, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, - local_http_client, replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, - signed_request_with_session_token, + RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client, + replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token, }; use crate::fake_s3_target::{ FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation, @@ -7699,11 +7698,6 @@ async fn test_site_replication_replicates_multiple_service_accounts_real_dual_no async fn test_site_replication_replicates_service_accounts_created_from_sts_session_real_dual_node() -> TestResult { init_logging(); - if !awscurl_available() { - eprintln!("Skipping STS site replication service-account test because awscurl is unavailable"); - return Ok(()); - } - let mut source_env = RustFSTestEnvironment::new().await?; source_env .start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV) diff --git a/crates/e2e_test/src/security_boundary_test.rs b/crates/e2e_test/src/security_boundary_test.rs index 0a62ef0f5..8c5c7d469 100644 --- a/crates/e2e_test/src/security_boundary_test.rs +++ b/crates/e2e_test/src/security_boundary_test.rs @@ -21,12 +21,11 @@ //! - SSRF prevention (internal/private endpoints rejected for tiering) //! - Race condition handling (concurrent writes converge without corruption) -use crate::common::{RustFSTestEnvironment, awscurl_available, awscurl_put, init_logging}; +use crate::common::{RustFSTestEnvironment, awscurl_put, init_logging, require_awscurl}; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Tag, Tagging}; use std::error::Error; -use tracing::info; /// Oversized tagging payloads must be rejected by the per-object tag limit. /// @@ -225,16 +224,12 @@ async fn test_concurrent_object_operations() -> Result<(), Box Result<(), Box> { init_logging(); - if !awscurl_available() { - info!("Skipping tiering URL validation test because awscurl is not available"); - return Ok(()); - } - + require_awscurl()?; let mut env = RustFSTestEnvironment::new().await?; env.start_rustfs_server(vec![]).await?; From b5b060a9a10a7e01990d6691c275ac85c84c587f Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 20:18:56 +0800 Subject: [PATCH 29/41] ci: pin s3tests Python tools (#6407) --- .github/workflows/e2e-s3tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-s3tests.yml b/.github/workflows/e2e-s3tests.yml index 1be61d73e..d617d8ee3 100644 --- a/.github/workflows/e2e-s3tests.yml +++ b/.github/workflows/e2e-s3tests.yml @@ -178,9 +178,14 @@ jobs: - name: Install Python tools run: | - python3 -m pip install --user --upgrade pip awscurl tox + python3 -m pip install --user --upgrade pip "awscurl==0.44" "tox==4.60.0" echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Verify Python tools + run: | + test "$(python3 -c 'import importlib.metadata as m; print(m.version("awscurl"))')" = "0.44" + test "$(python3 -c 'import importlib.metadata as m; print(m.version("tox"))')" = "4.60.0" + - name: Enable buildx uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 From d3c0714b3aa35ae4d91b8b41444bafd0a957a9e8 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 20:19:13 +0800 Subject: [PATCH 30/41] ci: add s3tests upstream HEAD canary (#6409) --- .github/workflows/e2e-s3tests.yml | 82 +++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/.github/workflows/e2e-s3tests.yml b/.github/workflows/e2e-s3tests.yml index d617d8ee3..6429d68f6 100644 --- a/.github/workflows/e2e-s3tests.yml +++ b/.github/workflows/e2e-s3tests.yml @@ -21,6 +21,9 @@ # suite and reports promotion candidates. Regressions, unclassified tests, # incomplete execution, and infrastructure errors fail the job; classified # failures for not-yet-implemented features remain informational. +# - Non-blocking upstream HEAD canary: collects current upstream node IDs and +# reports new, removed, duplicate, or overlapping classifications without +# making upstream drift a release gate. # - Manual runs (workflow_dispatch): same, with configurable mode/scope. # # All test execution is delegated to scripts/s3-tests/run.sh (single source of @@ -359,6 +362,85 @@ jobs: name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }} path: artifacts/** + upstream-head-canary: + name: Upstream HEAD classification canary + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + + - name: Install collection tool + run: | + python3 -m pip install --user "tox==4.60.0" + python3 - <<'PY' + from importlib.metadata import version + + assert version("tox") == "4.60.0" + PY + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Compare upstream HEAD classifications + id: upstream-compare + run: | + ARTIFACT_DIR="artifacts/s3tests-upstream-head" + UPSTREAM_DIR="${RUNNER_TEMP}/s3-tests-upstream" + mkdir -p "${ARTIFACT_DIR}" + git clone --depth 1 https://github.com/ceph/s3-tests.git "${UPSTREAM_DIR}" + git -C "${UPSTREAM_DIR}" rev-parse HEAD > "${ARTIFACT_DIR}/upstream-sha.txt" + cp "${UPSTREAM_DIR}/s3tests.conf.SAMPLE" "${UPSTREAM_DIR}/s3tests.conf" + ( + cd "${UPSTREAM_DIR}" + S3TEST_CONF="${UPSTREAM_DIR}/s3tests.conf" tox -- \ + -q --collect-only s3tests/functional/test_s3.py \ + -m "not rustfs_never_marker" + ) 2>&1 | tee "${ARTIFACT_DIR}/collect.log" + grep -E '^s3tests/functional/test_s3\.py::' \ + "${ARTIFACT_DIR}/collect.log" > "${ARTIFACT_DIR}/collected-nodeids.txt" + python3 scripts/s3-tests/report_compat.py \ + --lists-dir scripts/s3-tests \ + --collected-nodeids "${ARTIFACT_DIR}/collected-nodeids.txt" \ + --check-classifications-only 2>&1 | tee "${ARTIFACT_DIR}/classification-drift.txt" + + - name: Publish canary report + if: always() + env: + CANARY_OUTCOME: ${{ steps.upstream-compare.outcome }} + run: | + { + echo "## ceph/s3-tests upstream HEAD canary" + echo + if [ -f artifacts/s3tests-upstream-head/upstream-sha.txt ]; then + echo "Upstream HEAD: $(cat artifacts/s3tests-upstream-head/upstream-sha.txt)" + fi + echo + echo '```text' + if [ -s artifacts/s3tests-upstream-head/classification-drift.txt ]; then + cat artifacts/s3tests-upstream-head/classification-drift.txt + elif [ "${CANARY_OUTCOME}" != "success" ]; then + echo "Canary did not complete; inspect the collection log artifact." + else + echo "No classification drift detected." + fi + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload canary artifacts + if: always() && env.ACT != 'true' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: s3tests-upstream-head + path: artifacts/s3tests-upstream-head/** + retention-days: 14 + alert-on-failure: name: Alert on scheduled failure needs: [s3tests] From 06ef472defcdf73d4ad07a72b8e6736ed60f5e2b Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 20:20:01 +0800 Subject: [PATCH 31/41] fix(ci): make Warp ABBA evidence bounded and complete (#6417) --- .github/workflows/performance-ab.yml | 12 ++- scripts/hotpath_warp_ab_gate.sh | 85 +++++++++++----- scripts/run_hotpath_warp_abba.sh | 57 +++++------ scripts/run_object_batch_bench_enhanced.sh | 94 +++++++++++------- .../security/check_performance_ab_workflow.sh | 4 +- scripts/test_hotpath_warp_ab_gate.sh | 8 ++ scripts/test_hotpath_warp_abba.sh | 18 +++- scripts/test_object_batch_bench_enhanced.sh | 96 ++++++++++++++++++- 8 files changed, 274 insertions(+), 100 deletions(-) diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml index de4986388..61a8eed41 100644 --- a/.github/workflows/performance-ab.yml +++ b/.github/workflows/performance-ab.yml @@ -121,14 +121,14 @@ jobs: candidate_sha="$(git rev-parse HEAD)" if [[ "${{ github.event_name }}" == "schedule" ]]; then baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}" - if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then - echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2 - exit 1 - fi else baseline_sha="$(git rev-parse origin/main)" fi git cat-file -e "${baseline_sha}^{commit}" + if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then + echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2 + exit 1 + fi echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT" echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT" echo "baseline commit: $baseline_sha" @@ -342,6 +342,10 @@ jobs: if: always() run: | status="${{ steps.ab.outputs.status }}" + if [[ -z "$status" ]]; then + echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2 + exit 1 + fi if [[ "$status" != "0" ]]; then echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2 exit "$status" diff --git a/scripts/hotpath_warp_ab_gate.sh b/scripts/hotpath_warp_ab_gate.sh index d184661d6..ec2e839f9 100755 --- a/scripts/hotpath_warp_ab_gate.sh +++ b/scripts/hotpath_warp_ab_gate.sh @@ -30,12 +30,16 @@ REQUIRE_TAIL_ERROR="false" MARKDOWN_OUT="" EXEMPTION_REASON="deliberate correctness tradeoff" declare -a COMPARE_CSVS=() +declare -a COMPARE_LABELS=() usage() { cat <<'USAGE' Usage: hotpath_warp_ab_gate.sh --compare-csv [--compare-csv ...] [options] --compare-csv baseline_compare.csv to evaluate (repeatable). + --labeled-compare-csv