diff --git a/.config/e2e-distributed-selection.txt b/.config/e2e-distributed-selection.txt new file mode 100644 index 000000000..7291d5e03 --- /dev/null +++ b/.config/e2e-distributed-selection.txt @@ -0,0 +1,2 @@ +sha256-linux=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07 +sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07 diff --git a/.config/ecstore-required-tests.json b/.config/ecstore-required-tests.json index 6cadc7815..24b723e39 100644 --- a/.config/ecstore-required-tests.json +++ b/.config/ecstore-required-tests.json @@ -40,6 +40,21 @@ "invariant": "corrupt-part-arrays", "suite": "rustfs-filemeta", "name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics" + }, + { + "invariant": "odm-source-contract-s3", + "suite": "rustfs", + "name": "on_demand_migration::source_client::tests::s3_backend_satisfies_the_shared_backend_contract" + }, + { + "invariant": "odm-source-contract-azure", + "suite": "rustfs", + "name": "on_demand_migration::azure::tests::azure_backend_satisfies_the_shared_backend_contract" + }, + { + "invariant": "odm-source-contract-gcs", + "suite": "rustfs", + "name": "on_demand_migration::gcs::tests::gcs_native_backend_satisfies_the_shared_backend_contract" } ], "fixtures": [ diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 726ca04cf..7d29ad6fc 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -31,6 +31,7 @@ script-tests: ## Run shell script tests ./scripts/test_object_batch_bench_enhanced.sh ./scripts/test_hotpath_warp_ab_gate.sh ./scripts/test_hotpath_warp_abba.sh + ./scripts/test_scanner_validation_harness.sh ./scripts/test_exact_1mib_handoff_abba.sh ./scripts/test_pinned_paired_abba_bench.sh ./scripts/test_manual_transition_runbooks.sh diff --git a/.config/migration-gate-floor.txt b/.config/migration-gate-floor.txt index 1da7a3af2..9abadcffd 100644 --- a/.config/migration-gate-floor.txt +++ b/.config/migration-gate-floor.txt @@ -1,10 +1,11 @@ # Committed floor for the number of tests selected by the migration-critical # CI gate (see scripts/check_migration_gate_count.sh, backlog#1153 infra-12). # -# The floor equals the exact count of rustfs-ecstore --lib tests matching the -# gate filter (name substrings: data_movement, rebalance, decommission, -# source_cleanup, delete_marker) at the time this file was last updated. +# The floor equals the exact count of rustfs-ecstore --lib tests, with the +# test-util feature enabled, matching the gate filter (name substrings: +# data_movement, rebalance, decommission, source_cleanup, delete_marker) at +# the time this file was last updated. # CI fails if the selected count drops below this number, so renames or # removals that thin the gate must update this file in the same PR. # Adding tests does not require a bump, but bumping keeps the guard tight. -571 +946 diff --git a/.config/nextest.toml b/.config/nextest.toml index bc57b8b8f..17c5dc7a5 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -183,6 +183,13 @@ test-group = 'e2e-reliability' filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)' test-group = 'e2e-inline-boundaries' +# 4-node 4-drive distributed Actions suite: each case starts four rustfs +# processes and up to sixteen data directories. Serialize across nextest's +# process boundary so several 4x4 clusters never overlap. +[[profile.default.overrides]] +filter = 'package(e2e_test) & test(/^distributed::/)' +test-group = 'e2e-cluster-nightly' + # Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial] # does not cross nextest process boundaries, so keep every Vault-backed test in # one group. @@ -526,6 +533,27 @@ path = "junit.xml" filter = 'package(e2e_test)' test-group = 'e2e-cluster-nightly' +# --------------------------------------------------------------------------- +# e2e-distributed profile — 4-node 4-disk Actions suite +# --------------------------------------------------------------------------- +# Storage-sensitive PR / nightly / dispatch lane owned by +# .github/workflows/e2e-distributed.yml. +# Each case starts four rustfs processes (and for site replication, two +# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY. +# Serialized via e2e-cluster-nightly with no retries. +[profile.e2e-distributed] +default-filter = 'package(e2e_test) & test(/^distributed::/)' +fail-fast = false +# Decommission / rebalance cases poll for up to 180s with little stdout. +slow-timeout = { period = "120s", terminate-after = 6 } + +[profile.e2e-distributed.junit] +path = "junit.xml" + +[[profile.e2e-distributed.overrides]] +filter = 'package(e2e_test)' +test-group = 'e2e-cluster-nightly' + # --------------------------------------------------------------------------- # e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20) # --------------------------------------------------------------------------- @@ -586,6 +614,10 @@ path = "junit.xml" # cluster-fault lane. heal_erasure_disk_rebuild is intentionally not # excluded here because backlog#2213 promotes core heal rebuild coverage to # this merge/main lane while retaining nightly coverage. +# * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning, +# replication, quota, observability, expand/decommission/rebalance, site +# replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and +# .github/workflows/e2e-distributed.yml. # * on_demand_migration::interop_test — the ODM-20 provider interoperability # cases, which are meaningless without a source: they run in the dedicated # [profile.e2e-odm-interop] lane below, where the workflow points them at a @@ -607,6 +639,7 @@ default-filter = """ package(e2e_test) & !test(/^protocols::/) & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/) + & !test(/^distributed::/) & !test(/^replication_extension_test::/) & !test(/^replication_target_matrix_test::/) & !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/) diff --git a/.config/scanner-heal-required-tests.json b/.config/scanner-heal-required-tests.json new file mode 100644 index 000000000..abf25420c --- /dev/null +++ b/.config/scanner-heal-required-tests.json @@ -0,0 +1,40 @@ +{ + "schema": 1, + "cases": { + "background-target-restart": { + "gate": "G14", + "task": "W21", + "lane": "e2e-nightly", + "suite": "e2e_test", + "name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart", + "oracle": "background-target-restart.json", + "min_objects": 9, + "max_objects": 65, + "topology": {"nodes": 4, "drives_per_node": 1}, + "scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4." + } + }, + "release_pending": { + "G01": "W02/W04 complete root and quota authority coverage", + "G02": "W03 bounded checkpoint progress and independent version inventory", + "G03": "W17/W18 exact scoped ACK with durable publication and mixed peers", + "G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary", + "G05": "W06/W07 per-object outcomes and bounded terminal retention", + "G06": "W06/W08/W23 concurrent status, legacy clients and truncation", + "G07": "W12/W13/W14 durable MRF responsibility at every commit boundary", + "G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix", + "G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads", + "G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery", + "G11": "W04/W19/W24 maintenance and complete producer coverage", + "G12": "W02/W15/W16 both quota paths during reset and settlement", + "G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail", + "G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage", + "P1": "W20 measured cold-walk share and foreground latency/throughput", + "P2": "W20/W24 measured post-stop convergence and cold segment reuse", + "P3": "W20 measured two-hour pressure/heal capacity and recovery window", + "P4": "W20 measured MRF scale and replay cost with retained responsibility", + "R-E": "W03/W05 fixed-budget real process restart through enumeration and classification", + "R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace", + "R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement" + } +} diff --git a/.github/scheduled-validations.json b/.github/scheduled-validations.json index 9ac7f2614..e55ef3d56 100644 --- a/.github/scheduled-validations.json +++ b/.github/scheduled-validations.json @@ -4,6 +4,11 @@ { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 }, { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 }, { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 }, + { + "workflow": ".github/workflows/e2e-distributed.yml", + "max_age_hours": 36, + "never_ran_grace_until": "2026-09-18T00:00:00Z" + }, { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 }, { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 }, { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 }, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5250f8bae..3d12a8b68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -850,6 +850,11 @@ jobs: cache-save-if: 'false' install-build-packaging-tools: 'false' + - name: Install network fault-injection tools + run: | + sudo apt-get install -y iptables + sudo -n iptables --version + - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: diff --git a/.github/workflows/e2e-distributed.yml b/.github/workflows/e2e-distributed.yml new file mode 100644 index 000000000..c3a5ca0e1 --- /dev/null +++ b/.github/workflows/e2e-distributed.yml @@ -0,0 +1,206 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 4-node 4-disk distributed e2e lane. +# +# Each selected test starts a real localhost cluster via +# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the +# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is +# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs, +# nightly runs, and manual dispatches all execute the same fail-closed suite. +# Upgrade cases download the same pinned previous release as e2e-upgrade.yml. +# +# Isolated pool filesystems: expand/decommission/rebalance cases require +# independent `statfs` capacity. `sm-standard-4` is an ARC pod +# (`scripts/ci/check_runner_ephemerality.sh`) and usually has no +# `/dev/loop-control`, so `mount -o loop` fails with ENOENT ("mount failed: +# No such file or directory"). The prepare step therefore mounts four 1 GiB +# tmpfs instances and exports them as `RUSTFS_E2E_POOL_ROOTS`. + +name: e2e-distributed + +on: + pull_request: + paths: + - "Cargo.lock" + - "Cargo.toml" + - ".config/nextest.toml" + - ".github/workflows/e2e-distributed.yml" + - "crates/audit/**" + - "crates/common/**" + - "crates/config/**" + - "crates/e2e_test/**" + - "crates/ecstore/**" + - "crates/filemeta/**" + - "crates/heal/**" + - "crates/iam/**" + - "crates/lock/**" + - "crates/madmin/**" + - "crates/notify/**" + - "crates/replication/**" + - "crates/s3-client/**" + - "crates/s3-ops/**" + - "crates/s3-types/**" + - "crates/scanner/**" + - "crates/storage-api/**" + - "crates/utils/**" + - "rustfs/**" + workflow_dispatch: + inputs: + filter: + description: "Optional nextest -E filter (default: the whole e2e-distributed profile)" + required: false + default: "" + schedule: + # 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23). + - cron: "53 5 * * *" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name != 'schedule' }} + +jobs: + distributed: + name: Distributed 4-node 4-disk e2e + runs-on: sm-standard-4 + timeout-minutes: 180 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + NO_PROXY: 127.0.0.1,localhost + HTTP_PROXY: "" + HTTPS_PROXY: "" + # Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml). + UPGRADE_SOURCE_VERSION: 1.0.0-rc.2 + UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip + UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + rust-version: stable + cache-shared-key: ci-e2e-distributed + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + install-build-packaging-tools: 'false' + + - name: Prepare isolated filesystems for pool movement + run: | + set -euo pipefail + mount_base="${RUNNER_TEMP}/rustfs-e2e-pools" + mkdir -p "${mount_base}" + roots=() + for pool in 0 1 2 3; do + mountpoint="${mount_base}/pool-${pool}" + mkdir -p "${mountpoint}" + # sm-standard-4 is an ARC pod without usable loop devices, so + # `mount -o loop` fails with ENOENT. Sized tmpfs still reports a + # distinct st_dev and independent 1G statfs capacity. + sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}" + sudo chmod 1777 "${mountpoint}" + roots+=("${mountpoint}") + done + printf -v joined_roots '%s:' "${roots[@]}" + echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[0]}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[1]}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[2]}" + findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[3]}" + + - name: Download pinned previous release + env: + SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source + run: | + set -euo pipefail + mkdir -p "$SOURCE_DIR" + archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET" + curl --fail --location --retry 3 --output "$archive" \ + "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}" + echo "$UPGRADE_SOURCE_SHA256 $archive" | sha256sum --check --strict + unzip -q "$archive" -d "$SOURCE_DIR" + chmod +x "$SOURCE_DIR/rustfs" + test -x "$SOURCE_DIR/rustfs" + echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV" + + - name: Build rustfs binary + run: | + cargo build -p rustfs --bins + : > target/debug/rustfs.features + + - name: Verify distributed e2e membership + env: + NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json + run: | + cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}" + python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}" + + - name: Run distributed 4-node e2e suite + env: + RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs + FILTER: ${{ inputs.filter }} + run: | + set -euo pipefail + if [ -n "${FILTER}" ]; then + cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}" + else + cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail + fi + + - name: Upload distributed e2e diagnostics + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: e2e-distributed-${{ github.run_number }} + path: | + target/nextest/e2e-distributed/junit.xml + ${{ runner.temp }}/rustfs-e2e-distributed-list.json + ${{ runner.temp }}/rustfs-e2e-distributed-logs/ + retention-days: 7 + if-no-files-found: warn + + - name: Unmount isolated pool filesystems + if: always() + run: | + set -euo pipefail + mount_base="${RUNNER_TEMP}/rustfs-e2e-pools" + for pool in 0 1 2 3; do + mountpoint="${mount_base}/pool-${pool}" + if mountpoint --quiet "${mountpoint}"; then + sudo umount "${mountpoint}" + fi + done + + alert-on-failure: + name: Alert on scheduled failure + needs: [distributed] + if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/scheduled-validation-watchdog.yml b/.github/workflows/scheduled-validation-watchdog.yml index e778ec640..154d541d3 100644 --- a/.github/workflows/scheduled-validation-watchdog.yml +++ b/.github/workflows/scheduled-validation-watchdog.yml @@ -22,6 +22,7 @@ on: - "Continuous Integration" - "coverage" - "e2e-nightly" + - "e2e-distributed" - "e2e-s3tests" - "Fuzz" - "mint" diff --git a/crates/e2e_test/README.md b/crates/e2e_test/README.md index ed5a65d97..9c6d90c0a 100644 --- a/crates/e2e_test/README.md +++ b/crates/e2e_test/README.md @@ -26,6 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern: | **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) | | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) | | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` | +| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) | | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup | | **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory | @@ -171,6 +172,7 @@ the same profile for membership and execution with one nightly worker. | KMS suite | `e2e-full` job, merge queue + main | **Active** | | Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** | | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) | +| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) | | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) | | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) | | Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) | @@ -191,6 +193,9 @@ cargo nextest run --profile e2e-smoke -p e2e_test cargo nextest run --profile e2e-full -p e2e_test # Cluster fault nightly lane cargo nextest run --profile e2e-nightly -p e2e_test +# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade) +# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed. +cargo nextest run --profile e2e-distributed -p e2e_test # Replication nightly lane; awscurl is required for STS paths cargo nextest run --profile e2e-repl-nightly -p e2e_test # Fixed-port protocol nightly lane diff --git a/crates/e2e_test/build.rs b/crates/e2e_test/build.rs new file mode 100644 index 000000000..6d412d3d4 --- /dev/null +++ b/crates/e2e_test/build.rs @@ -0,0 +1,74 @@ +// Copyright 2024 RustFS Team +// Licensed under the Apache License, Version 2.0. + +use std::path::Path; +use std::process::Command; + +fn git(root: &Path, args: &[&str]) -> Option { + let output = Command::new("git").args(args).current_dir(root).output().ok()?; + output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned()) +} + +fn emit(name: &str, value: &str) { + let value = if value.contains(['\n', '\r']) { "unknown" } else { value }; + println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}"); +} + +fn main() { + let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default(); + let root = Path::new(&manifest).join("../.."); + // Cover dependency/common sources as well as this crate. HEAD/ref/index + // changes must refresh identity even when no Rust source mtime changes. + for path in [ + "crates", + "rustfs", + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + ".cargo", + ".config", + ] { + println!("cargo:rerun-if-changed={}", root.join(path).display()); + } + let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()]; + if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) { + git_paths.push(reference); + } + for path in git_paths { + if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) { + let path = Path::new(&path); + let path = if path.is_absolute() { + path.to_owned() + } else { + root.join(path) + }; + if path.exists() { + println!("cargo:rerun-if-changed={}", path.display()); + } + } + } + let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned()); + let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty()); + let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned()); + let mut features = std::env::vars() + .filter_map(|(key, _)| { + key.strip_prefix("CARGO_FEATURE_") + .map(|name| name.to_ascii_lowercase().replace('_', "-")) + }) + .collect::>(); + features.sort(); + emit("COMMIT", &revision); + emit("DIRTY", if dirty { "true" } else { "false" }); + emit("LOCK", &lock); + emit("FEATURES", &features.join(",")); + for name in ["TARGET", "PROFILE"] { + emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned())); + } + println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS"); + let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default(); + let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect(); + emit("RUSTFLAGS_HEX", &flags); +} diff --git a/crates/e2e_test/src/chaos.rs b/crates/e2e_test/src/chaos.rs index 920a87f0c..f0b2971e9 100644 --- a/crates/e2e_test/src/chaos.rs +++ b/crates/e2e_test/src/chaos.rs @@ -55,18 +55,20 @@ type ChaosResult = Result>; /// A successful S3 GET only proves that a quorum can serve an object. Replacement /// tests need this lower-level record to prove that the rebuilt target holds the /// `xl.meta` selected for a specific version and every `part.N` it declares. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)] pub(crate) struct VersionShardCensus { pub version_id: Option, pub has_xl_meta: bool, pub data_dir: Option, pub erasure_index: Option, + pub data_blocks: Option, + pub parity_blocks: Option, pub expected_part_numbers: BTreeSet, pub present_part_fingerprints: BTreeMap, pub inline_data_fingerprint: Option, } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)] pub(crate) struct PartShardFingerprint { pub size: u64, pub sha256: String, @@ -88,13 +90,15 @@ impl VersionShardCensus { && manifest.is_complete() && self.data_dir == manifest.data_dir && self.erasure_index == manifest.erasure_index + && self.data_blocks == manifest.data_blocks + && self.parity_blocks == manifest.parity_blocks && self.expected_part_numbers == manifest.expected_part_numbers && self.present_part_fingerprints == manifest.present_part_fingerprints && self.inline_data_fingerprint == manifest.inline_data_fingerprint } } -fn sha256_hex(data: &[u8]) -> String { +pub(crate) fn sha256_hex(data: &[u8]) -> String { let digest = Sha256::digest(data); digest.iter().map(|byte| format!("{byte:02x}")).collect() } @@ -313,6 +317,8 @@ pub(crate) fn census_object_version_on_disk( has_xl_meta: false, data_dir: None, erasure_index: None, + data_blocks: None, + parity_blocks: None, expected_part_numbers: BTreeSet::new(), present_part_fingerprints: BTreeMap::new(), inline_data_fingerprint: None, @@ -360,6 +366,8 @@ pub(crate) fn census_object_version_on_disk( has_xl_meta: true, data_dir, erasure_index, + data_blocks: Some(file_info.erasure.data_blocks), + parity_blocks: Some(file_info.erasure.parity_blocks), expected_part_numbers, present_part_fingerprints, inline_data_fingerprint, @@ -413,6 +421,8 @@ mod tests { has_xl_meta: true, data_dir: Some("data-dir".to_string()), erasure_index: Some(3), + data_blocks: Some(2), + parity_blocks: Some(2), expected_part_numbers: BTreeSet::from([1]), present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]), inline_data_fingerprint: None, diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index b108cd074..daa744ab8 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -1700,6 +1700,69 @@ impl RustFSTestClusterEnvironment { Ok(()) } + /// Append a new single-node erasure pool to a stopped multi-pool cluster. + /// + /// Used to simulate pool expansion on localhost: every pool already owns + /// exactly one node with `drives_per_node >= 2` (the only multi-pool layout + /// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is + /// allocated a fresh port and empty drive directories; callers must + /// [`Self::start`] afterwards so every process picks up the extended + /// volumes argument. Existing data directories are left untouched. + pub async fn append_single_node_pool(&mut self) -> Result> { + if self.nodes.iter().any(|node| node.process.is_some()) { + return Err("stop the cluster before appending a pool".into()); + } + if self.topology.drives_per_node < 2 { + return Err( + "append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)" + .into(), + ); + } + + let mut pools = self.topology.normalized_pools(); + for (pool_idx, nodes) in pools.iter().enumerate() { + if nodes.len() != 1 { + return Err(format!( + "pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool", + nodes.len() + ) + .into()); + } + } + + let new_idx = self.nodes.len(); + let port = RustFSTestEnvironment::find_available_port().await?; + let address = format!("127.0.0.1:{port}"); + let data_dirs: Vec = (0..self.topology.drives_per_node) + .map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive)) + .collect(); + for dir in &data_dirs { + fs::create_dir_all(dir).await?; + } + + self.nodes.push(ClusterNode { + url: format!("http://{address}"), + address, + data_dir: data_dirs[0].clone(), + data_dirs, + pool_idx: pools.len(), + process: None, + }); + pools.push(vec![new_idx]); + self.topology.node_count = self.nodes.len(); + self.topology.pools = pools; + self.node_extra_env.push(Vec::new()); + self.node_capture_log_paths.push(None); + self.volume_proxy_addresses.push(None); + + if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") { + self.extra_env + .push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string())); + } + + Ok(new_idx) + } + /// Gracefully stop one cluster node and wait for its process to exit. /// /// This is intentionally separate from [`Self::stop_node`]: the latter is diff --git a/crates/e2e_test/src/data_usage_test.rs b/crates/e2e_test/src/data_usage_test.rs index 141ab29c7..512d5e16c 100644 --- a/crates/e2e_test/src/data_usage_test.rs +++ b/crates/e2e_test/src/data_usage_test.rs @@ -35,11 +35,15 @@ where { let mut last_usage = DataUsageInfo::default(); let mut last_query_error = None; - for _ in 0..45 { + for _ in 0..90 { match get_data_usage_info(env).await { Ok(usage) => { last_query_error = None; - if usage.buckets_usage.contains_key(bucket) && predicate(&usage) { + if usage.is_complete_bucket_usage_snapshot() + && usage.usage_snapshot_converged != Some(false) + && usage.buckets_usage.contains_key(bucket) + && predicate(&usage) + { return Ok(usage); } last_usage = usage; diff --git a/crates/e2e_test/src/distributed/chaos_test.rs b/crates/e2e_test/src/distributed/chaos_test.rs new file mode 100644 index 000000000..df058fbda --- /dev/null +++ b/crates/e2e_test/src/distributed/chaos_test.rs @@ -0,0 +1,222 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket, + wait_for_ready, wait_until, +}; +use crate::chaos::{census_object_version_on_disk, signed_admin_post}; +use crate::common::{build_test_s3_config, init_logging}; +use crate::fault_proxy::FaultMode; +use aws_sdk_s3::Client; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{Barrier, mpsc}; +use tokio::time::timeout; + +#[tokio::test] +async fn kill_and_restart_node_preserves_objects() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("killnode"); + dist.create_bucket(&bucket).await?; + let body = vec![0x11u8; 128 * 1024]; + put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?; + + dist.cluster.stop_node(3)?; + retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?; + + dist.cluster.start_node(3).await?; + wait_for_ready(&dist.cluster).await?; + assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?; + Ok(()) +} + +#[tokio::test] +async fn full_cluster_restart_preserves_objects() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("pwr"); + dist.create_bucket(&bucket).await?; + let body = vec![0x44u8; 64 * 1024]; + put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?; + + dist.cluster.stop(); + dist.cluster.start().await?; + wait_for_ready(&dist.cluster).await?; + for node_idx in 0..dist.cluster.nodes.len() { + assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?; + } + Ok(()) +} + +#[tokio::test] +async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult { + init_logging(); + let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?; + let bucket = unique_bucket("baddrive"); + dist.create_bucket(&bucket).await?; + let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024); + put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?; + + let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]); + let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?; + assert!( + baseline.is_complete(), + "replacement target did not hold a complete baseline shard: {baseline:?}" + ); + assert!( + !baseline.expected_part_numbers.is_empty(), + "replacement witness must use physical part shards: {baseline:?}" + ); + + dist.cluster.stop_node(0)?; + let format_path = replaced_drive.join(".rustfs.sys/format.json"); + let format = std::fs::read(&format_path)?; + let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display())); + std::fs::rename(&replaced_drive, &retired_drive)?; + std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?; + std::fs::write(&format_path, format)?; + let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?; + assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}"); + + dist.cluster.start_node(0).await?; + wait_for_ready(&dist.cluster).await?; + let heal_body = + r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#; + let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url); + signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?; + wait_until( + Duration::from_secs(90), + || async { + let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?; + Ok(healed.matches_manifest(&baseline)) + }, + "fresh replacement contains the original complete shard manifest", + ) + .await?; + + for node_idx in 0..dist.cluster.nodes.len() { + assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?; + } + Ok(()) +} + +#[tokio::test] +async fn concurrent_gets_survive_peer_node_kill() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("getkill"); + dist.create_bucket(&bucket).await?; + let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024); + put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?; + + let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::, _>>()?; + let worker_count = 12; + let release = Arc::new(Barrier::new(worker_count + 1)); + let (started_tx, mut started_rx) = mpsc::unbounded_channel(); + let mut handles = Vec::new(); + for idx in 0..worker_count { + let client = live[idx % live.len()].clone(); + let bucket = bucket.clone(); + let body = body.clone(); + let release = release.clone(); + let started_tx = started_tx.clone(); + handles.push(tokio::spawn(async move { + let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?; + if response.content_length() != Some(body.len() as i64) { + return Err::<(), Box>( + format!("worker {idx} received a wrong content length").into(), + ); + } + started_tx.send(idx)?; + release.wait().await; + let actual = response.body.collect().await?.into_bytes(); + if actual.as_ref() != body.as_slice() { + return Err(format!("worker {idx} received corrupted bytes after peer kill").into()); + } + Ok(()) + })); + } + drop(started_tx); + for _ in 0..worker_count { + timeout(Duration::from_secs(30), started_rx.recv()) + .await? + .ok_or("a streaming GET exited before reaching the kill barrier")?; + } + + dist.cluster.stop_node(3)?; + release.wait().await; + for handle in handles { + handle.await??; + } + + dist.cluster.start_node(3).await?; + wait_for_ready(&dist.cluster).await?; + assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?; + Ok(()) +} + +#[tokio::test] +async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?; + let proxied_url = format!("http://{}", proxy.local_addr()); + let proxied_client = Client::from_conf(build_test_s3_config( + &proxied_url, + &dist.cluster.access_key, + &dist.cluster.secret_key, + None, + "distributed-network-chaos", + )); + + let result: TestResult = async { + let bucket = unique_bucket("netfault"); + dist.create_bucket(&bucket).await?; + let baseline = payload_for("network/baseline.bin", 1024 * 1024); + put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?; + assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?; + + proxy.set_mode(FaultMode::Blackhole); + assert_eq!(proxy.mode(), FaultMode::Blackhole); + if let Ok(Ok(_)) = timeout( + Duration::from_secs(5), + proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(), + ) + .await + { + return Err("blackholed node endpoint unexpectedly completed a GET".into()); + } + + let during = payload_for("network/during.bin", 1024 * 1024); + timeout(Duration::from_secs(30), async { + put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?; + assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?; + assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?; + Ok::<_, Box>(()) + }) + .await??; + + proxy.set_mode(FaultMode::Pass); + retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?; + Ok(()) + } + .await; + + proxy.set_mode(FaultMode::Pass); + proxy.shutdown().await; + result +} diff --git a/crates/e2e_test/src/distributed/concurrency_stability_test.rs b/crates/e2e_test/src/distributed/concurrency_stability_test.rs new file mode 100644 index 000000000..06a24e9a4 --- /dev/null +++ b/crates/e2e_test/src/distributed/concurrency_stability_test.rs @@ -0,0 +1,98 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket}; +use crate::common::init_logging; +use std::collections::BTreeSet; +use std::sync::Arc; +use tokio::sync::Barrier; + +#[tokio::test] +async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("conc"); + dist.create_bucket(&bucket).await?; + let clients = Arc::new(dist.clients()?); + let worker_count = 24; + let rounds = 4; + let barrier = Arc::new(Barrier::new(worker_count)); + + let mut handles = Vec::new(); + for idx in 0..worker_count { + let clients = clients.clone(); + let barrier = barrier.clone(); + let bucket = bucket.clone(); + handles.push(tokio::spawn(async move { + barrier.wait().await; + let writer = &clients[idx % clients.len()]; + let reader = &clients[(idx + 1) % clients.len()]; + let copier = &clients[(idx + 2) % clients.len()]; + let mut retained = Vec::with_capacity(rounds); + for round in 0..rounds { + let key = format!("source/worker-{idx:02}-round-{round}.bin"); + let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin"); + let body = payload_for(&key, 64 * 1024); + put_object(writer, &bucket, &key, body.clone()).await?; + + let head = reader.head_object().bucket(&bucket).key(&key).send().await?; + if head.content_length() != Some(body.len() as i64) { + return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into()); + } + assert_object_bytes(reader, &bucket, &key, &body).await?; + + copier + .copy_object() + .bucket(&bucket) + .key(©_key) + .copy_source(format!("{bucket}/{key}")) + .send() + .await?; + assert_object_bytes(writer, &bucket, ©_key, &body).await?; + + writer.delete_object().bucket(&bucket).key(&key).send().await?; + let missing = reader + .head_object() + .bucket(&bucket) + .key(&key) + .send() + .await + .expect_err("deleted source key must not remain visible"); + if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) { + return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into()); + } + retained.push((copy_key, body)); + } + Ok::<_, Box>(retained) + })); + } + + let mut inventory = Vec::new(); + for handle in handles { + inventory.extend(handle.await??); + } + + let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect(); + for (node_idx, client) in clients.iter().enumerate() { + let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?; + let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect(); + assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing"); + for (key, body) in &inventory { + assert_object_bytes(client, &bucket, key, body) + .await + .map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?; + } + } + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/concurrent_data_movement_test.rs b/crates/e2e_test/src/distributed/concurrent_data_movement_test.rs new file mode 100644 index 000000000..66b30f5de --- /dev/null +++ b/crates/e2e_test/src/distributed/concurrent_data_movement_test.rs @@ -0,0 +1,74 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress, + decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission, + unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress, +}; +use crate::common::init_logging; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Barrier; + +#[tokio::test] +async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?; + let bucket = unique_bucket("concdecom"); + dist.create_bucket(&bucket).await?; + let baseline_client = dist.client(0)?; + let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?; + dist.expand_to_four_pools().await?; + + start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?; + + let clients = Arc::new(dist.clients()?); + let barrier = Arc::new(Barrier::new(17)); + let mut handles = Vec::new(); + for idx in 0..16 { + let clients = clients.clone(); + let barrier = barrier.clone(); + let bucket = bucket.clone(); + handles.push(tokio::spawn(async move { + barrier.wait().await; + let client = &clients[idx % clients.len()]; + let key = format!("live/{idx:02}.bin"); + let body = payload_for(&key, 8 * 1024); + retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?; + Ok::<_, Box>((key, body)) + })); + } + + wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; + barrier.wait().await; + + let mut live_objects = Vec::new(); + for handle in handles { + live_objects.push(handle.await??); + } + let status = decommission_status_json(&dist.cluster).await?; + if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? { + return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into()); + } + + wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?; + + let checker = dist.client(2)?; + assert_inventory(&checker, &bucket, &inventory).await?; + for (key, body) in live_objects { + retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?; + } + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/data_integrity_movement_test.rs b/crates/e2e_test/src/distributed/data_integrity_movement_test.rs new file mode 100644 index 000000000..266f31f9d --- /dev/null +++ b/crates/e2e_test/src/distributed/data_integrity_movement_test.rs @@ -0,0 +1,156 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying, + sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete, +}; +use crate::common::init_logging; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use std::time::Duration; + +#[tokio::test] +async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?; + let bucket = unique_bucket("integrity"); + dist.create_bucket(&bucket).await?; + let client = dist.client(0)?; + enable_versioning(&client, &bucket).await?; + let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?; + let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect(); + + let versioned_key = "history/versioned.bin"; + let version_one = b"historical bytes before data movement".to_vec(); + let version_two = b"current bytes before data movement".to_vec(); + let version_one_id = client + .put_object() + .bucket(&bucket) + .key(versioned_key) + .body(ByteStream::from(version_one.clone())) + .send() + .await? + .version_id() + .ok_or("historical PUT omitted version ID")? + .to_string(); + let version_two_id = client + .put_object() + .bucket(&bucket) + .key(versioned_key) + .body(ByteStream::from(version_two.clone())) + .send() + .await? + .version_id() + .ok_or("current PUT omitted version ID")? + .to_string(); + + let multipart_key = "multipart/moved.bin"; + let first_part = vec![0x31; 5 * 1024 * 1024]; + let second_part = vec![0x72; 1024 * 1024]; + let upload = client + .create_multipart_upload() + .bucket(&bucket) + .key(multipart_key) + .send() + .await?; + let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?; + let uploaded_one = client + .upload_part() + .bucket(&bucket) + .key(multipart_key) + .upload_id(upload_id) + .part_number(1) + .body(ByteStream::from(first_part.clone())) + .send() + .await?; + let uploaded_two = client + .upload_part() + .bucket(&bucket) + .key(multipart_key) + .upload_id(upload_id) + .part_number(2) + .body(ByteStream::from(second_part.clone())) + .send() + .await?; + client + .complete_multipart_upload() + .bucket(&bucket) + .key(multipart_key) + .upload_id(upload_id) + .multipart_upload( + CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?) + .build(), + ) + .build(), + ) + .send() + .await?; + + dist.expand_to_four_pools().await?; + + start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?; + wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; + wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?; + + let after_client = dist.client(2)?; + assert_inventory(&after_client, &bucket, &inventory).await?; + for (key, expected_hash) in before { + let got = after_client.get_object().bucket(&bucket).key(&key).send().await?; + let body = got.body.collect().await?.into_bytes(); + assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission"); + } + for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] { + let got = after_client + .get_object() + .bucket(&bucket) + .key(versioned_key) + .version_id(version_id) + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission"); + } + let mut expected_multipart = first_part; + expected_multipart.extend_from_slice(&second_part); + let got_multipart = after_client + .get_object() + .bucket(&bucket) + .key(multipart_key) + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!( + sha256_hex(got_multipart.as_ref()), + sha256_hex(&expected_multipart), + "multipart checksum changed after decommission" + ); + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs b/crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs new file mode 100644 index 000000000..d76aa9e05 --- /dev/null +++ b/crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs @@ -0,0 +1,81 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory, + put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active, + wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete, +}; +use crate::common::init_logging; +use std::time::Duration; + +#[tokio::test] +async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?; + let bucket = unique_bucket("expand"); + dist.create_bucket(&bucket).await?; + let client = dist.client(0)?; + let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?; + assert_inventory(&client, &bucket, &inventory).await?; + + for expected_nodes in 2..=4 { + let new_node = dist.append_pool_and_restart().await?; + assert_eq!(new_node + 1, expected_nodes); + assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?; + } + assert_eq!(dist.cluster.nodes.len(), 4); + + // Prove that the expanded pool map is durable, and clear any recovery + // latch raised while the newly-added pool replicas converged. + dist.restart_current_binary_gracefully().await?; + + let after_expand = dist.client(0)?; + assert_inventory(&after_expand, &bucket, &inventory).await?; + let peer = dist.client(3)?; + assert_inventory(&peer, &bucket, &inventory).await?; + + let rebalance_id = start_rebalance(&dist.cluster).await?; + wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?; + wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?; + assert_inventory(&peer, &bucket, &inventory).await?; + Ok(()) +} + +#[tokio::test] +async fn four_pool_decommission_moves_objects_without_loss() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?; + let bucket = unique_bucket("decom"); + dist.create_bucket(&bucket).await?; + let client = dist.client(0)?; + let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?; + dist.expand_to_four_pools().await?; + + let pools_before = list_pools_json(&dist.cluster).await?; + let pool_count = pools_before + .as_array() + .map(Vec::len) + .or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len)) + .ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?; + assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}"); + + start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?; + wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; + wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?; + + let after = dist.client(2)?; + assert_inventory(&after, &bucket, &inventory).await?; + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/extra_test.rs b/crates/e2e_test/src/distributed/extra_test.rs new file mode 100644 index 000000000..590dd13f4 --- /dev/null +++ b/crates/e2e_test/src/distributed/extra_test.rs @@ -0,0 +1,149 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until, +}; +use crate::common::init_logging; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use std::time::Duration; + +#[tokio::test] +async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("extra"); + dist.create_bucket(&bucket).await?; + let client = dist.client(0)?; + + let key = "multipart.bin"; + let part1 = vec![0x41u8; 5 * 1024 * 1024]; + let part2 = vec![0x42u8; 5 * 1024 * 1024]; + let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?; + let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string(); + + let uploaded1 = client + .upload_part() + .bucket(&bucket) + .key(key) + .upload_id(&upload_id) + .part_number(1) + .body(ByteStream::from(part1.clone())) + .send() + .await?; + let uploaded2 = client + .upload_part() + .bucket(&bucket) + .key(key) + .upload_id(&upload_id) + .part_number(2) + .body(ByteStream::from(part2.clone())) + .send() + .await?; + + client + .complete_multipart_upload() + .bucket(&bucket) + .key(key) + .upload_id(&upload_id) + .multipart_upload( + CompletedMultipartUpload::builder() + .parts( + CompletedPart::builder() + .part_number(1) + .e_tag(uploaded1.e_tag().unwrap_or_default()) + .build(), + ) + .parts( + CompletedPart::builder() + .part_number(2) + .e_tag(uploaded2.e_tag().unwrap_or_default()) + .build(), + ) + .build(), + ) + .send() + .await?; + + let mut expected = part1; + expected.extend_from_slice(&part2); + for node_idx in 0..dist.cluster.nodes.len() { + assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?; + } + + put_object(&client, &bucket, "list/a", b"a".to_vec()).await?; + put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?; + let mut seen = Vec::new(); + for node_idx in 0..dist.cluster.nodes.len() { + let listed = dist + .client(node_idx)? + .list_objects_v2() + .bucket(&bucket) + .prefix("list/") + .send() + .await?; + let keys: Vec = listed + .contents() + .iter() + .filter_map(|object| object.key().map(str::to_string)) + .collect(); + seen.push(keys); + } + for keys in &seen[1..] { + assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}"); + } + + let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?; + assert_eq!(got, b"a"); + Ok(()) +} + +#[tokio::test] +async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("listed"); + dist.create_bucket(&bucket).await?; + put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?; + + for node_idx in 0..dist.cluster.nodes.len() { + let client = dist.client(node_idx)?; + let name = bucket.clone(); + wait_until( + Duration::from_secs(20), + || { + let client = client.clone(); + let name = name.clone(); + async move { + let listed = client.list_buckets().send().await?; + Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str()))) + } + }, + &format!("node {node_idx} lists {bucket}"), + ) + .await?; + wait_until( + Duration::from_secs(20), + || { + let client = dist.client(node_idx).expect("client"); + let name = bucket.clone(); + async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) } + }, + &format!("node {node_idx} reads seed.bin"), + ) + .await?; + } + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/harness.rs b/crates/e2e_test/src/distributed/harness.rs new file mode 100644 index 000000000..f610beaa4 --- /dev/null +++ b/crates/e2e_test/src/distributed/harness.rs @@ -0,0 +1,1235 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared 4-node distributed e2e helpers. +//! +//! Two localhost-expressible layouts cover the suite: +//! +//! * **4×4 single pool** (`four_by_four`) — four processes, four drives each, +//! one `DistErasure` pool (16 explicit volume endpoints). This is the +//! default S3 / lock / versioning / chaos topology. +//! * **4×4 four pool** — start one durable single-node pool, then append three +//! single-node pools one at a time. Required for decommission/rebalance/expand, +//! which the server rejects on a single pool. Cold-starting multiple empty +//! pools races their bootstrap identities on localhost DistErasure. +//! +//! Genuine multi-node *striped* pools still need multi-host CI (backlog +//! #1313 / #1314). Site replication uses two 4-node 1-drive clusters so the +//! process count stays at eight rather than sixteen. + +use crate::common::{ + ClusterTopology, FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, admin_request, build_test_s3_config, + local_http_client, replication_fast_env, signed_request, +}; +use crate::replication_extension_test::LOOPBACK_REPLICATION_TARGET_ENV; +use aws_sdk_s3::Client; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; +use http::{Method, StatusCode}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tokio::time::{Instant, sleep}; +use uuid::Uuid; + +pub(crate) type TestResult = Result>; + +pub(crate) const NODE_COUNT: usize = 4; +pub(crate) const DRIVES_PER_NODE: usize = 4; +/// Retire the seed pool after test data is written there, proving that user +/// objects—not only internal metadata—move to the expansion pools. +pub(crate) const DECOMMISSION_POOL_ID: usize = 0; +const POOL_ROOTS_ENV: &str = "RUSTFS_E2E_POOL_ROOTS"; +const POOL_META_V3_ENV: [(&str, &str); 2] = [ + ("RUSTFS_POOL_META_V3_WRITE", "true"), + ("RUSTFS_POOL_META_V3_FLEET_CONFIRMED", "true"), +]; + +#[derive(Clone, Copy, Debug)] +pub(crate) enum DistLayout { + /// 4 nodes × 4 drives, one erasure pool spanning every endpoint. + FourByFour, + /// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout). + FourNodeFourDisk, + /// 1 node × 4 drives, used only as the durable seed for pool expansion. + SingleNodeFourDrive, +} + +pub(crate) struct DistCluster { + pub cluster: RustFSTestClusterEnvironment, + pool_storage_roots: Option>, + owned_pool_dirs: Vec, +} + +impl DistCluster { + pub async fn start(layout: DistLayout) -> TestResult { + Self::start_with_env(layout, &[]).await + } + + pub async fn start_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult { + let mut dist = Self::new_stopped_with_env(layout, extra_env).await?; + dist.cluster.start().await?; + Ok(dist) + } + + /// Allocate ports and data dirs without spawning processes. + /// + /// Upgrade tests configure capture logs, then start a pinned previous + /// binary against the same directories. + pub async fn new_stopped(layout: DistLayout) -> TestResult { + Self::new_stopped_with_env(layout, &[]).await + } + + pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult { + let topology = match layout { + DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE), + DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT), + DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]), + }; + let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?; + let pool_storage_roots = match layout { + DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?), + DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None, + }; + let mut owned_pool_dirs = Vec::new(); + if let Some(roots) = pool_storage_roots.as_deref() { + owned_pool_dirs.push(relocate_pool_storage(&mut cluster, 0, roots)?); + } + cluster.set_env("NO_PROXY", "127.0.0.1,localhost"); + cluster.set_env("HTTP_PROXY", ""); + cluster.set_env("HTTPS_PROXY", ""); + if matches!(layout, DistLayout::SingleNodeFourDrive) { + // This fresh, same-version fleet is safe to initialize at V3. The + // durable generation protocol is required for concurrent + // decommission progress updates and crash-recoverable movement. + for (key, value) in POOL_META_V3_ENV { + cluster.set_env(key, value); + } + } + for &(key, value) in extra_env { + cluster.set_env(key, value); + } + configure_node_logs(&mut cluster)?; + Ok(Self { + cluster, + pool_storage_roots, + owned_pool_dirs, + }) + } + + /// Start every node with a specific `rustfs` binary, keeping the allocated + /// data directories. Used to seed an old on-disk format before upgrading. + pub async fn start_from_binary(&mut self, binary: &Path) -> TestResult { + self.cluster.start_with_binary(binary).await?; + wait_for_ready(&self.cluster).await?; + Ok(()) + } + + /// Stop every node and bring the same data directories up on the workspace + /// binary (direct upgrade). + pub async fn restart_with_current_binary(&mut self) -> TestResult { + self.cluster.stop(); + self.cluster.start().await?; + wait_for_ready(&self.cluster).await?; + Ok(()) + } + + /// Planned topology changes use graceful shutdown. Crash semantics are + /// exercised separately by the chaos cases. + pub async fn restart_current_binary_gracefully(&mut self) -> TestResult { + self.stop_all_gracefully().await?; + self.cluster.start().await?; + wait_for_ready(&self.cluster).await?; + Ok(()) + } + + /// Replace one running node with the workspace binary (rolling upgrade). + pub async fn replace_node_with_current_binary(&mut self, node_idx: usize) -> TestResult { + self.cluster.stop_node(node_idx)?; + self.cluster.start_node(node_idx).await?; + wait_for_ready(&self.cluster).await?; + Ok(()) + } + + pub fn client_with_credentials(&self, node_idx: usize, access_key: &str, secret_key: &str) -> TestResult { + if node_idx >= self.cluster.nodes.len() { + return Err("node_idx is invalid".into()); + } + Ok(Client::from_conf(build_test_s3_config( + &self.cluster.nodes[node_idx].url, + access_key, + secret_key, + None, + "cluster-iam-test", + ))) + } + + pub async fn append_pool_and_restart(&mut self) -> TestResult { + self.stop_all_gracefully().await?; + let node_idx = self.cluster.append_single_node_pool().await?; + let roots = self + .pool_storage_roots + .as_deref() + .ok_or("pool expansion requires configured isolated pool filesystems")?; + self.owned_pool_dirs + .push(relocate_pool_storage(&mut self.cluster, node_idx, roots)?); + configure_node_logs(&mut self.cluster)?; + self.cluster.start().await?; + wait_for_ready(&self.cluster).await?; + Ok(node_idx) + } + + async fn stop_all_gracefully(&mut self) -> TestResult { + for node_idx in 0..self.cluster.nodes.len() { + self.cluster.stop_node_gracefully(node_idx).await?; + } + Ok(()) + } + + /// Expand the durable seed into four single-node pools through three + /// serialized additions. Callers can seed objects before this step so a + /// later pool-0 decommission proves that user data crosses pool boundaries. + pub async fn expand_to_four_pools(&mut self) -> TestResult { + for _ in 0..3 { + self.append_pool_and_restart().await?; + } + self.restart_current_binary_gracefully().await?; + Ok(()) + } + + pub async fn start_replication_pair() -> TestResult<(Self, Self)> { + let mut extra: Vec<(&str, &str)> = replication_fast_env(); + extra.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV); + extra.extend_from_slice(FAST_DATA_USAGE_SCANNER_ENV); + let source = Self::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?; + let target = Self::start_with_env(DistLayout::FourNodeFourDisk, &extra).await?; + Ok((source, target)) + } + + pub fn client(&self, node_idx: usize) -> TestResult { + self.cluster.create_s3_client(node_idx) + } + + pub fn clients(&self) -> TestResult> { + self.cluster.create_all_clients() + } + + pub async fn create_bucket(&self, bucket: &str) -> TestResult { + self.cluster.create_test_bucket(bucket).await + } +} + +impl Drop for DistCluster { + fn drop(&mut self) { + self.cluster.stop(); + for path in &self.owned_pool_dirs { + if let Err(error) = std::fs::remove_dir_all(path) { + eprintln!("failed to clean up isolated distributed E2E pool directory {}: {error}", path.display()); + } + } + } +} + +fn configured_pool_storage_roots() -> TestResult> { + let raw = std::env::var_os(POOL_ROOTS_ENV).ok_or_else(|| { + format!( + "{POOL_ROOTS_ENV} must name {NODE_COUNT} isolated filesystems for pool expansion, decommission, and rebalance tests" + ) + })?; + validate_pool_storage_roots(&raw) +} + +fn validate_pool_storage_roots(raw: &std::ffi::OsStr) -> TestResult> { + let roots: Vec = std::env::split_paths(raw).collect(); + if roots.len() != NODE_COUNT { + return Err(format!("{POOL_ROOTS_ENV} must contain exactly {NODE_COUNT} paths, got {}", roots.len()).into()); + } + + let mut canonical_roots = Vec::with_capacity(roots.len()); + for root in roots { + if !root.is_absolute() { + return Err(format!("{POOL_ROOTS_ENV} path must be absolute: {}", root.display()).into()); + } + let canonical = root + .canonicalize() + .map_err(|error| format!("{POOL_ROOTS_ENV} path {} is unavailable: {error}", root.display()))?; + if !canonical.is_dir() { + return Err(format!("{POOL_ROOTS_ENV} path is not a directory: {}", canonical.display()).into()); + } + if canonical_roots.contains(&canonical) { + return Err(format!("{POOL_ROOTS_ENV} contains a duplicate path: {}", canonical.display()).into()); + } + canonical_roots.push(canonical); + } + + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + + let devices: std::collections::BTreeSet = canonical_roots + .iter() + .map(|root| std::fs::metadata(root).map(|metadata| metadata.dev())) + .collect::>()?; + if devices.len() != canonical_roots.len() { + return Err(format!( + "{POOL_ROOTS_ENV} paths must be backed by distinct filesystems; found {} devices for {} paths", + devices.len(), + canonical_roots.len() + ) + .into()); + } + } + + Ok(canonical_roots) +} + +fn relocate_pool_storage(cluster: &mut RustFSTestClusterEnvironment, node_idx: usize, roots: &[PathBuf]) -> TestResult { + let root = roots + .get(node_idx) + .ok_or_else(|| format!("no isolated pool filesystem configured for node {node_idx}"))?; + let cluster_name = Path::new(&cluster.temp_dir) + .file_name() + .ok_or("cluster temp directory omitted a basename")?; + let pool_run_root = root.join(cluster_name); + let node_root = pool_run_root.join(format!("node{node_idx}")); + let data_dirs: Vec = (0..cluster.topology.drives_per_node) + .map(|drive| node_root.join(format!("drive{drive}")).to_string_lossy().into_owned()) + .collect(); + for data_dir in &data_dirs { + std::fs::create_dir_all(data_dir)?; + } + let node = cluster + .nodes + .get_mut(node_idx) + .ok_or_else(|| format!("cannot relocate missing cluster node {node_idx}"))?; + node.data_dir = data_dirs[0].clone(); + node.data_dirs = data_dirs; + Ok(pool_run_root) +} + +fn configure_node_logs(cluster: &mut RustFSTestClusterEnvironment) -> TestResult { + let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else { + return Ok(()); + }; + std::fs::create_dir_all(&log_dir)?; + let cluster_id = Uuid::new_v4().simple().to_string(); + for node_idx in 0..cluster.nodes.len() { + let path = Path::new(&log_dir).join(format!("cluster-{cluster_id}-node-{node_idx}.log")); + cluster.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?; + } + Ok(()) +} + +pub(crate) fn unique_bucket(prefix: &str) -> String { + let id = Uuid::new_v4().simple().to_string(); + format!("{prefix}-{}", &id[..12]) +} + +pub(crate) fn sha256_hex(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +pub(crate) fn payload_for(key: &str, size: usize) -> Vec { + let seed = key.as_bytes(); + (0..size) + .map(|idx| seed.get(idx % seed.len()).copied().unwrap_or(0) ^ (idx as u8)) + .collect() +} + +pub(crate) async fn put_object(client: &Client, bucket: &str, key: &str, body: Vec) -> TestResult { + client + .put_object() + .bucket(bucket) + .key(key) + .body(ByteStream::from(body)) + .send() + .await?; + Ok(()) +} + +pub(crate) async fn get_object_bytes(client: &Client, bucket: &str, key: &str) -> TestResult> { + let output = client.get_object().bucket(bucket).key(key).send().await?; + Ok(output.body.collect().await?.into_bytes().to_vec()) +} + +pub(crate) async fn assert_object_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult { + let got = get_object_bytes(client, bucket, key).await?; + if got.as_slice() != expected { + return Err(format!( + "object {bucket}/{key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}", + expected.len(), + sha256_hex(expected), + got.len(), + sha256_hex(&got) + ) + .into()); + } + Ok(()) +} + +pub(crate) async fn put_inventory( + client: &Client, + bucket: &str, + count: usize, + size: usize, +) -> TestResult>> { + let mut inventory = BTreeMap::new(); + for idx in 0..count { + let key = format!("obj-{idx:04}"); + let body = payload_for(&key, size); + put_object(client, bucket, &key, body.clone()).await?; + inventory.insert(key, body); + } + Ok(inventory) +} + +/// Retry only transport-level service availability failures while a data +/// movement operation changes the pool map. Generic InternalError responses +/// remain fatal because accepting them would hide server defects. +pub(crate) async fn put_inventory_retrying( + client: &Client, + bucket: &str, + count: usize, + size: usize, + timeout: Duration, +) -> TestResult>> { + let mut inventory = BTreeMap::new(); + for idx in 0..count { + let key = format!("obj-{idx:04}"); + let body = payload_for(&key, size); + retrying_put(client, bucket, &key, body.clone(), timeout).await?; + inventory.insert(key, body); + } + Ok(inventory) +} + +pub(crate) async fn assert_inventory(client: &Client, bucket: &str, inventory: &BTreeMap>) -> TestResult { + for (key, expected) in inventory { + assert_object_bytes(client, bucket, key, expected).await?; + } + Ok(()) +} + +pub(crate) async fn enable_versioning(client: &Client, bucket: &str) -> TestResult { + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + Ok(()) +} + +pub(crate) async fn wait_until(timeout: Duration, mut probe: F, label: &str) -> TestResult +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let deadline = Instant::now() + timeout; + let mut delay = Duration::from_millis(50); + loop { + let last_error = match probe().await { + Ok(true) => return Ok(()), + Ok(false) => format!("{label} still false"), + Err(error) => error.to_string(), + }; + if Instant::now() >= deadline { + return Err(format!("{label} did not become true within {timeout:?}: {last_error}").into()); + } + sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(1)); + } +} + +pub(crate) async fn cluster_admin( + cluster: &RustFSTestClusterEnvironment, + method: Method, + path_and_query: &str, + body: Option, +) -> TestResult<(StatusCode, String)> { + admin_request( + &cluster.nodes[0].url, + method, + path_and_query, + body, + &cluster.access_key, + &cluster.secret_key, + ) + .await +} + +pub(crate) async fn cluster_admin_ok( + cluster: &RustFSTestClusterEnvironment, + method: Method, + path_and_query: &str, + body: Option, +) -> TestResult { + let (status, response) = cluster_admin(cluster, method.clone(), path_and_query, body).await?; + if !status.is_success() { + return Err(format!("{method} {path_and_query} failed: {status} {response}").into()); + } + Ok(response) +} + +pub(crate) async fn wait_for_ready(cluster: &RustFSTestClusterEnvironment) -> TestResult { + let client = local_http_client(); + for node in &cluster.nodes { + let url = format!("{}/health/ready", node.url); + wait_until( + Duration::from_secs(30), + || { + let client = client.clone(); + let url = url.clone(); + async move { + match client.get(&url).send().await { + Ok(response) if response.status().is_success() => Ok(true), + _ => Ok(false), + } + } + }, + &format!("node {} ready", node.address), + ) + .await?; + } + Ok(()) +} + +pub(crate) async fn set_remote_target( + source: &RustFSTestClusterEnvironment, + source_bucket: &str, + target: &RustFSTestClusterEnvironment, + target_bucket: &str, +) -> TestResult { + let body = serde_json::json!({ + "endpoint": target.nodes[0].address, + "credentials": { + "accessKey": target.access_key, + "secretKey": target.secret_key + }, + "targetbucket": target_bucket, + "secure": false, + "type": "replication" + }); + let url = format!( + "{}/rustfs/admin/v3/set-remote-target?bucket={}", + source.nodes[0].url, + urlencoding::encode(source_bucket) + ); + let response = signed_request( + Method::PUT, + &url, + &source.access_key, + &source.secret_key, + Some(body.to_string().into_bytes()), + Some("application/json"), + ) + .await?; + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("set remote target failed: {status} {body}").into()); + } + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +pub(crate) async fn put_bucket_replication(source: &RustFSTestClusterEnvironment, bucket: &str, target_arn: &str) -> TestResult { + let body = format!( + r#" + + + rule-1 + 1 + Enabled + + Enabled + + + Enabled + + + {target_arn} + + +"# + ); + let url = format!("{}/{bucket}?replication", source.nodes[0].url); + let response = signed_request( + Method::PUT, + &url, + &source.access_key, + &source.secret_key, + Some(body.into_bytes()), + Some("application/xml"), + ) + .await?; + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("put bucket replication failed: {status} {body}").into()); + } + Ok(()) +} + +pub(crate) async fn wait_for_replicated_bytes( + client: &Client, + bucket: &str, + key: &str, + expected: &[u8], + timeout: Duration, +) -> TestResult { + wait_until( + timeout, + || async { + match get_object_bytes(client, bucket, key).await { + Ok(got) if got.as_slice() == expected => Ok(true), + Ok(_) => Ok(false), + Err(error) => { + let message = error.to_string(); + if message.contains("NoSuchKey") || message.contains("NotFound") { + Ok(false) + } else { + Err(error) + } + } + } + }, + &format!("replicated object {bucket}/{key}"), + ) + .await +} + +pub(crate) async fn set_bucket_quota(cluster: &RustFSTestClusterEnvironment, bucket: &str, quota_bytes: u64) -> TestResult { + wait_until( + Duration::from_secs(30), + || async { + let (status, _) = + cluster_admin(cluster, Method::GET, &format!("/rustfs/admin/v3/quota-stats/{bucket}"), None).await?; + Ok(status.is_success() || status == StatusCode::NOT_FOUND) + }, + "quota stats ready", + ) + .await?; + let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string(); + wait_until( + Duration::from_secs(30), + || async { + let (status, response) = + cluster_admin(cluster, Method::PUT, &format!("/rustfs/admin/v3/quota/{bucket}"), Some(body.clone())).await?; + if status.is_success() { + return Ok(true); + } + if status == StatusCode::SERVICE_UNAVAILABLE { + return Ok(false); + } + Err(format!("failed to set quota for {bucket}: {status} {response}").into()) + }, + "set hard quota", + ) + .await +} + +/// Start decommission and fail closed unless the admin API acknowledges it. +pub(crate) async fn start_decommission(cluster: &RustFSTestClusterEnvironment, pool_id: usize) -> TestResult { + let path = format!("/rustfs/admin/v3/pools/decommission?pool={pool_id}&by-id=true"); + let deadline = Instant::now() + Duration::from_secs(45); + loop { + let (status, response) = cluster_admin(cluster, Method::POST, &path, None).await?; + if status.is_success() { + return Ok(()); + } + if status == StatusCode::INTERNAL_SERVER_ERROR + && response.contains("requires a live fleet capability proof") + && Instant::now() < deadline + { + sleep(Duration::from_secs(1)).await; + continue; + } + return Err(format!("POST {path} did not start decommission: {status} {response}").into()); + } +} + +pub(crate) async fn decommission_status_json(cluster: &RustFSTestClusterEnvironment) -> TestResult { + let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/decommission/status", None).await?; + Ok(serde_json::from_str(&body)?) +} + +fn pool_entry(status: &serde_json::Value, pool_id: usize) -> Option<&serde_json::Value> { + if let Some(pools) = status.get("pools").and_then(serde_json::Value::as_array) { + return pools + .iter() + .find(|pool| pool.get("id").and_then(serde_json::Value::as_u64) == Some(pool_id as u64)); + } + if status.get("id").and_then(serde_json::Value::as_u64) == Some(pool_id as u64) { + Some(status) + } else { + None + } +} + +fn nonzero_u64(value: Option<&serde_json::Value>) -> bool { + value.and_then(serde_json::Value::as_u64).is_some_and(|count| count > 0) +} + +fn decommission_failure(pool: &serde_json::Value) -> Option { + let info = pool.get("decommissionInfo"); + let flagged = |key: &str| info.and_then(|value| value.get(key)).and_then(serde_json::Value::as_bool) == Some(true); + let terminal_flag = flagged("failed") + || flagged("canceled") + || pool + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|status| status.eq_ignore_ascii_case("failed") || status.eq_ignore_ascii_case("canceled")); + let object_failures = nonzero_u64(info.and_then(|value| value.get("objectsDecommissionedFailed"))); + let byte_failures = nonzero_u64(info.and_then(|value| value.get("bytesDecommissionedFailed"))); + let unresolved = info + .and_then(|value| value.get("unresolvedEntries")) + .and_then(serde_json::Value::as_array) + .is_some_and(|entries| !entries.is_empty()); + terminal_flag + .then(|| "decommission reported failed or canceled".to_string()) + .or_else(|| object_failures.then(|| "decommission reported failed objects".to_string())) + .or_else(|| byte_failures.then(|| "decommission reported failed bytes".to_string())) + .or_else(|| unresolved.then(|| "decommission reported unresolved entries".to_string())) +} + +pub(crate) fn decommission_active(status: &serde_json::Value, pool_id: usize) -> TestResult { + let pool = pool_entry(status, pool_id).ok_or_else(|| format!("pool {pool_id} missing from decommission status: {status}"))?; + if let Some(reason) = decommission_failure(pool) { + return Err(format!("{reason}: {pool}").into()); + } + let info = pool + .get("decommissionInfo") + .ok_or_else(|| format!("pool {pool_id} has no decommissionInfo: {pool}"))?; + let status_text = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or(""); + let pool_status = pool.get("poolStatus").and_then(serde_json::Value::as_str).unwrap_or(""); + let queued = info.get("queued").and_then(serde_json::Value::as_bool) == Some(true); + Ok(queued || status_text.eq_ignore_ascii_case("running") || pool_status.eq_ignore_ascii_case("decommissioning")) +} + +pub(crate) fn decommission_running_with_progress(status: &serde_json::Value, pool_id: usize) -> TestResult { + let pool = pool_entry(status, pool_id).ok_or_else(|| format!("pool {pool_id} missing from decommission status: {status}"))?; + if let Some(reason) = decommission_failure(pool) { + return Err(format!("{reason}: {pool}").into()); + } + let info = pool + .get("decommissionInfo") + .ok_or_else(|| format!("pool {pool_id} has no decommissionInfo: {pool}"))?; + let status_text = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or(""); + let pool_status = pool.get("poolStatus").and_then(serde_json::Value::as_str).unwrap_or(""); + let running = status_text.eq_ignore_ascii_case("running") || pool_status.eq_ignore_ascii_case("decommissioning"); + let progressed = nonzero_u64(info.get("objectsDecommissioned")) || nonzero_u64(info.get("bytesDecommissioned")); + Ok(running && progressed) +} + +pub(crate) fn decommission_complete(status: &serde_json::Value, pool_id: usize) -> TestResult { + let pool = pool_entry(status, pool_id).ok_or_else(|| format!("pool {pool_id} missing from decommission status: {status}"))?; + if let Some(reason) = decommission_failure(pool) { + return Err(format!("{reason}: {pool}").into()); + } + let info = pool + .get("decommissionInfo") + .ok_or_else(|| format!("pool {pool_id} has no decommissionInfo: {pool}"))?; + let complete = info.get("complete").and_then(serde_json::Value::as_bool) == Some(true); + let status_text = pool.get("status").and_then(serde_json::Value::as_str).unwrap_or(""); + let pool_status = pool.get("poolStatus").and_then(serde_json::Value::as_str).unwrap_or(""); + let terminal = status_text.eq_ignore_ascii_case("complete") && pool_status.eq_ignore_ascii_case("decommissioned"); + let moved_data = nonzero_u64(info.get("objectsDecommissioned")) && nonzero_u64(info.get("bytesDecommissioned")); + Ok(complete && terminal && moved_data) +} + +pub(crate) async fn wait_for_decommission_active( + cluster: &RustFSTestClusterEnvironment, + pool_id: usize, + timeout: Duration, +) -> TestResult { + wait_for_decommission_state(cluster, pool_id, timeout, "active", decommission_active).await +} + +pub(crate) async fn wait_for_decommission_running_with_progress( + cluster: &RustFSTestClusterEnvironment, + pool_id: usize, + timeout: Duration, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = decommission_status_json(cluster).await?; + if decommission_running_with_progress(&status, pool_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!( + "decommission did not become active with non-zero progress within {timeout:?}; last status: {status}" + ) + .into()); + } + sleep(Duration::from_millis(100)).await; + } +} + +pub(crate) async fn wait_for_decommission_complete( + cluster: &RustFSTestClusterEnvironment, + pool_id: usize, + timeout: Duration, +) -> TestResult { + wait_for_decommission_state(cluster, pool_id, timeout, "complete with non-zero progress", decommission_complete).await +} + +async fn wait_for_decommission_state( + cluster: &RustFSTestClusterEnvironment, + pool_id: usize, + timeout: Duration, + expected: &str, + predicate: fn(&serde_json::Value, usize) -> TestResult, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = decommission_status_json(cluster).await?; + if predicate(&status, pool_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!("decommission did not become {expected} within {timeout:?}; last status: {status}").into()); + } + sleep(Duration::from_secs(1)).await; + } +} + +pub(crate) async fn start_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult { + let path = "/rustfs/admin/v3/rebalance/start"; + let deadline = Instant::now() + Duration::from_secs(45); + let response = loop { + let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?; + if status.is_success() { + break response; + } + if status == StatusCode::INTERNAL_SERVER_ERROR + && response.contains("requires a live fleet capability proof") + && Instant::now() < deadline + { + sleep(Duration::from_secs(1)).await; + continue; + } + return Err(format!("POST {path} did not start rebalance: {status} {response}").into()); + }; + let parsed: serde_json::Value = serde_json::from_str(&response)?; + parsed + .get("id") + .and_then(serde_json::Value::as_str) + .filter(|id| !id.is_empty()) + .map(str::to_string) + .ok_or_else(|| format!("rebalance start response omitted id: {response}").into()) +} + +pub(crate) async fn rebalance_status_json(cluster: &RustFSTestClusterEnvironment) -> TestResult { + let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/rebalance/status", None).await?; + Ok(serde_json::from_str(&body)?) +} + +fn validate_rebalance_status<'a>(status: &'a serde_json::Value, expected_id: &str) -> TestResult<&'a [serde_json::Value]> { + let id = status + .get("id") + .and_then(serde_json::Value::as_str) + .ok_or("rebalance status omitted id")?; + if id != expected_id { + return Err(format!("rebalance status id changed: expected {expected_id}, got {id}").into()); + } + let pools = status + .get("pools") + .and_then(serde_json::Value::as_array) + .filter(|pools| !pools.is_empty()) + .ok_or("rebalance status omitted non-empty pools")?; + for pool in pools { + if pool.get("status").and_then(serde_json::Value::as_str).is_some_and(|status| { + ["failed", "stopped", "canceled", "cancelled"] + .iter() + .any(|terminal| status.eq_ignore_ascii_case(terminal)) + }) { + return Err(format!("rebalance entered an unsuccessful terminal state: {status}").into()); + } + if pool.get("stopping").and_then(serde_json::Value::as_bool) == Some(true) { + return Err(format!("rebalance entered stopping state: {status}").into()); + } + if pool + .get("lastError") + .and_then(serde_json::Value::as_str) + .is_some_and(|error| !error.is_empty()) + { + return Err(format!("rebalance reported lastError: {status}").into()); + } + if nonzero_u64(pool.get("cleanupWarnings").and_then(|warnings| warnings.get("count"))) { + return Err(format!("rebalance reported cleanup warnings: {status}").into()); + } + } + let propagation_failed = status.get("stopPropagation").is_some_and(|propagation| { + propagation + .get("failedPeers") + .and_then(serde_json::Value::as_array) + .is_some_and(|peers| !peers.is_empty()) + || propagation + .get("terminalReloadFailedPeers") + .and_then(serde_json::Value::as_array) + .is_some_and(|peers| !peers.is_empty()) + || propagation.get("pendingTerminalReload").and_then(serde_json::Value::as_bool) == Some(true) + }); + if propagation_failed { + return Err(format!("rebalance stop propagation is incomplete: {status}").into()); + } + Ok(pools) +} + +pub(crate) fn rebalance_active(status: &serde_json::Value, expected_id: &str) -> TestResult { + Ok(validate_rebalance_status(status, expected_id)?.iter().any(|pool| { + pool.get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| value.eq_ignore_ascii_case("started")) + })) +} + +pub(crate) fn rebalance_running_with_progress(status: &serde_json::Value, expected_id: &str) -> TestResult { + Ok(validate_rebalance_status(status, expected_id)?.iter().any(|pool| { + let started = pool + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| value.eq_ignore_ascii_case("started")); + let progress = pool.get("progress"); + started + && (nonzero_u64(progress.and_then(|value| value.get("objects"))) + || nonzero_u64(progress.and_then(|value| value.get("versions"))) + || nonzero_u64(progress.and_then(|value| value.get("bytes")))) + })) +} + +pub(crate) fn rebalance_complete(status: &serde_json::Value, expected_id: &str) -> TestResult { + let pools = validate_rebalance_status(status, expected_id)?; + let completed: Vec<&serde_json::Value> = pools + .iter() + .filter(|pool| { + pool.get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| value.eq_ignore_ascii_case("completed")) + }) + .collect(); + let all_terminal = pools.iter().all(|pool| { + pool.get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|value| value.eq_ignore_ascii_case("completed") || value.eq_ignore_ascii_case("none")) + }); + let any_progress = completed.iter().any(|pool| { + let progress = pool.get("progress"); + nonzero_u64(progress.and_then(|value| value.get("objects"))) + || nonzero_u64(progress.and_then(|value| value.get("versions"))) + || nonzero_u64(progress.and_then(|value| value.get("bytes"))) + }); + Ok(all_terminal && !completed.is_empty() && any_progress) +} + +pub(crate) async fn wait_for_rebalance_active( + cluster: &RustFSTestClusterEnvironment, + expected_id: &str, + timeout: Duration, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = rebalance_status_json(cluster).await?; + if rebalance_active(&status, expected_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!("rebalance did not become active within {timeout:?}; last status: {status}").into()); + } + sleep(Duration::from_secs(1)).await; + } +} + +pub(crate) async fn wait_for_rebalance_running_with_progress( + cluster: &RustFSTestClusterEnvironment, + expected_id: &str, + timeout: Duration, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = rebalance_status_json(cluster).await?; + if rebalance_running_with_progress(&status, expected_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!( + "rebalance did not become active with non-zero progress within {timeout:?}; last status: {status}" + ) + .into()); + } + sleep(Duration::from_millis(100)).await; + } +} + +pub(crate) async fn wait_for_rebalance_complete( + cluster: &RustFSTestClusterEnvironment, + expected_id: &str, + timeout: Duration, +) -> TestResult { + let deadline = Instant::now() + timeout; + loop { + let status = rebalance_status_json(cluster).await?; + if rebalance_complete(&status, expected_id)? { + return Ok(()); + } + if Instant::now() >= deadline { + return Err( + format!("rebalance did not complete with non-zero progress within {timeout:?}; last status: {status}").into(), + ); + } + sleep(Duration::from_secs(1)).await; + } +} + +pub(crate) async fn list_pools_json(cluster: &RustFSTestClusterEnvironment) -> TestResult { + let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/pools/list", None).await?; + Ok(serde_json::from_str(&body)?) +} + +pub(crate) async fn retrying_put(client: &Client, bucket: &str, key: &str, body: Vec, timeout: Duration) -> TestResult { + wait_until( + timeout, + || { + let client = client.clone(); + let bucket = bucket.to_string(); + let key = key.to_string(); + let body = body.clone(); + async move { + match put_object(&client, &bucket, &key, body).await { + Ok(()) => Ok(true), + Err(error) => { + let message = error.to_string(); + if message.contains("SlowDown") || message.contains("ServiceUnavailable") || message.contains("503") { + Ok(false) + } else { + Err(error) + } + } + } + } + }, + &format!("put {bucket}/{key} during data movement"), + ) + .await +} + +pub(crate) async fn retrying_get_equals( + client: &Client, + bucket: &str, + key: &str, + expected: &[u8], + timeout: Duration, +) -> TestResult { + wait_until( + timeout, + || async { + match get_object_bytes(client, bucket, key).await { + Ok(got) if got.as_slice() == expected => Ok(true), + Ok(_) => Ok(false), + Err(error) => { + let message = error.to_string(); + if message.contains("NoSuchKey") + || message.contains("SlowDown") + || message.contains("ServiceUnavailable") + || message.contains("503") + { + Ok(false) + } else { + Err(error) + } + } + } + }, + &format!("get {bucket}/{key} during data movement"), + ) + .await +} + +#[tokio::test] +async fn append_single_node_pool_extends_ellipses_volumes() { + let mut env = RustFSTestClusterEnvironment::with_topology(ClusterTopology::per_node_pools(2, vec![vec![0], vec![1]])) + .await + .expect("two-pool seed topology"); + assert_eq!(env.rustfs_volumes_arg().split(' ').count(), 2); + + let added = env.append_single_node_pool().await.expect("append third pool"); + assert_eq!(added, 2); + assert_eq!(env.nodes.len(), 3); + assert_eq!(env.nodes[2].pool_idx, 2); + assert_eq!(env.nodes[2].data_dirs.len(), 2); + let volumes = env.rustfs_volumes_arg(); + assert_eq!(volumes.split(' ').count(), 3, "expected three pool arguments, got: {volumes}"); + assert!(volumes.contains("/drive{0...1}"), "expanded layout must keep drive ellipses: {volumes}"); +} + +#[tokio::test] +async fn append_single_node_pool_rejects_striped_single_pool() { + let mut env = RustFSTestClusterEnvironment::new(4).await.expect("four-node single pool"); + let err = env + .append_single_node_pool() + .await + .expect_err("a striped single pool cannot gain a localhost pool"); + let message = err.to_string(); + assert!( + message.contains("drives_per_node") || message.contains("one node per pool"), + "unexpected error: {message}" + ); +} + +#[test] +fn pool_storage_roots_require_exactly_four_unique_paths() { + let temp_root = std::env::temp_dir(); + let too_few = std::env::join_paths([temp_root.as_path()]).expect("join one path"); + let error = validate_pool_storage_roots(&too_few).expect_err("one pool root must be rejected"); + assert!(error.to_string().contains("exactly 4 paths")); + + let duplicates = std::env::join_paths([ + temp_root.as_path(), + temp_root.as_path(), + temp_root.as_path(), + temp_root.as_path(), + ]) + .expect("join duplicate paths"); + let error = validate_pool_storage_roots(&duplicates).expect_err("duplicate pool roots must be rejected"); + assert!(error.to_string().contains("duplicate path")); +} + +#[cfg(unix)] +#[tokio::test] +async fn pool_storage_roots_reject_distinct_paths_on_the_same_device() { + let env = RustFSTestClusterEnvironment::new(NODE_COUNT) + .await + .expect("create same-device pool root fixture"); + let roots: Vec = env.nodes.iter().map(|node| PathBuf::from(&node.data_dir)).collect(); + let joined = std::env::join_paths(&roots).expect("join same-device paths"); + let error = validate_pool_storage_roots(&joined).expect_err("same-device pool roots must be rejected"); + assert!(error.to_string().contains("distinct filesystems"), "unexpected error: {error}"); +} + +#[test] +fn decommission_complete_requires_terminal_status_and_clean_counters() { + let status = serde_json::json!({ + "pools": [ + { + "id": 0, + "status": "complete", + "poolStatus": "decommissioned", + "decommissionInfo": { + "complete": true, + "failed": false, + "canceled": false, + "objectsDecommissioned": 1, + "bytesDecommissioned": 1024 + } + }, + { "id": 1, "status": "none", "poolStatus": "active" } + ] + }); + assert!(decommission_complete(&status, 0).expect("complete fixture should be accepted")); + assert!(decommission_complete(&status, 1).is_err()); + + for missing_counter in ["objectsDecommissioned", "bytesDecommissioned"] { + let mut one_sided = status.clone(); + one_sided["pools"][0]["decommissionInfo"][missing_counter] = serde_json::json!(0); + assert!( + !decommission_complete(&one_sided, 0).expect("one-sided progress fixture should be readable"), + "completion must require both movement counters; zeroed {missing_counter}" + ); + } +} + +#[test] +fn decommission_overlap_requires_running_state_and_progress() { + let mut status = serde_json::json!({ + "pools": [{ + "id": 0, + "status": "queued", + "poolStatus": "active", + "decommissionInfo": { + "queued": true, + "objectsDecommissioned": 1, + "bytesDecommissioned": 1024 + } + }] + }); + assert!( + !decommission_running_with_progress(&status, 0).expect("queued fixture should be readable"), + "queued work is not temporal overlap" + ); + status["pools"][0]["status"] = serde_json::json!("running"); + assert!(decommission_running_with_progress(&status, 0).expect("running fixture should be readable")); + status["pools"][0]["decommissionInfo"]["objectsDecommissioned"] = serde_json::json!(0); + status["pools"][0]["decommissionInfo"]["bytesDecommissioned"] = serde_json::json!(0); + assert!( + !decommission_running_with_progress(&status, 0).expect("zero-progress fixture should be readable"), + "running state alone does not prove movement started" + ); +} + +#[test] +fn rebalance_active_treats_started_as_in_progress() { + let started = serde_json::json!({ "id": "run-1", "pools": [{ "id": 0, "status": "Started", "stopping": false }] }); + let done = serde_json::json!({ "id": "run-1", "pools": [{ "id": 0, "status": "Completed", "stopping": false }] }); + assert!(rebalance_active(&started, "run-1").unwrap()); + assert!(!rebalance_active(&done, "run-1").unwrap()); +} + +#[test] +fn rebalance_overlap_requires_started_state_and_progress() { + let mut status = serde_json::json!({ + "id": "run-1", + "pools": [{ "id": 0, "status": "Started", "stopping": false, "progress": { "objects": 0, "bytes": 0 } }] + }); + assert!( + !rebalance_running_with_progress(&status, "run-1").expect("zero-progress fixture should be readable"), + "started state alone does not prove movement" + ); + status["pools"][0]["progress"]["objects"] = serde_json::json!(1); + assert!(rebalance_running_with_progress(&status, "run-1").expect("progress fixture should be readable")); + status["pools"][0]["status"] = serde_json::json!("Completed"); + assert!( + !rebalance_running_with_progress(&status, "run-1").expect("completed fixture should be readable"), + "completed movement is not temporal overlap" + ); +} + +#[test] +fn rebalance_complete_accepts_non_participating_pools_but_requires_progress() { + let completed = serde_json::json!({ + "id": "run-1", + "pools": [ + { "id": 0, "status": "Completed", "stopping": false, "progress": { "objects": 2, "bytes": 1024 } }, + { "id": 1, "status": "None", "stopping": false, "progress": null }, + { "id": 2, "status": "None", "stopping": false, "progress": null }, + { "id": 3, "status": "None", "stopping": false, "progress": null } + ] + }); + assert!(rebalance_complete(&completed, "run-1").unwrap()); + + let no_movement = serde_json::json!({ + "id": "run-1", + "pools": [ + { "id": 0, "status": "None", "stopping": false, "progress": null }, + { "id": 1, "status": "None", "stopping": false, "progress": null } + ] + }); + assert!(!rebalance_complete(&no_movement, "run-1").unwrap()); +} diff --git a/crates/e2e_test/src/distributed/mod.rs b/crates/e2e_test/src/distributed/mod.rs new file mode 100644 index 000000000..108721f4c --- /dev/null +++ b/crates/e2e_test/src/distributed/mod.rs @@ -0,0 +1,35 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! 4-node 4-drive distributed e2e coverage. +//! +//! Selected by `[profile.e2e-distributed]` and run from +//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because +//! each case starts four real `rustfs` processes. + +mod chaos_test; +mod concurrency_stability_test; +mod concurrent_data_movement_test; +mod data_integrity_movement_test; +mod expand_decommission_rebalance_test; +mod extra_test; +mod harness; +mod object_lock_test; +mod observability_test; +mod replication_quota_test; +mod s3_basic_test; +mod s3_during_data_movement_test; +mod site_replication_test; +mod upgrade_test; +mod versioning_test; diff --git a/crates/e2e_test/src/distributed/object_lock_test.rs b/crates/e2e_test/src/distributed/object_lock_test.rs new file mode 100644 index 000000000..b6681e624 --- /dev/null +++ b/crates/e2e_test/src/distributed/object_lock_test.rs @@ -0,0 +1,219 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket}; +use crate::common::init_logging; +use crate::object_lock::common::{ + delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention, +}; +use aws_sdk_s3::Client; +use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::error::SdkError; +use aws_sdk_s3::operation::delete_object::DeleteObjectError; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{ + DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode, + ObjectLockRule, +}; +use chrono::{Duration as ChronoDuration, Utc}; + +fn delete_denied(error: &SdkError, context: &str) -> TestResult { + let code = error.as_service_error().and_then(ProvideErrorMetadata::code); + if code == Some("AccessDenied") { + Ok(()) + } else { + Err(format!("{context}: expected AccessDenied, got {error:?}").into()) + } +} + +async fn expect_versioned_delete_denied( + client: &Client, + bucket: &str, + key: &str, + version_id: &str, + bypass: bool, + context: &str, +) -> TestResult { + match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await { + Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()), + Err(error) => delete_denied(error.as_ref(), context), + } +} + +#[tokio::test] +async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let client = dist.client(0)?; + let peer = dist.client(2)?; + let bucket = unique_bucket("objlock"); + + client + .create_bucket() + .bucket(&bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + + let retain_until = Utc::now() + ChronoDuration::days(1); + + let compliance_key = "compliance.bin"; + let compliance_version = put_object_with_retention( + &client, + &bucket, + compliance_key, + b"locked-compliance", + ObjectLockRetentionMode::Compliance, + retain_until, + ) + .await?; + + // Unversioned DELETE is allowed: it only creates a delete marker. WORM + // applies to a specific version id. + let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?; + assert_eq!( + marker.delete_marker(), + Some(true), + "unversioned DELETE on a locked object must create a delete marker" + ); + + expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass") + .await?; + expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?; + + let governance_key = "governance.bin"; + let governance_version = put_object_with_retention( + &client, + &bucket, + governance_key, + b"locked-governance", + ObjectLockRetentionMode::Governance, + retain_until, + ) + .await?; + + expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass") + .await?; + delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?; + let deleted_governance = peer + .head_object() + .bucket(&bucket) + .key(governance_key) + .version_id(&governance_version) + .send() + .await + .expect_err("GOVERNANCE bypass must remove the retained version"); + assert_eq!( + deleted_governance.raw_response().map(|response| response.status().as_u16()), + Some(404), + "deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}" + ); + + let hold_key = "legal-hold.bin"; + let hold_version = + put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?; + expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?; + expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?; + + Ok(()) +} + +#[tokio::test] +async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let writer = dist.client(0)?; + let reader = dist.client(3)?; + let bucket = unique_bucket("default-lock"); + + writer + .create_bucket() + .bucket(&bucket) + .object_lock_enabled_for_bucket(true) + .send() + .await?; + put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?; + + let key = "default-governance.bin"; + let put = writer + .put_object() + .bucket(&bucket) + .key(key) + .body(ByteStream::from_static(b"default retention payload")) + .send() + .await?; + let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?; + + let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?; + let default_retention = config + .object_lock_configuration() + .and_then(|configuration| configuration.rule()) + .and_then(|rule| rule.default_retention()) + .ok_or("GetObjectLockConfiguration omitted default retention")?; + assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE")); + assert_eq!(default_retention.days(), Some(1)); + + let retention = reader + .get_object_retention() + .bucket(&bucket) + .key(key) + .version_id(version_id) + .send() + .await?; + let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?; + assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE")); + let retain_until = retention + .retain_until_date() + .ok_or("default retention omitted retain-until date")?; + assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future"); + + let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?; + assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled")); + expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass") + .await?; + + let plain_bucket = unique_bucket("no-lock"); + dist.create_bucket(&plain_bucket).await?; + let configuration = ObjectLockConfiguration::builder() + .object_lock_enabled(ObjectLockEnabled::Enabled) + .rule( + ObjectLockRule::builder() + .default_retention( + DefaultRetention::builder() + .mode(ObjectLockRetentionMode::Governance) + .days(1) + .build(), + ) + .build(), + ) + .build(); + let error = writer + .put_object_lock_configuration() + .bucket(&plain_bucket) + .object_lock_configuration(configuration) + .send() + .await + .expect_err("an unversioned bucket must reject Object Lock enablement"); + let service_error = error + .as_service_error() + .ok_or("non-lock bucket rejection was not an S3 service error")?; + assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}"); + assert_eq!( + service_error.message(), + Some("Object Lock configuration cannot be enabled on existing buckets"), + "unexpected error: {error:?}" + ); + + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/observability_test.rs b/crates/e2e_test/src/distributed/observability_test.rs new file mode 100644 index 000000000..bb722b92e --- /dev/null +++ b/crates/e2e_test/src/distributed/observability_test.rs @@ -0,0 +1,236 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready}; +use crate::common::{admin_request, init_logging, local_http_client}; +use aws_sdk_s3::operation::RequestId; +use aws_sdk_s3::primitives::ByteStream; +use bytes::Bytes; +use http::Method; +use http_body_util::{BodyExt, Empty}; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper::{Request, Response}; +use hyper_util::rt::TokioIo; +use local_ip_address::local_ip; +use rustfs_madmin::metrics::RealtimeMetrics; +use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; +use serde_json::Value; +use std::convert::Infallible; +use std::time::Duration; +use tokio::net::TcpListener; +use tokio::sync::mpsc; +use tokio::task::JoinHandle; +use tokio::time::{Instant, timeout}; + +async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver, JoinHandle<()>)> { + let listener = TcpListener::bind("0.0.0.0:0").await?; + let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port())); + let (tx, rx) = mpsc::unbounded_channel(); + let handle = tokio::spawn(async move { + loop { + let Ok((stream, _)) = listener.accept().await else { + return; + }; + let tx = tx.clone(); + tokio::spawn(async move { + let service = service_fn(move |request: Request| { + let tx = tx.clone(); + async move { + let method = request.method().clone(); + if let Ok(body) = request.into_body().collect().await + && method == Method::POST + && let Ok(payload) = serde_json::from_slice::(&body.to_bytes()) + { + if let Some(records) = payload["Records"].as_array() { + for entry in records { + let _ = tx.send(entry.clone()); + } + } else { + let _ = tx.send(payload); + } + } + Ok::<_, Infallible>(Response::new(Empty::::new())) + } + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + Ok((endpoint, rx, handle)) +} + +async fn wait_for_audit_entry( + rx: &mut mpsc::UnboundedReceiver, + bucket: &str, + key: &str, + request_id: &str, +) -> TestResult { + let deadline = Instant::now() + Duration::from_secs(30); + let mut seen = Vec::new(); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(format!( + "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}", + seen.len() + ) + .into()); + } + let entry = match timeout(remaining, rx.recv()).await { + Ok(Some(entry)) => entry, + Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()), + Err(_) => { + return Err(format!( + "audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}", + seen.len() + ) + .into()); + } + }; + if entry["api"]["name"].as_str() == Some("s3:PutObject") + && entry["api"]["bucket"].as_str() == Some(bucket) + && entry["api"]["object"].as_str() == Some(key) + && entry["requestID"].as_str() == Some(request_id) + { + return Ok(entry); + } + if seen.len() < 8 { + seen.push(format!( + "api={:?} bucket={:?} object={:?} requestID={:?}", + entry["api"]["name"].as_str(), + entry["api"]["bucket"].as_str(), + entry["api"]["object"].as_str(), + entry["requestID"].as_str() + )); + } + } +} + +#[tokio::test] +async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult { + init_logging(); + let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?; + let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization(); + let audit_env = [ + ("RUSTFS_AUDIT_ENABLE", "true"), + ("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"), + ("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()), + (ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()), + ]; + let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?; + for node_idx in 0..dist.cluster.nodes.len() { + let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir); + tokio::fs::create_dir_all(&queue_dir).await?; + dist.cluster + .set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?; + } + dist.cluster.start().await?; + wait_for_ready(&dist.cluster).await?; + + let http = local_http_client(); + for node in &dist.cluster.nodes { + for probe in ["ready", "live"] { + let response = http.get(format!("{}/health/{probe}", node.url)).send().await?; + assert!( + response.status().is_success(), + "node {} {probe} probe failed: {}", + node.address, + response.status() + ); + } + } + + let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?; + let info: Value = serde_json::from_str(&info_body)?; + let servers = info["info"]["servers"] + .as_array() + .ok_or_else(|| format!("admin info omitted servers: {info}"))?; + assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}"); + + let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?; + let storage: Value = serde_json::from_str(&storage_body)?; + let disks = storage["info"]["disks"] + .as_array() + .ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?; + assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}"); + assert!( + disks.iter().all(|disk| { + disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok")) + && disk["runtimeState"] + .as_str() + .is_some_and(|state| state.eq_ignore_ascii_case("online")) + }), + "storageinfo reported a drive that was not healthy and online: {storage}" + ); + + for (node_idx, node) in dist.cluster.nodes.iter().enumerate() { + let (status, metrics_body) = admin_request( + &node.url, + Method::GET, + "/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true", + None, + &dist.cluster.access_key, + &dist.cluster.secret_key, + ) + .await?; + assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}"); + let sample: RealtimeMetrics = serde_json::from_str( + metrics_body + .lines() + .next() + .ok_or_else(|| format!("node {node_idx} returned empty metrics"))?, + )?; + assert!(sample.finally, "node {node_idx} metrics sample was not terminal"); + assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors); + assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts"); + } + + let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?; + let targets: Value = serde_json::from_str(&targets_body)?; + let configured = targets["audit_endpoints"] + .as_array() + .ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))? + .iter() + .any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook")); + assert!(configured, "configured audit webhook was missing: {targets}"); + + let bucket = unique_bucket("audit"); + dist.create_bucket(&bucket).await?; + let key = "correlated/audit-object.bin"; + let put = dist + .client(2)? + .put_object() + .bucket(&bucket) + .key(key) + .body(ByteStream::from_static(b"distributed audit payload")) + .send() + .await?; + let request_id = put.request_id().ok_or("PutObject response omitted request ID")?; + let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?; + assert_eq!( + audit["api"]["status_code"].as_i64(), + Some(200), + "audit entry did not report success: {audit}" + ); + assert!( + !audit.to_string().contains(&dist.cluster.secret_key), + "audit entry leaked the root secret key" + ); + + collector.abort(); + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/replication_quota_test.rs b/crates/e2e_test/src/distributed/replication_quota_test.rs new file mode 100644 index 000000000..ac7ed0d40 --- /dev/null +++ b/crates/e2e_test/src/distributed/replication_quota_test.rs @@ -0,0 +1,191 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota, + set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until, +}; +use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging}; +use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::primitives::ByteStream; +use http::Method; +use std::time::Duration; + +async fn wait_for_replication_status( + client: &aws_sdk_s3::Client, + bucket: &str, + key: &str, + expected: &[&str], + timeout: Duration, +) -> TestResult { + wait_until( + timeout, + || async { + let head = client.head_object().bucket(bucket).key(key).send().await?; + Ok(head + .replication_status() + .is_some_and(|status| expected.contains(&status.as_str()))) + }, + &format!("replication status for {bucket}/{key} in {expected:?}"), + ) + .await +} + +#[tokio::test] +async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult { + init_logging(); + let (source, mut target) = DistCluster::start_replication_pair().await?; + let source_bucket = unique_bucket("replsrc"); + let target_bucket = unique_bucket("repldst"); + source.create_bucket(&source_bucket).await?; + target.create_bucket(&target_bucket).await?; + + let source_client = source.client(0)?; + let target_client = target.client(0)?; + enable_versioning(&source_client, &source_bucket).await?; + enable_versioning(&target_client, &target_bucket).await?; + + let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?; + put_bucket_replication(&source.cluster, &source_bucket, &arn).await?; + + let key = "replicated/metadata-and-tags.bin"; + let body = b"distributed-bucket-replication".to_vec(); + source_client + .put_object() + .bucket(&source_bucket) + .key(key) + .metadata("origin", "four-node-source") + .tagging("suite=distributed&shape=metadata") + .body(ByteStream::from(body.clone())) + .send() + .await?; + wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?; + wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?; + + let peer_read = target.client(3)?; + wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?; + let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?; + assert_eq!( + replica_head + .metadata() + .and_then(|metadata| metadata.get("origin")) + .map(String::as_str), + Some("four-node-source") + ); + assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA")); + let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?; + let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect(); + assert_eq!(tags.get("suite"), Some(&"distributed")); + assert_eq!(tags.get("shape"), Some(&"metadata")); + + target.cluster.stop(); + let outage_key = "replicated/queued-during-target-outage.bin"; + let outage_body = b"retry-after-target-restart".to_vec(); + put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?; + wait_for_replication_status( + &source_client, + &source_bucket, + outage_key, + &["PENDING", "FAILED"], + Duration::from_secs(30), + ) + .await?; + + target.cluster.start().await?; + wait_for_ready(&target.cluster).await?; + wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?; + wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?; + Ok(()) +} + +#[tokio::test] +async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult { + init_logging(); + let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?; + let bucket = unique_bucket("quota"); + dist.create_bucket(&bucket).await?; + set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?; + + let client = dist.client(1)?; + retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?; + wait_until( + Duration::from_secs(30), + || async { + let (status, body) = super::harness::cluster_admin( + &dist.cluster, + Method::GET, + &format!("/rustfs/admin/v3/quota-stats/{bucket}"), + None, + ) + .await?; + if !status.is_success() { + return Ok(false); + } + let stats: serde_json::Value = + serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?; + let usage = stats + .get("current_usage") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?; + Ok(usage >= 1024) + }, + "quota stats observe small object", + ) + .await?; + + let oversized_key = "too-big.bin"; + let error = client + .put_object() + .bucket(&bucket) + .key(oversized_key) + .body(vec![0u8; 16 * 1024].into()) + .send() + .await + .expect_err("hard quota must reject the oversized PUT"); + let service_error = error + .as_service_error() + .ok_or("quota rejection was not an S3 service error")?; + assert_eq!( + error.raw_response().map(|response| response.status().as_u16()), + Some(400), + "quota rejection must be HTTP 400: {error:?}" + ); + assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}"); + assert!( + service_error + .message() + .is_some_and(|message| message.starts_with("Bucket quota exceeded")), + "PUT must fail specifically at quota admission: {error:?}" + ); + + let missing = client + .head_object() + .bucket(&bucket) + .key(oversized_key) + .send() + .await + .expect_err("an object rejected by quota must not become visible"); + assert_eq!( + missing.raw_response().map(|response| response.status().as_u16()), + Some(404), + "quota-rejected object returned an unexpected HEAD result: {missing:?}" + ); + + let listed = client.list_objects_v2().bucket(&bucket).send().await?; + assert!( + listed.contents().iter().all(|object| object.key() != Some(oversized_key)), + "quota-rejected key leaked into ListObjectsV2" + ); + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/s3_basic_test.rs b/crates/e2e_test/src/distributed/s3_basic_test.rs new file mode 100644 index 000000000..cd0048f10 --- /dev/null +++ b/crates/e2e_test/src/distributed/s3_basic_test.rs @@ -0,0 +1,258 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket}; +use crate::common::{init_logging, local_http_client}; +use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::presigning::PresigningConfig; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier}; +use std::time::Duration; + +#[tokio::test] +async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("s3basic"); + dist.create_bucket(&bucket).await?; + + let writer = dist.client(0)?; + let reader = dist.client(3)?; + let key = "dir/object.bin"; + let body = vec![0xA5u8; 256 * 1024]; + put_object(&writer, &bucket, key, body.clone()).await?; + + let head = reader.head_object().bucket(&bucket).key(key).send().await?; + assert_eq!(head.content_length(), Some(body.len() as i64)); + assert_object_bytes(&reader, &bucket, key, &body).await?; + + let ranged = reader + .get_object() + .bucket(&bucket) + .key(key) + .range("bytes=0-15") + .send() + .await?; + let ranged_body = ranged.body.collect().await?.into_bytes(); + assert_eq!(ranged_body.as_ref(), &body[..16]); + + let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?; + let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect(); + assert_eq!(keys, vec![key]); + + let copy_key = "dir/object-copy.bin"; + reader + .copy_object() + .bucket(&bucket) + .key(copy_key) + .copy_source(format!("{bucket}/{key}")) + .metadata_directive(MetadataDirective::Copy) + .send() + .await?; + assert_object_bytes(&writer, &bucket, copy_key, &body).await?; + + let moved_key = "dir/object-moved.bin"; + writer + .copy_object() + .bucket(&bucket) + .key(moved_key) + .copy_source(format!("{bucket}/{copy_key}")) + .send() + .await?; + writer.delete_object().bucket(&bucket).key(copy_key).send().await?; + match writer.head_object().bucket(&bucket).key(copy_key).send().await { + Ok(_) => return Err("copied source still present after rename delete".into()), + Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {} + Err(error) => return Err(error.into()), + } + assert_object_bytes(&reader, &bucket, moved_key, &body).await?; + + let presigned = writer + .get_object() + .bucket(&bucket) + .key(key) + .presigned(PresigningConfig::expires_in(Duration::from_secs(120))?) + .await?; + let response = local_http_client().get(presigned.uri().to_string()).send().await?; + assert!(response.status().is_success(), "presigned GET failed: {}", response.status()); + let presigned_body = response.bytes().await?; + assert_eq!(presigned_body.as_ref(), body.as_slice()); + + let empty_key = "empty"; + put_object(&writer, &bucket, empty_key, Vec::new()).await?; + let empty = get_object_bytes(&reader, &bucket, empty_key).await?; + assert!(empty.is_empty()); + + let deleted = writer + .delete_objects() + .bucket(&bucket) + .delete( + Delete::builder() + .objects(ObjectIdentifier::builder().key(key).build()?) + .objects(ObjectIdentifier::builder().key(moved_key).build()?) + .objects(ObjectIdentifier::builder().key(empty_key).build()?) + .build()?, + ) + .send() + .await?; + assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}"); + assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key"); + + let remaining = reader.list_objects_v2().bucket(&bucket).send().await?; + assert!(remaining.contents().is_empty(), "bucket still has objects after delete"); + Ok(()) +} + +#[tokio::test] +async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("s3matrix"); + dist.create_bucket(&bucket).await?; + let writer = dist.client(0)?; + let reader = dist.client(3)?; + + let special_key = "unicode/测试 space+percent%25.txt"; + let special_body = b"metadata and tagging survive distributed routing".to_vec(); + let put = writer + .put_object() + .bucket(&bucket) + .key(special_key) + .metadata("test-meta", "distributed") + .tagging("purpose=compatibility&scope=four-by-four") + .body(ByteStream::from(special_body.clone())) + .send() + .await?; + let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string(); + + let head = reader.head_object().bucket(&bucket).key(special_key).send().await?; + assert_eq!( + head.metadata() + .and_then(|metadata| metadata.get("test-meta")) + .map(String::as_str), + Some("distributed") + ); + assert_eq!(head.e_tag(), Some(etag.as_str())); + let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?; + let actual_tags: std::collections::BTreeMap<_, _> = tags + .tag_set() + .iter() + .map(|tag| (tag.key().to_string(), tag.value().to_string())) + .collect(); + assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility")); + assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four")); + + let conditional = reader + .get_object() + .bucket(&bucket) + .key(special_key) + .if_match(&etag) + .send() + .await?; + assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice()); + let invalid_range = reader + .get_object() + .bucket(&bucket) + .key(special_key) + .range("bytes=999999-1000000") + .send() + .await + .expect_err("an unsatisfiable range must fail"); + assert_eq!( + invalid_range.as_service_error().and_then(ProvideErrorMetadata::code), + Some("InvalidRange"), + "unexpected invalid-range error: {invalid_range:?}" + ); + + let upload_key = "multipart/aborted.bin"; + let upload = writer + .create_multipart_upload() + .bucket(&bucket) + .key(upload_key) + .send() + .await?; + let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?; + writer + .upload_part() + .bucket(&bucket) + .key(upload_key) + .upload_id(upload_id) + .part_number(1) + .body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024])) + .send() + .await?; + let pending = reader + .list_multipart_uploads() + .bucket(&bucket) + .prefix("multipart/") + .send() + .await?; + assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id))); + writer + .abort_multipart_upload() + .bucket(&bucket) + .key(upload_key) + .upload_id(upload_id) + .send() + .await?; + let after_abort = reader + .list_multipart_uploads() + .bucket(&bucket) + .prefix("multipart/") + .send() + .await?; + assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id))); + let aborted_head = reader + .head_object() + .bucket(&bucket) + .key(upload_key) + .send() + .await + .expect_err("aborted multipart upload must not create an object"); + assert_eq!( + aborted_head.raw_response().map(|response| response.status().as_u16()), + Some(404), + "aborted multipart object returned an unexpected HEAD result: {aborted_head:?}" + ); + + for index in 0..113 { + let key = format!("page/{index:04}.txt"); + put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?; + } + let mut token = None; + let mut paged_keys = Vec::new(); + loop { + let page = reader + .list_objects_v2() + .bucket(&bucket) + .prefix("page/") + .max_keys(37) + .set_continuation_token(token.take()) + .send() + .await?; + paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string))); + if page.is_truncated() != Some(true) { + break; + } + token = Some( + page.next_continuation_token() + .ok_or("truncated ListObjectsV2 page omitted next continuation token")? + .to_string(), + ); + } + assert_eq!(paged_keys.len(), 113); + let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect(); + assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys"); + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs b/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs new file mode 100644 index 000000000..66c835bbd --- /dev/null +++ b/crates/e2e_test/src/distributed/s3_during_data_movement_test.rs @@ -0,0 +1,94 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress, + decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json, + retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete, + wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress, +}; +use crate::common::init_logging; +use std::time::Duration; + +#[tokio::test] +async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult { + init_logging(); + let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?; + let bucket = unique_bucket("s3move"); + dist.create_bucket(&bucket).await?; + let client = dist.client(0)?; + let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?; + dist.expand_to_four_pools().await?; + + start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?; + wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?; + let live = dist.client(2)?; + retrying_put( + &live, + &bucket, + "during-decommission.bin", + b"written-while-decommissioning".to_vec(), + Duration::from_secs(30), + ) + .await?; + retrying_get_equals( + &live, + &bucket, + "during-decommission.bin", + b"written-while-decommissioning", + Duration::from_secs(30), + ) + .await?; + let listed = live.list_objects_v2().bucket(&bucket).send().await?; + assert!( + listed + .contents() + .iter() + .any(|object| object.key() == Some("during-decommission.bin")), + "list during decommission missed the newly written key" + ); + let status = decommission_status_json(&dist.cluster).await?; + if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? { + return Err(format!("decommission did not remain active across the S3 operations: {status}").into()); + } + + wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?; + assert_inventory(&live, &bucket, &inventory).await?; + + let rebalance_id = start_rebalance(&dist.cluster).await?; + wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?; + retrying_put( + &live, + &bucket, + "during-rebalance.bin", + b"written-while-rebalancing".to_vec(), + Duration::from_secs(30), + ) + .await?; + retrying_get_equals( + &live, + &bucket, + "during-rebalance.bin", + b"written-while-rebalancing", + Duration::from_secs(30), + ) + .await?; + let status = rebalance_status_json(&dist.cluster).await?; + if !rebalance_running_with_progress(&status, &rebalance_id)? { + return Err(format!("rebalance did not remain active across the S3 operations: {status}").into()); + } + wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?; + assert_inventory(&dist.client(1)?, &bucket, &inventory).await?; + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/site_replication_test.rs b/crates/e2e_test/src/distributed/site_replication_test.rs new file mode 100644 index 000000000..1a5b33e42 --- /dev/null +++ b/crates/e2e_test/src/distributed/site_replication_test.rs @@ -0,0 +1,128 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{ + DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes, + wait_until, +}; +use crate::common::{init_logging, signed_request}; +use http::{Method, StatusCode}; +use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus}; +use std::time::Duration; + +async fn site_replication_add( + cluster: &crate::common::RustFSTestClusterEnvironment, + sites: &[PeerSite], +) -> TestResult { + let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url); + let response = signed_request( + Method::PUT, + &url, + &cluster.access_key, + &cluster.secret_key, + Some(serde_json::to_vec(sites)?), + Some("application/json"), + ) + .await?; + if response.status() != StatusCode::OK { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(format!("site replication add failed: {status} {body}").into()); + } + Ok(serde_json::from_slice(&response.bytes().await?)?) +} + +async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult { + let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?; + Ok(serde_json::from_str(&body)?) +} + +async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult { + wait_until( + Duration::from_secs(30), + || async { + let info = site_replication_info(cluster).await?; + Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable)) + }, + "site replication enabled with two synchronized sites", + ) + .await +} + +#[tokio::test] +async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult { + init_logging(); + let (site_a, site_b) = DistCluster::start_replication_pair().await?; + let bucket = unique_bucket("siterepl"); + site_a.create_bucket(&bucket).await?; + site_b.create_bucket(&bucket).await?; + + let client_a = site_a.client(0)?; + let client_b = site_b.client(0)?; + enable_versioning(&client_a, &bucket).await?; + enable_versioning(&client_b, &bucket).await?; + + let sites = vec![ + PeerSite { + name: "site-a".to_string(), + endpoint: site_a.cluster.nodes[0].url.clone(), + access_key: site_a.cluster.access_key.clone(), + secret_key: site_a.cluster.secret_key.clone(), + ..Default::default() + }, + PeerSite { + name: "site-b".to_string(), + endpoint: site_b.cluster.nodes[0].url.clone(), + access_key: site_b.cluster.access_key.clone(), + secret_key: site_b.cluster.secret_key.clone(), + ..Default::default() + }, + ]; + let add_status = site_replication_add(&site_a.cluster, &sites).await?; + assert!( + add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(), + "site replication add reported failure: {add_status:?}" + ); + wait_for_site_replication_enabled(&site_a.cluster).await?; + wait_for_site_replication_enabled(&site_b.cluster).await?; + + let info_a = site_replication_info(&site_a.cluster).await?; + let remote = info_a + .sites + .iter() + .find(|site| site.name == "site-b") + .ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?; + assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url); + let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect(); + assert!( + deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2, + "site peers must have two distinct non-empty deployment IDs: {info_a:?}" + ); + assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}"); + assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}"); + + let key = "site-object.bin"; + let body = b"four-node-site-replication".to_vec(); + put_object(&client_a, &bucket, key, body.clone()).await?; + wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?; + + let peer_b = site_b.client(3)?; + wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?; + + let reverse_key = "reverse/site-object.bin"; + let reverse_body = b"site-b-to-site-a".to_vec(); + put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?; + wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?; + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/upgrade_test.rs b/crates/e2e_test/src/distributed/upgrade_test.rs new file mode 100644 index 000000000..33d3c1c7a --- /dev/null +++ b/crates/e2e_test/src/distributed/upgrade_test.rs @@ -0,0 +1,345 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! 4-node upgrade coverage for historical objects and IAM AK/SK. +//! +//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and +//! mixed-version listing). This module pins the distributed contract the +//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects +//! written on the previous release still read back, and IAM user credentials +//! created before the upgrade still authenticate. +//! +//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous +//! release. The `e2e-distributed` workflow downloads that binary; a local run +//! without it fails closed rather than skipping. + +use super::harness::{ + DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object, + unique_bucket, wait_until, +}; +use crate::common::{ + AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging, +}; +use aws_sdk_s3::Client; +use aws_sdk_s3::error::ProvideErrorMetadata; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use uuid::Uuid; + +const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY"; +const IAM_SECRET: &str = "UpgradeTestSecretKey1"; +const WRONG_SECRET: &str = "WrongSecretKey000000"; +const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30); + +struct UpgradeSeed { + history_bucket: String, + history_key: &'static str, + history_body: Vec, + versioned_bucket: String, + versioned_key: &'static str, + version1: String, + version1_body: Vec, + version2: String, + version2_body: Vec, + iam_bucket: String, + iam_key: &'static str, + iam_body: Vec, + iam_user: String, + iam_secret: &'static str, +} + +fn source_binary() -> TestResult { + let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| { + format!( + "{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)" + ) + })?; + if !path.is_file() { + return Err(format!("upgrade source binary does not exist: {}", path.display()).into()); + } + Ok(path) +} + +fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult { + let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else { + return Ok(()); + }; + std::fs::create_dir_all(&log_dir)?; + for node_idx in 0..cluster.cluster.nodes.len() { + let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log")); + cluster + .cluster + .set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?; + } + Ok(()) +} + +fn iam_rw_policy(bucket: &str) -> String { + serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": [ + format!("arn:aws:s3:::{bucket}"), + format!("arn:aws:s3:::{bucket}/*") + ] + }] + }) + .to_string() +} + +async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult { + let url = &dist.cluster.nodes[0].url; + let access = &dist.cluster.access_key; + let admin_secret = &dist.cluster.secret_key; + admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?; + admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?; + admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?; + Ok(()) +} + +async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec, label: &str) -> TestResult { + wait_until( + CREDENTIAL_TIMEOUT, + || { + let client = client.clone(); + let bucket = bucket.to_string(); + let key = key.to_string(); + let body = body.clone(); + async move { + put_object(&client, &bucket, &key, body).await?; + Ok(true) + } + }, + label, + ) + .await +} + +async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult { + wait_until( + CREDENTIAL_TIMEOUT, + || { + let client = client.clone(); + let bucket = bucket.to_string(); + let key = key.to_string(); + let expected = expected.to_vec(); + async move { + let got = get_object_bytes(&client, &bucket, &key).await?; + Ok(got == expected) + } + }, + label, + ) + .await +} + +async fn seed_history_and_iam(dist: &DistCluster) -> TestResult { + let history_bucket = unique_bucket("upg-hist"); + let versioned_bucket = unique_bucket("upg-ver"); + let iam_bucket = unique_bucket("upg-iam"); + dist.create_bucket(&history_bucket).await?; + dist.create_bucket(&versioned_bucket).await?; + dist.create_bucket(&iam_bucket).await?; + + let root = dist.client(0)?; + enable_versioning(&root, &versioned_bucket).await?; + + let history_key = "plain-history.bin"; + let history_body = b"written by the previous 4-node release".to_vec(); + put_object(&root, &history_bucket, history_key, history_body.clone()).await?; + + let versioned_key = "versioned-history.txt"; + let version1_body = b"version-one-before-upgrade".to_vec(); + let version1 = root + .put_object() + .bucket(&versioned_bucket) + .key(versioned_key) + .body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone())) + .send() + .await? + .version_id() + .ok_or("first versioned PUT omitted version ID")? + .to_string(); + let version2_body = b"version-two-before-upgrade".to_vec(); + let version2 = root + .put_object() + .bucket(&versioned_bucket) + .key(versioned_key) + .body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone())) + .send() + .await? + .version_id() + .ok_or("second versioned PUT omitted version ID")? + .to_string(); + + let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]); + let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]); + create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?; + + let iam_key = "iam-history.bin"; + let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec(); + let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?; + wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?; + + Ok(UpgradeSeed { + history_bucket, + history_key, + history_body, + versioned_bucket, + versioned_key, + version1, + version1_body, + version2, + version2_body, + iam_bucket, + iam_key, + iam_body, + iam_user, + iam_secret: IAM_SECRET, + }) +} + +async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult { + let root_a = dist.client(0)?; + let root_b = dist.client(3)?; + wait_for_bytes( + &root_b, + &seed.history_bucket, + seed.history_key, + &seed.history_body, + &format!("{context}: root GET historical object"), + ) + .await?; + assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?; + + let v1 = root_b + .get_object() + .bucket(&seed.versioned_bucket) + .key(seed.versioned_key) + .version_id(&seed.version1) + .send() + .await?; + let v1_body = v1.body.collect().await?.into_bytes(); + if v1_body.as_ref() != seed.version1_body.as_slice() { + return Err(format!("{context}: version 1 bytes changed after upgrade").into()); + } + let v2 = root_a + .get_object() + .bucket(&seed.versioned_bucket) + .key(seed.versioned_key) + .version_id(&seed.version2) + .send() + .await?; + let v2_body = v2.body.collect().await?.into_bytes(); + if v2_body.as_ref() != seed.version2_body.as_slice() { + return Err(format!("{context}: version 2 bytes changed after upgrade").into()); + } + + let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?; + if !users.contains(&seed.iam_user) { + return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into()); + } + + let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?; + let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?; + wait_for_bytes( + &iam_on_upgraded, + &seed.iam_bucket, + seed.iam_key, + &seed.iam_body, + &format!("{context}: IAM GET historical object on node 0"), + ) + .await?; + wait_for_bytes( + &iam_on_peer, + &seed.iam_bucket, + seed.iam_key, + &seed.iam_body, + &format!("{context}: IAM GET historical object on node 3"), + ) + .await?; + + let post_key = format!("after-upgrade-{context}.txt"); + let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes(); + wait_for_put( + &iam_on_peer, + &seed.iam_bucket, + &post_key, + post_body.clone(), + &format!("{context}: IAM PUT after upgrade"), + ) + .await?; + assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?; + + let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?; + match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await { + Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()), + Err(error) => { + let code = error.as_service_error().and_then(ProvideErrorMetadata::code); + let rejected = code == Some("SignatureDoesNotMatch") + || code == Some("InvalidAccessKeyId") + || code == Some("AccessDenied") + || code == Some("InvalidArgument") + || error.raw_response().is_some_and(|response| response.status().as_u16() == 403); + if !rejected { + return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into()); + } + } + } + + let post_root_key = format!("root-after-{context}.bin"); + let post_root_body = format!("{context}: root write after upgrade").into_bytes(); + put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?; + assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?; + Ok(()) +} + +#[tokio::test] +async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult { + init_logging(); + let previous = source_binary()?; + let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?; + capture_upgrade_logs(&mut dist, "direct-upgrade")?; + dist.start_from_binary(&previous).await?; + + let seed = seed_history_and_iam(&dist).await?; + dist.restart_with_current_binary().await?; + assert_history_and_iam(&dist, &seed, "direct").await?; + Ok(()) +} + +#[tokio::test] +async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult { + init_logging(); + let previous = source_binary()?; + let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?; + capture_upgrade_logs(&mut dist, "rolling-upgrade")?; + dist.start_from_binary(&previous).await?; + + let seed = seed_history_and_iam(&dist).await?; + + dist.replace_node_with_current_binary(0).await?; + assert_history_and_iam(&dist, &seed, "one-current-node").await?; + + for node_idx in [1, 2] { + dist.replace_node_with_current_binary(node_idx).await?; + } + assert_history_and_iam(&dist, &seed, "one-previous-node").await?; + + dist.replace_node_with_current_binary(3).await?; + assert_history_and_iam(&dist, &seed, "homogeneous-current").await?; + Ok(()) +} diff --git a/crates/e2e_test/src/distributed/versioning_test.rs b/crates/e2e_test/src/distributed/versioning_test.rs new file mode 100644 index 000000000..0a86fe145 --- /dev/null +++ b/crates/e2e_test/src/distributed/versioning_test.rs @@ -0,0 +1,188 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket}; +use crate::common::init_logging; +use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; + +#[tokio::test] +async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("version"); + dist.create_bucket(&bucket).await?; + let writer = dist.client(0)?; + let reader = dist.client(3)?; + enable_versioning(&writer, &bucket).await?; + + let key = "versioned.txt"; + let v1_id = writer + .put_object() + .bucket(&bucket) + .key(key) + .body(b"v1".to_vec().into()) + .send() + .await? + .version_id() + .ok_or("v1 PUT omitted version ID")? + .to_string(); + let v2_id = writer + .put_object() + .bucket(&bucket) + .key(key) + .body(b"v2".to_vec().into()) + .send() + .await? + .version_id() + .ok_or("v2 PUT omitted version ID")? + .to_string(); + + let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?; + let matching_versions: Vec<_> = versions + .versions() + .iter() + .filter(|version| version.key() == Some(key)) + .collect(); + assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}"); + assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker"); + assert!( + matching_versions + .iter() + .any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)), + "v1 was not the historical version: {versions:?}" + ); + assert!( + matching_versions + .iter() + .any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)), + "v2 was not the latest version: {versions:?}" + ); + + let latest = get_object_bytes(&reader, &bucket, key).await?; + assert_eq!(latest, b"v2"); + + let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?; + let older_body = older.body.collect().await?.into_bytes(); + assert_eq!(older_body.as_ref(), b"v1"); + + let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?; + assert_eq!(deleted.delete_marker(), Some(true)); + let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?; + let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?; + let matching_markers: Vec<_> = after_delete + .delete_markers() + .iter() + .filter(|marker| marker.key() == Some(key)) + .collect(); + assert_eq!( + matching_markers.len(), + 1, + "delete marker missing or duplicated after current-version delete: {after_delete:?}" + ); + assert!( + matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true), + "DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}" + ); + + let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await; + match latest_after_delete { + Ok(_) => return Err("current version should be a delete marker".into()), + Err(error) + if error + .as_service_error() + .and_then(ProvideErrorMetadata::code) + .is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {} + Err(error) => return Err(error.into()), + } + + let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?; + let restored_body = restored.body.collect().await?.into_bytes(); + assert_eq!(restored_body.as_ref(), b"v1"); + + writer + .delete_object() + .bucket(&bucket) + .key(key) + .version_id(marker_id) + .send() + .await?; + assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2"); + Ok(()) +} + +#[tokio::test] +async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult { + init_logging(); + let dist = DistCluster::start(DistLayout::FourByFour).await?; + let bucket = unique_bucket("suspend"); + dist.create_bucket(&bucket).await?; + let writer = dist.client(0)?; + let reader = dist.client(3)?; + enable_versioning(&writer, &bucket).await?; + + let key = "suspended.txt"; + let original = writer + .put_object() + .bucket(&bucket) + .key(key) + .body(b"enabled-history".to_vec().into()) + .send() + .await? + .version_id() + .ok_or("enabled PUT omitted version ID")? + .to_string(); + writer + .put_bucket_versioning() + .bucket(&bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Suspended) + .build(), + ) + .send() + .await?; + + put_object(&writer, &bucket, key, b"null-one".to_vec()).await?; + put_object(&writer, &bucket, key, b"null-two".to_vec()).await?; + assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two"); + + let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?; + let matching: Vec<_> = versions + .versions() + .iter() + .filter(|version| version.key() == Some(key)) + .collect(); + assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str()))); + let null_version_count = matching + .iter() + .filter(|version| { + matches!( + version.version_id(), + None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000") + ) + }) + .count(); + assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}"); + + let historical = reader + .get_object() + .bucket(&bucket) + .key(key) + .version_id(&original) + .send() + .await?; + assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history"); + Ok(()) +} diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index 8201210ec..68b0006b1 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -16,15 +16,18 @@ #[cfg(test)] mod tests { - use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post}; + use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post}; use crate::common::{ FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, + rustfs_binary_path, }; use crate::storage_api::RUSTFS_META_BUCKET; use aws_sdk_s3::primitives::ByteStream; use http::Method; + use sha2::{Digest, Sha256}; use std::collections::HashSet; use std::error::Error; + use std::io::{Read, Write}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::process::Command; @@ -34,6 +37,76 @@ mod tests { const POOL_METADATA_OBJECT: &str = "pool.bin"; + #[derive(serde::Deserialize)] + struct EvidenceBuild { + sha256: String, + } + + #[derive(serde::Deserialize)] + struct RestartEvidenceRun { + schema: u32, + run_id: String, + source_revision: String, + test_build: serde_json::Value, + binary: EvidenceBuild, + test_binary: EvidenceBuild, + } + + fn file_sha256(path: &Path) -> Result> { + let mut file = std::fs::File::open(path)?; + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + } + Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect()) + } + + fn restart_evidence_run(binary: &Path) -> Result, Box> { + let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else { + return Ok(None); + }; + let directory = PathBuf::from(directory); + let receipt = directory.join("run.json"); + if receipt.metadata()?.len() > 1024 * 1024 { + return Err("oversized scanner/heal execution receipt".into()); + } + let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?; + if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 { + return Err("invalid scanner/heal execution identity".into()); + } + let built = compiled_test_identity(); + for key in ["source_revision", "dirty", "lock_blob", "features"] { + assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}"); + } + assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt"); + assert_eq!( + file_sha256(&std::env::current_exe()?)?, + run.test_binary.sha256, + "test executable must match the run receipt" + ); + if directory.join("background-target-restart.json").exists() { + return Err("scanner/heal oracle already exists; create a new execution receipt".into()); + } + Ok(Some((directory, run))) + } + + fn compiled_test_identity() -> serde_json::Value { + serde_json::json!({ + "source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"), + "dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false", + "lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"), + "features": env!("RUSTFS_E2E_BUILD_FEATURES"), + "target": env!("RUSTFS_E2E_BUILD_TARGET"), + "profile": env!("RUSTFS_E2E_BUILD_PROFILE"), + "rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"), + }) + } + struct TcpPortBlackhole { port: u16, comment: String, @@ -195,8 +268,9 @@ mod tests { clients: &[aws_sdk_s3::Client], bucket: &str, expected_keys: &HashSet, - ) -> Result<(), Box> { + ) -> Result>, Box> { const PAGE_SIZE: i32 = 10; + let mut node_listings = Vec::with_capacity(clients.len()); for (node_index, client) in clients.iter().enumerate() { let mut listed_keys = Vec::new(); let mut continuation_token = None; @@ -243,8 +317,10 @@ mod tests { &listed_key_set, expected_keys, "node {node_index} did not expose the complete recovered namespace" ); + listed_keys.sort(); + node_listings.push(listed_keys); } - Ok(()) + Ok(node_listings) } fn heal_task_status_diagnostic(body: &str) -> String { @@ -808,6 +884,13 @@ mod tests { } async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box> { + let server_binary = rustfs_binary_path(); + let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart { + restart_evidence_run(&server_binary)? + } else { + None + }; + let mut evidence_objects = Vec::new(); let (background_enabled, interruption_node, interruption_kind) = match scenario { InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"), InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"), @@ -855,7 +938,7 @@ mod tests { for node_index in 0..cluster.nodes.len() { cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?; } - cluster.start().await?; + cluster.start_with_binary(&server_binary).await?; let clients = cluster.create_all_clients()?; let bucket = "heal-restart-during-rebuild"; @@ -996,7 +1079,7 @@ mod tests { } } - cluster.start_node(1).await?; + cluster.start_node_from_binary(1, &server_binary).await?; let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url); let recovery_deadline = Instant::now() + Duration::from_secs(60); @@ -1274,7 +1357,7 @@ mod tests { } } } - cluster.start_node(interruption_node).await?; + cluster.start_node_from_binary(interruption_node, &server_binary).await?; if interruption_node == 0 { let target = cluster.nodes[1] .process @@ -1373,7 +1456,7 @@ mod tests { .map(|manifest| manifest.key.clone()) .collect::>(); assert!(expected_keys.insert(outage_key.to_string())); - assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?; + let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?; let target_client = cluster.create_s3_client(1)?; for expected in &expected_manifests { @@ -1381,11 +1464,31 @@ mod tests { let actual = response.body.collect().await?.into_bytes(); let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed); assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key); + if evidence_run.is_some() { + evidence_objects.push(serde_json::json!({ + "key": expected.key, "version_id": expected.shard_census.version_id, + "expected_bytes": expected_body.len(), "actual_bytes": actual.len(), + "expected_sha256": sha256_hex(&expected_body), + "actual_sha256": sha256_hex(&actual), + "expected_physical": expected.shard_census, + "physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?, + })); + } } let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?; let actual = response.body.collect().await?.into_bytes(); let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed); assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}"); + if evidence_run.is_some() { + evidence_objects.push(serde_json::json!({ + "key": outage_key, "version_id": null, + "expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(), + "expected_sha256": sha256_hex(&expected_outage_body), + "actual_sha256": sha256_hex(&actual), + "expected_physical": null, + "physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?, + })); + } let terminal_deadline = Instant::now() + Duration::from_secs(30); loop { @@ -1432,6 +1535,31 @@ mod tests { return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into()); } + if let Some((directory, run)) = evidence_run { + let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id(); + assert_ne!(target_pid, restarted_pid, "target must be a new process"); + assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart"); + let evidence = serde_json::json!({ + "schema": 1, "case": "background-target-restart", "evidence": "process-restart", + "run_id": run.run_id, "source_revision": run.source_revision, + "test_build": compiled_test_identity(), + "binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256, + "topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()}, + "pid_before": target_pid, "pid_after": restarted_pid, + "objects": evidence_objects, "node_listings": node_listings, + }); + let data = serde_json::to_vec(&evidence)?; + if data.len() > 1024 * 1024 { + return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into()); + } + let mut output = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(directory.join("background-target-restart.json"))?; + output.write_all(&data)?; + output.sync_all()?; + } + Ok(()) } diff --git a/crates/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs index 9279c39c4..72f5121fb 100644 --- a/crates/e2e_test/src/lib.rs +++ b/crates/e2e_test/src/lib.rs @@ -378,6 +378,11 @@ mod bucket_stats_regression_test; #[cfg(test)] mod distributed_startup_regression_test; +// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication, +// quota, observability, expand/decommission/rebalance, site replication, chaos). +#[cfg(test)] +mod distributed; + // P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024) #[cfg(test)] mod tier_transition_regression_test; diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index b19397a7f..b72ce3e93 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -32,7 +32,7 @@ pub mod bucket { pub mod bucket_target_sys { pub use crate::bucket::bucket_target_sys::{ AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, - SsecPassthroughCapability, TargetClient, append_version_id_query, + SsecPassthroughCapability, TargetClient, UnreadableTargetsPolicy, append_version_id_query, }; } @@ -69,6 +69,13 @@ pub mod bucket { }; } + pub mod recovery_control { + pub use crate::bucket::lifecycle::recovery_control::{ + IlmRecoveryClassification, IlmRecoveryControlPage, IlmRecoveryControlView, IlmRecoveryProtocol, + inspect_recovery_control, list_recovery_controls, + }; + } + pub mod transition_transaction { pub use crate::bucket::lifecycle::transition_transaction::{ TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus, @@ -89,8 +96,9 @@ pub mod bucket { #[allow(clippy::module_inception)] pub mod lifecycle { pub use crate::bucket::lifecycle::lifecycle::{ - Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate, - TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time, object_opts_from_object_info, + Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, + ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time, + object_opts_from_object_info, }; } diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index db4e80f29..ce8ec5a67 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -369,6 +369,26 @@ struct SsecPassthroughRecord { recorded_at: Instant, } +/// What a target write does when the bucket's persisted target set exists but +/// cannot be decoded. +/// +/// `docs/architecture/remote-credential-sealing-adr.md` forbids rewriting a +/// configuration that could not be fully read, because re-serializing a +/// partial in-memory view is the one mechanism by which a configured target +/// really disappears. That rule guards against an *unintentional* overwrite, +/// so an operator who names the hazard keeps a repair path +/// (rustfs/backlog#2309); everything that does not name it stays refused. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum UnreadableTargetsPolicy { + /// Refuse the write with [`BucketTargetError::BucketRemoteTargetsUnreadable`]. + #[default] + FailClosed, + /// Discard the unreadable set; the target being written becomes the whole + /// configuration. Reachable only from an admin request that asked for it + /// explicitly, and audited by the caller. + Replace, +} + #[derive(Debug, Default)] pub struct BucketTargetSys { pub arn_remotes_map: Arc>>, @@ -791,20 +811,45 @@ impl BucketTargetSys { bucket: &str, target: &BucketTarget, update: bool, + unreadable_policy: UnreadableTargetsPolicy, ) -> Result { self.validate_target(bucket, target).await?; - let mut bucket_targets = match self.list_bucket_targets(bucket).await { - Ok(targets) => targets, - Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => BucketTargets::default(), - Err(err) => return Err(err), - }; + let mut bucket_targets = self.targets_base_for_write(bucket, unreadable_policy).await?; Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?; Ok(bucket_targets) } + /// The persisted target set a write merges into. + /// + /// An absent configuration starts from the empty set. An unreadable one is + /// refused, because re-serializing a partial view of a set this node could + /// not decode is how a configured target disappears for good — unless the + /// caller carries the operator's explicit + /// [`UnreadableTargetsPolicy::Replace`] opt-in, which discards it + /// deliberately (rustfs/backlog#2309). + async fn targets_base_for_write( + &self, + bucket: &str, + unreadable_policy: UnreadableTargetsPolicy, + ) -> Result { + match self.list_bucket_targets(bucket).await { + Ok(targets) => Ok(targets), + Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => Ok(BucketTargets::default()), + // The opt-in discards only a set this node genuinely cannot read. + // A readable set still merges through the arm above, so the policy + // can never drop a target that was visible here. + Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) + if unreadable_policy == UnreadableTargetsPolicy::Replace => + { + Ok(BucketTargets::default()) + } + Err(err) => Err(err), + } + } + pub async fn validate_target(&self, bucket: &str, target: &BucketTarget) -> Result<(), BucketTargetError> { if !target.target_type.is_valid() { return Err(BucketTargetError::BucketRemoteArnTypeInvalid { @@ -4313,4 +4358,75 @@ mod tests { let window = LastMinuteLatency::new(); assert_eq!(window.get_total().avg, Duration::from_secs(0)); } + + fn repair_target(bucket: &str, id: &str) -> BucketTarget { + BucketTarget { + source_bucket: bucket.to_string(), + endpoint: "remote.example.com".to_string(), + target_bucket: "remote".to_string(), + arn: format!("arn:rustfs:replication:us-east-1:{bucket}:{id}"), + target_type: BucketTargetType::ReplicationService, + region: "us-east-1".to_string(), + ..Default::default() + } + } + + /// rustfs/backlog#2309: after rustfs/rustfs#7172 an undecodable + /// `bucket-targets.json` left the bucket with no API repair path at all. + /// The refusal is the default and stays the default; the operator's + /// explicit opt-in is the only thing that discards the set, and it starts + /// the replacement from empty rather than from a partial view of bytes + /// this node never decoded. + #[tokio::test] + async fn an_unreadable_target_set_is_replaced_only_with_the_explicit_opt_in() { + let sys = BucketTargetSys::default(); + let bucket = "targets-repair-opt-in"; + sys.mark_targets_unreadable(bucket).await; + + assert!( + matches!( + sys.targets_base_for_write(bucket, UnreadableTargetsPolicy::FailClosed).await, + Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) + ), + "without the opt-in an unreadable target set must still refuse the write" + ); + assert_eq!( + UnreadableTargetsPolicy::default(), + UnreadableTargetsPolicy::FailClosed, + "a caller that says nothing must get the refusal" + ); + + let base = sys + .targets_base_for_write(bucket, UnreadableTargetsPolicy::Replace) + .await + .expect("the explicit opt-in must let an operator replace an unreadable set"); + assert!( + base.is_empty(), + "the replacement must start from an empty set, never from a partial decode" + ); + } + + /// The opt-in is not a wipe switch. On a set this node can read, both + /// policies take the same merge path, so a stray `replace-unreadable=true` + /// cannot drop a visible target — which is what makes the flag safe to + /// repeat in an operator's repair script. + #[tokio::test] + async fn the_opt_in_never_discards_a_readable_target_set() { + let sys = BucketTargetSys::default(); + let bucket = "targets-repair-readable"; + let existing = repair_target(bucket, "keep"); + sys.targets_map + .write() + .await + .insert(bucket.to_string(), vec![existing.clone()]); + + for policy in [UnreadableTargetsPolicy::FailClosed, UnreadableTargetsPolicy::Replace] { + let base = sys + .targets_base_for_write(bucket, policy) + .await + .expect("a readable target set must be readable under either policy"); + assert_eq!(base.targets.len(), 1, "{policy:?} must keep the persisted target"); + assert_eq!(base.targets[0].arn, existing.arn, "{policy:?} must not rewrite the persisted target"); + } + } } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index dbb5dd154..426daa722 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -150,6 +150,18 @@ static XXHASH_SEED: u64 = 0; static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new(); static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new(); +#[cfg(test)] +#[derive(Default)] +struct FreeVersionPostRemoteDeleteTestBarrier { + arrived: Notify, + release: Notify, +} + +#[cfg(test)] +tokio::task_local! { + static FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER: Arc; +} + pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging"; #[allow( dead_code, @@ -910,6 +922,11 @@ async fn cleanup_free_version_exact(api: Arc, oi: &ObjectInfo, cancel: })??; } } + #[cfg(test)] + if let Ok(barrier) = FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER.try_with(Arc::clone) { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) { // Remote DELETE is idempotent, but a changed fence makes the local // outcome ambiguous. Keep every marker for a fully fenced retry. @@ -5831,7 +5848,7 @@ mod tests { #[cfg(feature = "test-util")] use crate::services::tier::test_util::register_mock_tier; #[cfg(feature = "test-util")] - use crate::services::tier::tier::TierConfigMgr; + use crate::services::tier::tier::{TIER_DRIVER_TEST_FACTORY, TierConfigMgr, TierDriverTestFactory}; #[cfg(feature = "test-util")] use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _}; use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause}; @@ -7830,6 +7847,119 @@ mod tests { } } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial] + async fn tier_remove_waits_for_inflight_free_version_local_commit() { + let (disk_paths, ecstore) = setup_test_env().await; + let bucket = format!("tier-remove-free-version-{}", Uuid::new_v4()); + let object = "free-version"; + create_test_bucket(&ecstore, &bucket).await; + let (backend, identity_hex) = register_recovery_mock_tier(&ecstore).await; + let tier_manager = ecstore.tier_config_mgr(); + { + let manager = tier_manager.read().await; + manager + .save_tiering_config(Arc::clone(&ecstore)) + .await + .expect("mock tier configuration should persist before removal"); + } + seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity_hex)).await; + let page = list_tier_free_versions(Arc::clone(&ecstore), 1, None, None, CancellationToken::new()) + .await + .expect("seeded free version should be listed"); + let oi = page + .items + .into_iter() + .next() + .expect("seeded free version should be recoverable"); + + backend + .set_put_remote_version(Some(oi.transitioned_object.version_id.clone())) + .await; + let seed_lease = TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM") + .await + .expect("mock tier lease should be available"); + seed_lease + .put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4) + .await + .expect("remote free-version tuple should be seeded"); + drop(seed_lease); + + let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default()); + let cleanup_barrier = Arc::clone(&barrier); + let cleanup_store = Arc::clone(&ecstore); + let cleanup_oi = oi.clone(); + let cleanup = tokio::spawn(async move { + super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER + .scope(cleanup_barrier, async move { + super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await + }) + .await + }); + tokio::time::timeout(StdDuration::from_secs(30), barrier.arrived.notified()) + .await + .expect("free-version cleanup should pause after the remote delete"); + assert!(!backend.contains(&oi.transitioned_object.name).await); + + let remove_manager = Arc::clone(&tier_manager); + let remove_store = Arc::clone(&ecstore); + let remove_backend = backend.clone(); + let remove_driver_factory: TierDriverTestFactory = Arc::new(move |_| Ok(Box::new(remove_backend.clone()))); + let mut remove = tokio::spawn(async move { + TIER_DRIVER_TEST_FACTORY + .scope( + remove_driver_factory, + TierConfigMgr::remove_and_save(&remove_manager, remove_store, "WARM", true), + ) + .await + }); + let prepared = tokio::time::timeout(StdDuration::from_secs(30), async { + loop { + match TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM").await { + Ok(lease) => drop(lease), + Err(err) if TierConfigMgr::operation_lease_blocked_by_mutation(&err) => break, + Err(err) => panic!("tier remove should only block new operations while cleanup is paused: {err}"), + } + tokio::task::yield_now().await; + } + }); + tokio::select! { + prepared = prepared => { + prepared.expect("tier remove should install its prepared admission fence"); + } + result = &mut remove => { + panic!("tier remove finished before installing its prepared admission fence: {result:?}"); + } + } + assert!(!remove.is_finished(), "tier remove must wait for the leased local cleanup commit"); + + barrier.release.notify_one(); + tokio::time::timeout(StdDuration::from_secs(30), cleanup) + .await + .expect("free-version cleanup should finish after release") + .expect("free-version cleanup task should join") + .expect("free-version cleanup should keep its generation current"); + tokio::time::timeout(StdDuration::from_secs(30), remove) + .await + .expect("tier remove should finish after local cleanup") + .expect("tier remove task should join") + .expect("tier remove should pass its fresh authoritative proof"); + + assert!(!tier_manager.read().await.is_tier_valid("WARM")); + for disk_path in &disk_paths { + assert!( + !fs::try_exists(disk_path.join(&bucket).join(object)) + .await + .expect("post-removal free-version path check should succeed") + ); + } + ecstore + .delete_bucket(&bucket, &DeleteBucketOptions::default()) + .await + .expect("empty free-version test bucket should be removed"); + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial] diff --git a/crates/ecstore/src/bucket/lifecycle/core.rs b/crates/ecstore/src/bucket/lifecycle/core.rs index 897874a25..7a4ee71e2 100644 --- a/crates/ecstore/src/bucket/lifecycle/core.rs +++ b/crates/ecstore/src/bucket/lifecycle/core.rs @@ -15,9 +15,9 @@ use crate::object_api::ObjectInfo; pub use rustfs_lifecycle::{ - Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate, TRANSITION_COMPLETE, - TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due, expected_expiry_time, - expiration_action_has_valid_target, + Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts, + RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due, + expected_expiry_time, expiration_action_has_valid_target, }; pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts { diff --git a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs index a9957a25c..58e7ae607 100644 --- a/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs +++ b/crates/ecstore/src/bucket/lifecycle/durable_namespace.rs @@ -22,7 +22,7 @@ use super::{ bucket_lifecycle_ops::{ ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token, }, - manual_transition_job, tier_delete_journal, transition_transaction, + manual_transition_job, recovery_control, tier_delete_journal, transition_transaction, }; use crate::error::{Error, Result}; use crate::services::tier::tier_probe_intent; @@ -41,6 +41,7 @@ pub(crate) enum DurableIlmRecordKind { ManualTransitionScope, ManualTransitionTask, ManualTransitionWorkerResult, + RecoveryControl, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -105,8 +106,14 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE, kind: DurableIlmRecordKind::ManualTransitionWorkerResult, }; +pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace { + name: "recovery-control", + prefix: recovery_control::ILM_RECOVERY_CONTROL_PREFIX, + max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE, + kind: DurableIlmRecordKind::RecoveryControl, +}; -pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [ +pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [ TIER_DELETE_JOURNAL_NAMESPACE, TIER_DELETE_JOURNAL_V6_NAMESPACE, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE, @@ -116,6 +123,7 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [ MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE, MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE, + RECOVERY_CONTROL_NAMESPACE, ]; #[derive(Debug, Clone, PartialEq, Eq)] @@ -241,6 +249,18 @@ pub(crate) enum DurableIlmRecordCheckpoint { ManualTransitionWorkerResult { content_sha256: String, }, + RecoveryControl { + content_sha256: String, + identity_sha256: String, + source_generation_sha256: String, + first_seen_at_unix_nanos: i64, + revision: u64, + classification: recovery_control::IlmRecoveryClassification, + attempt_count: u64, + consecutive_failure_count: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + owner_fence_sha256: Option, + }, } impl DurableIlmRecordCheckpoint { @@ -254,7 +274,8 @@ impl DurableIlmRecordCheckpoint { | Self::ManualTransitionJob { content_sha256, .. } | Self::ManualTransitionScope { content_sha256, .. } | Self::ManualTransitionTask { content_sha256 } - | Self::ManualTransitionWorkerResult { content_sha256 } => content_sha256, + | Self::ManualTransitionWorkerResult { content_sha256 } + | Self::RecoveryControl { content_sha256, .. } => content_sha256, } } @@ -528,6 +549,51 @@ impl DurableIlmRecordCheckpoint { .. }, ) => previous_identity == next_identity && next_updated_at > previous_updated_at, + ( + Self::RecoveryControl { + identity_sha256: previous_identity, + source_generation_sha256: previous_generation, + first_seen_at_unix_nanos: previous_first_seen, + revision: previous_revision, + classification: previous_classification, + attempt_count: previous_attempts, + consecutive_failure_count: previous_failures, + owner_fence_sha256: previous_owner, + .. + }, + Self::RecoveryControl { + identity_sha256: next_identity, + source_generation_sha256: next_generation, + first_seen_at_unix_nanos: next_first_seen, + revision: next_revision, + classification: next_classification, + attempt_count: next_attempts, + consecutive_failure_count: next_failures, + owner_fence_sha256: next_owner, + .. + }, + ) => { + let adjacent = previous_identity == next_identity + && previous_first_seen == next_first_seen + && previous_revision.checked_add(1) == Some(*next_revision); + let claim = next_owner.is_some() + && *previous_classification == recovery_control::IlmRecoveryClassification::Retrying + && *next_classification == recovery_control::IlmRecoveryClassification::Retrying + && previous_attempts.checked_add(1) == Some(*next_attempts) + && previous_failures == next_failures; + let source_refresh = previous_owner.is_some() + && previous_owner == next_owner + && *previous_classification == recovery_control::IlmRecoveryClassification::Retrying + && *next_classification == recovery_control::IlmRecoveryClassification::Retrying + && previous_attempts == next_attempts + && previous_failures == next_failures + && previous_generation != next_generation; + let completion = previous_owner.is_some() + && next_owner.is_none() + && previous_generation == next_generation + && previous_attempts == next_attempts; + adjacent && (claim || source_refresh || completion) + } _ => false, }; @@ -553,6 +619,14 @@ impl DurableIlmRecordCheckpoint { { return false; } + if let Self::RecoveryControl { classification, .. } = terminal + && !matches!( + classification, + recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned + ) + { + return false; + } if self == terminal || self.validate_successor(terminal).is_ok() { return true; } @@ -652,6 +726,32 @@ impl DurableIlmRecordCheckpoint { .is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance)) && (!previous_remote_version_known || previous_remote_version == terminal_remote_version) } + ( + Self::RecoveryControl { + identity_sha256: previous_identity, + source_generation_sha256: previous_generation, + first_seen_at_unix_nanos: previous_first_seen, + revision: previous_revision, + attempt_count: previous_attempts, + .. + }, + Self::RecoveryControl { + identity_sha256: terminal_identity, + source_generation_sha256: terminal_generation, + first_seen_at_unix_nanos: terminal_first_seen, + revision: terminal_revision, + attempt_count: terminal_attempts, + classification: + recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned, + .. + }, + ) => { + previous_identity == terminal_identity + && (previous_generation == terminal_generation || terminal_attempts > previous_attempts) + && previous_first_seen == terminal_first_seen + && terminal_revision > previous_revision + && terminal_attempts >= previous_attempts + } _ => false, } } @@ -1219,6 +1319,35 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result { + let (protocol, control_id) = recovery_control::recovery_control_id_from_record_object_name(path) + .map_err(|err| Error::other(err.to_string()))?; + let control = + recovery_control::IlmRecoveryControl::decode(&control_id, data).map_err(|err| Error::other(err.to_string()))?; + let canonical = recovery_control::recovery_control_record_object_name(protocol, &control_id) + .map_err(|err| Error::other(err.to_string()))?; + if canonical != path || control.identity.protocol != protocol { + return Err(Error::other("ILM recovery control path is not canonical")); + } + let identity_sha256 = checkpoint_hash(&control.identity)?; + let source_generation_sha256 = checkpoint_hash(&control.observed_source_generation)?; + let owner_fence_sha256 = control.owner.as_ref().map(checkpoint_hash).transpose()?; + ( + "control_id", + control_id, + DurableIlmRecordCheckpoint::RecoveryControl { + content_sha256, + identity_sha256, + source_generation_sha256, + first_seen_at_unix_nanos: control.first_seen_at_unix_nanos, + revision: control.revision, + classification: control.classification, + attempt_count: control.attempt_count, + consecutive_failure_count: control.consecutive_failure_count, + owner_fence_sha256, + }, + ) + } DurableIlmRecordKind::ManualTransitionJob => { let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path) .map_err(|err| Error::other(err.to_string()))?; @@ -1412,6 +1541,94 @@ mod tests { .checkpoint } + fn recovery_control_fixture() -> recovery_control::IlmRecoveryControl { + let source_path = "ilm/transition-transactions/records/12/34/1234567890abcdef1234567890abcdef.json"; + let generation = recovery_control::IlmRecoverySourceGeneration::new( + transition_transaction::TRANSITION_TRANSACTION_SCHEMA, + "source-etag", + "a".repeat(64), + vec![recovery_control::IlmRecoverySourceCopy { + authority: "pool-0/set-0".to_string(), + canonical_path: source_path.to_string(), + etag: "source-etag".to_string(), + encoded_len: 128, + content_sha256: "a".repeat(64), + }], + ) + .expect("source generation should build"); + recovery_control::IlmRecoveryControl::new( + recovery_control::IlmRecoveryControlIdentity { + protocol: recovery_control::IlmRecoveryProtocol::TransitionTransaction, + canonical_source_path: source_path.to_string(), + stable_operation_identity: "12345678-90ab-cdef-1234-567890abcdef".to_string(), + record_class: "transition_transaction_v1".to_string(), + }, + generation, + recovery_control::IlmRecoveryClassification::Retrying, + 1_000_000_000, + recovery_control::IlmRecoveryErrorCode::None, + ) + .expect("recovery control should build") + } + + fn recovery_control_checkpoint(control: &recovery_control::IlmRecoveryControl) -> DurableIlmRecordCheckpoint { + let control_id = control.identity.source_operation_digest().expect("control id should derive"); + let path = recovery_control::recovery_control_record_object_name(control.identity.protocol, &control_id) + .expect("control path should build"); + let encoded = control.encode().expect("control should encode"); + let namespace = classify_durable_ilm_record(&path) + .expect("recovery control namespace should classify") + .expect("recovery control should be durable"); + assert_eq!(namespace, &RECOVERY_CONTROL_NAMESPACE); + validate_durable_ilm_record(&path, &encoded) + .expect("recovery control should validate") + .checkpoint + } + + #[test] + fn recovery_control_checkpoint_tracks_claim_retry_and_terminal_generations() { + let initial_control = recovery_control_fixture(); + let initial = recovery_control_checkpoint(&initial_control); + + let mut claimed_control = initial_control; + let mut advanced_generation = claimed_control.observed_source_generation.clone(); + advanced_generation.source_schema = "rustfs-transition-transaction-v2".to_string(); + claimed_control + .claim_for_source_generation("node-a", Uuid::new_v4(), 2_000_000_000, 300_000_000_000, advanced_generation) + .expect("control should claim"); + let claimed = recovery_control_checkpoint(&claimed_control); + initial.validate_successor(&claimed).expect("claim should advance receipt"); + + let mut retry_control = claimed_control; + retry_control + .record_retryable_failure(3_000_000_000, recovery_control::IlmRecoveryErrorCode::BackendTimeout) + .expect("retry should persist"); + let retry = recovery_control_checkpoint(&retry_control); + claimed.validate_successor(&retry).expect("retry should advance receipt"); + + let ready_at = retry_control + .next_attempt_at_unix_nanos + .expect("retry deadline should persist"); + let mut terminal_control = retry_control; + terminal_control + .claim("node-b", Uuid::new_v4(), ready_at, 300_000_000_000) + .expect("retry should claim"); + let reclaimed = recovery_control_checkpoint(&terminal_control); + retry.validate_successor(&reclaimed).expect("reclaim should advance receipt"); + terminal_control + .finish_attempt( + recovery_control::IlmRecoveryClassification::Terminal, + recovery_control::IlmRecoveryErrorCode::None, + ) + .expect("control should terminate"); + let terminal = recovery_control_checkpoint(&terminal_control); + reclaimed + .validate_successor(&terminal) + .expect("terminal state should advance receipt"); + assert!(initial.is_predecessor_of_terminal(&terminal)); + assert!(!initial.is_predecessor_of_terminal(&retry)); + } + #[test] fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() { let initial_intent = tier_probe_intent_fixture(); diff --git a/crates/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs index a2290c8ea..3eb93cca4 100644 --- a/crates/ecstore/src/bucket/lifecycle/mod.rs +++ b/crates/ecstore/src/bucket/lifecycle/mod.rs @@ -24,6 +24,7 @@ pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, g mod object_handlers_common; mod object_lock_boundary; pub use self::core as lifecycle; +pub mod recovery_control; mod replication_sink; pub mod rule; mod runtime_boundary; diff --git a/crates/ecstore/src/bucket/lifecycle/recovery_control.rs b/crates/ecstore/src/bucket/lifecycle/recovery_control.rs new file mode 100644 index 000000000..599849ca9 --- /dev/null +++ b/crates/ecstore/src/bucket/lifecycle/recovery_control.rs @@ -0,0 +1,1280 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::config_boundary; +use crate::disk::RUSTFS_META_BUCKET; +use crate::error::{Error, Result as EcstoreResult}; +use crate::object_api::ObjectOptions; +use crate::storage_api_contracts::{list::ListOperations as _, object::HTTPPreconditions}; +use crate::store::ECStore; + +pub const ILM_RECOVERY_CONTROL_SCHEMA: &str = "rustfs-ilm-recovery-control-v1"; +pub const ILM_RECOVERY_CONTROL_PREFIX: &str = "ilm/recovery-controls"; +pub const MAX_ILM_RECOVERY_CONTROL_SIZE: usize = 16 * 1024; +pub const MAX_RECOVERY_ATTEMPTS: u32 = 32; +const MAX_RECOVERY_AGE_NANOS: i64 = 7 * 24 * 60 * 60 * 1_000_000_000; +const MIN_RETRY_DELAY_NANOS: i64 = 60 * 1_000_000_000; +const MAX_RETRY_DELAY_NANOS: i64 = 60 * 60 * 1_000_000_000; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum IlmRecoveryControlError { + #[error("ILM recovery control is corrupt: {0}")] + Corrupt(&'static str), + #[error("ILM recovery control schema is unsupported: {0}")] + UnsupportedSchema(String), + #[error("ILM recovery control checksum mismatch")] + ChecksumMismatch, + #[error("ILM recovery control successor is invalid: {0}")] + InvalidSuccessor(&'static str), + #[error("ILM recovery control json error: {0}")] + Json(#[from] serde_json::Error), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IlmRecoveryProtocol { + TransitionTransaction, + TierDeleteJournal, + TierDeleteManifest, +} + +impl IlmRecoveryProtocol { + pub fn as_str(self) -> &'static str { + match self { + Self::TransitionTransaction => "transition_transaction", + Self::TierDeleteJournal => "tier_delete_journal", + Self::TierDeleteManifest => "tier_delete_manifest", + } + } + + pub const fn all() -> [Self; 3] { + [Self::TransitionTransaction, Self::TierDeleteJournal, Self::TierDeleteManifest] + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IlmRecoveryClassification { + Retrying, + RetainedAmbiguous, + Corrupt, + OperatorRequired, + Abandoned, + Terminal, +} + +impl IlmRecoveryClassification { + pub fn permits_automatic_attempt(self) -> bool { + self == Self::Retrying + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IlmRecoveryErrorCode { + None, + SourceUnavailable, + SourceDivergent, + SourceCorrupt, + SourceGenerationChanged, + BackendUnavailable, + BackendTimeout, + BackendThrottled, + BackendServerError, + AttemptLeaseExpired, + RemoteVersionUnknown, + RemoteProbeAmbiguous, + RemoteProbeUnsupported, + LocalCommitAmbiguous, + CasConflict, + CleanupFailed, + OperatorDispositionRequired, + UnsupportedSchema, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IlmRecoverySourceCopy { + pub authority: String, + pub canonical_path: String, + pub etag: String, + pub encoded_len: u64, + pub content_sha256: String, +} + +impl IlmRecoverySourceCopy { + fn validate(&self) -> Result<()> { + if self.authority.trim().is_empty() { + return Err(IlmRecoveryControlError::Corrupt("source copy authority is empty")); + } + validate_canonical_source_path(&self.canonical_path)?; + if self.etag.trim().is_empty() { + return Err(IlmRecoveryControlError::Corrupt("source copy ETag is empty")); + } + if self.encoded_len == 0 { + return Err(IlmRecoveryControlError::Corrupt("source copy encoded length is zero")); + } + validate_sha256(&self.content_sha256, "source copy content checksum is invalid") + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IlmRecoverySourceGeneration { + pub source_schema: String, + pub source_etag: String, + pub content_sha256: String, + pub copy_set_sha256: String, + pub copies: Vec, +} + +impl IlmRecoverySourceGeneration { + pub fn new( + source_schema: impl Into, + source_etag: impl Into, + content_sha256: impl Into, + mut copies: Vec, + ) -> Result { + copies.sort_by(|left, right| (&left.authority, &left.canonical_path).cmp(&(&right.authority, &right.canonical_path))); + let copy_set_sha256 = copy_set_digest(&copies)?; + let generation = Self { + source_schema: source_schema.into(), + source_etag: source_etag.into(), + content_sha256: content_sha256.into(), + copy_set_sha256, + copies, + }; + generation.validate()?; + Ok(generation) + } + + fn validate(&self) -> Result<()> { + if self.source_schema.trim().is_empty() { + return Err(IlmRecoveryControlError::Corrupt("source schema is empty")); + } + if self.source_etag.trim().is_empty() { + return Err(IlmRecoveryControlError::Corrupt("source ETag is empty")); + } + validate_sha256(&self.content_sha256, "source content checksum is invalid")?; + validate_sha256(&self.copy_set_sha256, "source copy-set checksum is invalid")?; + if self.copies.is_empty() { + return Err(IlmRecoveryControlError::Corrupt("source copy set is empty")); + } + for copy in &self.copies { + copy.validate()?; + } + if !self + .copies + .windows(2) + .all(|pair| (&pair[0].authority, &pair[0].canonical_path) < (&pair[1].authority, &pair[1].canonical_path)) + { + return Err(IlmRecoveryControlError::Corrupt("source copies are not in unique canonical order")); + } + if copy_set_digest(&self.copies)? != self.copy_set_sha256 { + return Err(IlmRecoveryControlError::Corrupt("source copy-set checksum does not match copies")); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IlmRecoveryControlIdentity { + pub protocol: IlmRecoveryProtocol, + pub canonical_source_path: String, + pub stable_operation_identity: String, + pub record_class: String, +} + +impl IlmRecoveryControlIdentity { + pub fn source_operation_digest(&self) -> Result { + self.validate()?; + Ok(length_delimited_digest(&[ + self.protocol.as_str().as_bytes(), + self.canonical_source_path.as_bytes(), + self.stable_operation_identity.as_bytes(), + ])) + } + + fn validate(&self) -> Result<()> { + validate_canonical_source_path(&self.canonical_source_path)?; + if self.stable_operation_identity.trim().is_empty() { + return Err(IlmRecoveryControlError::Corrupt("stable operation identity is empty")); + } + if self.record_class.trim().is_empty() { + return Err(IlmRecoveryControlError::Corrupt("record class is empty")); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IlmRecoveryOwnerLease { + pub owner_id: String, + pub owner_epoch: Uuid, + pub lease_acquired_at_unix_nanos: i64, + pub lease_expires_at_unix_nanos: i64, +} + +impl IlmRecoveryOwnerLease { + fn validate(&self) -> Result<()> { + if self.owner_id.trim().is_empty() { + return Err(IlmRecoveryControlError::Corrupt("owner id is empty")); + } + if self.owner_epoch.is_nil() { + return Err(IlmRecoveryControlError::Corrupt("owner epoch is nil")); + } + if self.lease_acquired_at_unix_nanos <= 0 { + return Err(IlmRecoveryControlError::Corrupt("lease acquisition timestamp is not positive")); + } + if self.lease_expires_at_unix_nanos <= self.lease_acquired_at_unix_nanos { + return Err(IlmRecoveryControlError::Corrupt("lease expiry does not follow acquisition")); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IlmRecoveryControl { + pub identity: IlmRecoveryControlIdentity, + pub first_seen_at_unix_nanos: i64, + pub observed_source_generation: IlmRecoverySourceGeneration, + pub revision: u64, + pub classification: IlmRecoveryClassification, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner: Option, + pub attempt_count: u64, + pub consecutive_failure_count: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub first_failure_at_unix_nanos: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_failure_at_unix_nanos: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next_attempt_at_unix_nanos: Option, + pub last_error_code: IlmRecoveryErrorCode, +} + +impl IlmRecoveryControl { + pub fn new( + identity: IlmRecoveryControlIdentity, + observed_source_generation: IlmRecoverySourceGeneration, + classification: IlmRecoveryClassification, + now_unix_nanos: i64, + last_error_code: IlmRecoveryErrorCode, + ) -> Result { + let control = Self { + identity, + first_seen_at_unix_nanos: now_unix_nanos, + observed_source_generation, + revision: 1, + classification, + owner: None, + attempt_count: 0, + consecutive_failure_count: 0, + first_failure_at_unix_nanos: None, + last_failure_at_unix_nanos: None, + next_attempt_at_unix_nanos: None, + last_error_code, + }; + control.validate()?; + Ok(control) + } + + pub fn validate(&self) -> Result<()> { + self.identity.validate()?; + self.observed_source_generation.validate()?; + if self.first_seen_at_unix_nanos <= 0 { + return Err(IlmRecoveryControlError::Corrupt("first-seen timestamp is not positive")); + } + if self.revision == 0 { + return Err(IlmRecoveryControlError::Corrupt("revision is zero")); + } + if let Some(owner) = &self.owner { + owner.validate()?; + if !self.classification.permits_automatic_attempt() { + return Err(IlmRecoveryControlError::Corrupt("non-retrying control carries an owner lease")); + } + } + match ( + self.consecutive_failure_count, + self.first_failure_at_unix_nanos, + self.last_failure_at_unix_nanos, + self.next_attempt_at_unix_nanos, + ) { + (0, None, None, None) => {} + (0, Some(first), Some(last), None) if first > 0 && last >= first => {} + (0, _, _, _) => return Err(IlmRecoveryControlError::Corrupt("zero failures carry inconsistent history")), + (_, Some(first), Some(last), next) if first > 0 && last >= first => { + if self.classification == IlmRecoveryClassification::Retrying && next.is_none_or(|next| next <= last) { + return Err(IlmRecoveryControlError::Corrupt("retrying control has no future retry timestamp")); + } + } + _ => return Err(IlmRecoveryControlError::Corrupt("failure counters and timestamps are inconsistent")), + } + if u64::from(self.consecutive_failure_count) > self.attempt_count { + return Err(IlmRecoveryControlError::Corrupt("consecutive failures exceed lifetime attempts")); + } + if self.classification == IlmRecoveryClassification::Retrying + && self.last_error_code == IlmRecoveryErrorCode::None + && self.consecutive_failure_count > 0 + { + return Err(IlmRecoveryControlError::Corrupt("failed retry has no bounded error code")); + } + Ok(()) + } + + pub fn should_attempt_at(&self, now_unix_nanos: i64) -> bool { + self.classification.permits_automatic_attempt() + && self + .owner + .as_ref() + .is_none_or(|owner| owner.lease_expires_at_unix_nanos <= now_unix_nanos) + && self.next_attempt_at_unix_nanos.is_none_or(|next| next <= now_unix_nanos) + } + + pub fn claim( + &mut self, + owner_id: impl Into, + owner_epoch: Uuid, + now_unix_nanos: i64, + lease_duration_nanos: i64, + ) -> Result<()> { + self.claim_for_source_generation( + owner_id, + owner_epoch, + now_unix_nanos, + lease_duration_nanos, + self.observed_source_generation.clone(), + ) + } + + pub fn claim_for_source_generation( + &mut self, + owner_id: impl Into, + owner_epoch: Uuid, + now_unix_nanos: i64, + lease_duration_nanos: i64, + observed_source_generation: IlmRecoverySourceGeneration, + ) -> Result<()> { + if !self.should_attempt_at(now_unix_nanos) { + return Err(IlmRecoveryControlError::InvalidSuccessor("control is not ready for an attempt")); + } + let lease_expires_at_unix_nanos = now_unix_nanos + .checked_add(lease_duration_nanos) + .ok_or(IlmRecoveryControlError::Corrupt("lease timestamp overflow"))?; + self.bump_revision()?; + self.observed_source_generation = observed_source_generation; + self.owner = Some(IlmRecoveryOwnerLease { + owner_id: owner_id.into(), + owner_epoch, + lease_acquired_at_unix_nanos: now_unix_nanos, + lease_expires_at_unix_nanos, + }); + self.attempt_count = self + .attempt_count + .checked_add(1) + .ok_or(IlmRecoveryControlError::Corrupt("attempt count overflow"))?; + self.validate() + } + + pub fn record_retryable_failure(&mut self, now_unix_nanos: i64, code: IlmRecoveryErrorCode) -> Result<()> { + if code == IlmRecoveryErrorCode::None { + return Err(IlmRecoveryControlError::InvalidSuccessor("retryable failure requires an error code")); + } + let attempt_started_at = self + .owner + .as_ref() + .ok_or(IlmRecoveryControlError::InvalidSuccessor("retryable failure requires an owner lease"))? + .lease_acquired_at_unix_nanos; + if now_unix_nanos < attempt_started_at { + return Err(IlmRecoveryControlError::InvalidSuccessor("retryable failure predates its owner claim")); + } + let failures = self + .consecutive_failure_count + .checked_add(1) + .ok_or(IlmRecoveryControlError::Corrupt("failure count overflow"))?; + let first_failure_at = self.first_failure_at_unix_nanos.unwrap_or(attempt_started_at); + let age = now_unix_nanos.saturating_sub(first_failure_at); + self.bump_revision()?; + self.owner = None; + self.consecutive_failure_count = failures; + self.first_failure_at_unix_nanos = Some(first_failure_at); + self.last_failure_at_unix_nanos = Some(now_unix_nanos); + self.last_error_code = code; + if failures >= MAX_RECOVERY_ATTEMPTS + || self.attempt_count >= u64::from(MAX_RECOVERY_ATTEMPTS) + || age >= MAX_RECOVERY_AGE_NANOS + { + self.classification = IlmRecoveryClassification::OperatorRequired; + self.next_attempt_at_unix_nanos = None; + } else { + self.classification = IlmRecoveryClassification::Retrying; + self.next_attempt_at_unix_nanos = Some( + now_unix_nanos + .checked_add(retry_delay_nanos( + &self.observed_source_generation.copy_set_sha256, + self.attempt_count, + failures, + )) + .ok_or(IlmRecoveryControlError::Corrupt("retry timestamp overflow"))?, + ); + } + self.validate() + } + + pub fn record_expired_attempt(&mut self, now_unix_nanos: i64) -> Result<()> { + let lease_expires_at = self + .owner + .as_ref() + .ok_or(IlmRecoveryControlError::InvalidSuccessor("expired attempt requires an owner lease"))? + .lease_expires_at_unix_nanos; + if lease_expires_at > now_unix_nanos { + return Err(IlmRecoveryControlError::InvalidSuccessor("attempt owner lease is still active")); + } + self.record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::AttemptLeaseExpired) + } + + pub fn refresh_owned_source_generation(&mut self, observed_source_generation: IlmRecoverySourceGeneration) -> Result<()> { + if self.owner.is_none() || self.classification != IlmRecoveryClassification::Retrying { + return Err(IlmRecoveryControlError::InvalidSuccessor( + "source generation refresh requires a retrying owner", + )); + } + self.bump_revision()?; + self.observed_source_generation = observed_source_generation; + self.validate() + } + + pub fn finish_attempt(&mut self, classification: IlmRecoveryClassification, code: IlmRecoveryErrorCode) -> Result<()> { + if self.owner.is_none() { + return Err(IlmRecoveryControlError::InvalidSuccessor("finishing an attempt requires an owner lease")); + } + if classification == IlmRecoveryClassification::Retrying { + return Err(IlmRecoveryControlError::InvalidSuccessor( + "successful attempt result cannot remain retrying", + )); + } + self.bump_revision()?; + self.owner = None; + self.classification = classification; + self.consecutive_failure_count = 0; + self.next_attempt_at_unix_nanos = None; + self.last_error_code = code; + self.validate() + } + + pub fn validate_successor(&self, next: &Self) -> Result<()> { + self.validate()?; + next.validate()?; + if self.identity != next.identity || self.first_seen_at_unix_nanos != next.first_seen_at_unix_nanos { + return Err(IlmRecoveryControlError::InvalidSuccessor("immutable identity changed")); + } + if self.revision.checked_add(1) != Some(next.revision) { + return Err(IlmRecoveryControlError::InvalidSuccessor("revision did not advance by one")); + } + match (&self.owner, &next.owner) { + (Some(current_owner), Some(next_owner)) if current_owner == next_owner => { + self.validate_source_refresh_successor(next) + } + (_, Some(_)) => self.validate_claim_successor(next), + (Some(_), None) + if self + .consecutive_failure_count + .checked_add(1) + .is_some_and(|failures| next.consecutive_failure_count == failures) => + { + self.validate_failure_successor(next) + } + (Some(_), None) => self.validate_finish_successor(next), + (None, None) => Err(IlmRecoveryControlError::InvalidSuccessor( + "ownerless control cannot advance without a claim", + )), + } + } + + fn validate_claim_successor(&self, next: &Self) -> Result<()> { + if self.classification != IlmRecoveryClassification::Retrying + || next.classification != IlmRecoveryClassification::Retrying + || self + .attempt_count + .checked_add(1) + .is_none_or(|attempts| next.attempt_count != attempts) + || next.consecutive_failure_count != self.consecutive_failure_count + || next.first_failure_at_unix_nanos != self.first_failure_at_unix_nanos + || next.last_failure_at_unix_nanos != self.last_failure_at_unix_nanos + || next.next_attempt_at_unix_nanos != self.next_attempt_at_unix_nanos + || next.last_error_code != self.last_error_code + { + return Err(IlmRecoveryControlError::InvalidSuccessor("claim changed non-owner recovery state")); + } + Ok(()) + } + + fn validate_source_refresh_successor(&self, next: &Self) -> Result<()> { + if self.classification != IlmRecoveryClassification::Retrying + || next.classification != IlmRecoveryClassification::Retrying + || next.attempt_count != self.attempt_count + || next.consecutive_failure_count != self.consecutive_failure_count + || next.first_failure_at_unix_nanos != self.first_failure_at_unix_nanos + || next.last_failure_at_unix_nanos != self.last_failure_at_unix_nanos + || next.next_attempt_at_unix_nanos != self.next_attempt_at_unix_nanos + || next.last_error_code != self.last_error_code + { + return Err(IlmRecoveryControlError::InvalidSuccessor( + "source generation refresh changed non-source recovery state", + )); + } + Ok(()) + } + + fn validate_failure_successor(&self, next: &Self) -> Result<()> { + let owner = self + .owner + .as_ref() + .ok_or(IlmRecoveryControlError::InvalidSuccessor("retry failure has no owner claim"))?; + if next.observed_source_generation != self.observed_source_generation + || next.attempt_count != self.attempt_count + || next.first_failure_at_unix_nanos != self.first_failure_at_unix_nanos.or(Some(owner.lease_acquired_at_unix_nanos)) + || next.last_error_code == IlmRecoveryErrorCode::None + { + return Err(IlmRecoveryControlError::InvalidSuccessor("retry failure changed immutable attempt state")); + } + let last_failure = next + .last_failure_at_unix_nanos + .ok_or(IlmRecoveryControlError::InvalidSuccessor("retry failure has no timestamp"))?; + let first_failure = next + .first_failure_at_unix_nanos + .ok_or(IlmRecoveryControlError::InvalidSuccessor("retry failure has no first timestamp"))?; + if self + .owner + .as_ref() + .is_none_or(|owner| last_failure < owner.lease_acquired_at_unix_nanos) + { + return Err(IlmRecoveryControlError::InvalidSuccessor("retry failure predates its owner claim")); + } + let exhausted = next.consecutive_failure_count >= MAX_RECOVERY_ATTEMPTS + || next.attempt_count >= u64::from(MAX_RECOVERY_ATTEMPTS) + || last_failure.saturating_sub(first_failure) >= MAX_RECOVERY_AGE_NANOS; + let expected_next = if exhausted { + None + } else { + Some( + last_failure + .checked_add(retry_delay_nanos( + &next.observed_source_generation.copy_set_sha256, + next.attempt_count, + next.consecutive_failure_count, + )) + .ok_or(IlmRecoveryControlError::InvalidSuccessor("retry timestamp overflowed"))?, + ) + }; + if next.classification + != if exhausted { + IlmRecoveryClassification::OperatorRequired + } else { + IlmRecoveryClassification::Retrying + } + || next.next_attempt_at_unix_nanos != expected_next + { + return Err(IlmRecoveryControlError::InvalidSuccessor( + "retry failure has an invalid terminal or backoff state", + )); + } + Ok(()) + } + + fn validate_finish_successor(&self, next: &Self) -> Result<()> { + if next.observed_source_generation != self.observed_source_generation + || next.attempt_count != self.attempt_count + || next.classification == IlmRecoveryClassification::Retrying + || next.consecutive_failure_count != 0 + || next.first_failure_at_unix_nanos != self.first_failure_at_unix_nanos + || next.last_failure_at_unix_nanos != self.last_failure_at_unix_nanos + || next.next_attempt_at_unix_nanos.is_some() + { + return Err(IlmRecoveryControlError::InvalidSuccessor( + "finished attempt changed immutable recovery state", + )); + } + Ok(()) + } + + pub fn encode(&self) -> Result> { + self.validate()?; + let control_bytes = serde_json::to_vec(self)?; + let persisted = PersistedIlmRecoveryControl { + schema: ILM_RECOVERY_CONTROL_SCHEMA.to_string(), + content_sha256: hex_sha256(&control_bytes, ToOwned::to_owned), + control: self.clone(), + }; + let encoded = serde_json::to_vec(&persisted)?; + if encoded.len() > MAX_ILM_RECOVERY_CONTROL_SIZE { + return Err(IlmRecoveryControlError::Corrupt("encoded control exceeds maximum size")); + } + Ok(encoded) + } + + pub fn decode(expected_control_id: &str, data: &[u8]) -> Result { + validate_sha256(expected_control_id, "control id is invalid")?; + if data.len() > MAX_ILM_RECOVERY_CONTROL_SIZE { + return Err(IlmRecoveryControlError::Corrupt("encoded control exceeds maximum size")); + } + let persisted: PersistedIlmRecoveryControl = serde_json::from_slice(data)?; + if persisted.schema != ILM_RECOVERY_CONTROL_SCHEMA { + return Err(IlmRecoveryControlError::UnsupportedSchema(persisted.schema)); + } + validate_sha256(&persisted.content_sha256, "content checksum is invalid")?; + let control_bytes = serde_json::to_vec(&persisted.control)?; + if hex_sha256(&control_bytes, ToOwned::to_owned) != persisted.content_sha256 { + return Err(IlmRecoveryControlError::ChecksumMismatch); + } + if persisted.control.identity.source_operation_digest()? != expected_control_id { + return Err(IlmRecoveryControlError::Corrupt("control id does not match record key")); + } + persisted.control.validate()?; + Ok(persisted.control) + } + + fn bump_revision(&mut self) -> Result<()> { + self.revision = self + .revision + .checked_add(1) + .ok_or(IlmRecoveryControlError::Corrupt("revision overflow"))?; + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObservedIlmRecoveryControl { + pub control: IlmRecoveryControl, + pub etag: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObservedIlmRecoverySource { + pub generation: IlmRecoverySourceGeneration, + pub canonical_data: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct IlmRecoveryControlView { + pub control_id: String, + pub protocol: IlmRecoveryProtocol, + pub classification: IlmRecoveryClassification, + pub schema: &'static str, + pub revision: u64, + pub attempt_count: u64, + pub consecutive_failure_count: u32, + pub first_seen_at_unix_nanos: i64, + pub first_failure_at_unix_nanos: Option, + pub last_failure_at_unix_nanos: Option, + pub next_attempt_at_unix_nanos: Option, + pub last_error_code: IlmRecoveryErrorCode, + pub source_schema: String, + pub source_generation_sha256: String, + pub copy_set_sha256: String, + pub source_copy_count: usize, +} + +impl IlmRecoveryControlView { + fn from_control(control_id: String, control: &IlmRecoveryControl) -> EcstoreResult { + let generation = serde_json::to_vec(&control.observed_source_generation).map_err(Error::other)?; + Ok(Self { + control_id, + protocol: control.identity.protocol, + classification: control.classification, + schema: ILM_RECOVERY_CONTROL_SCHEMA, + revision: control.revision, + attempt_count: control.attempt_count, + consecutive_failure_count: control.consecutive_failure_count, + first_seen_at_unix_nanos: control.first_seen_at_unix_nanos, + first_failure_at_unix_nanos: control.first_failure_at_unix_nanos, + last_failure_at_unix_nanos: control.last_failure_at_unix_nanos, + next_attempt_at_unix_nanos: control.next_attempt_at_unix_nanos, + last_error_code: control.last_error_code, + source_schema: control.observed_source_generation.source_schema.clone(), + source_generation_sha256: hex_sha256(&generation, ToOwned::to_owned), + copy_set_sha256: control.observed_source_generation.copy_set_sha256.clone(), + source_copy_count: control.observed_source_generation.copies.len(), + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct IlmRecoveryControlPage { + pub records: Vec, + pub next_marker: Option, + pub truncated: bool, + pub incomplete: bool, +} + +impl ObservedIlmRecoverySource { + pub fn is_consistent(&self) -> bool { + self.canonical_data.is_some() + && self.generation.copies.iter().all(|copy| { + copy.etag == self.generation.source_etag + && copy.content_sha256 == self.generation.content_sha256 + && copy.encoded_len + == self + .canonical_data + .as_ref() + .map_or(0, |data| u64::try_from(data.len()).unwrap_or(u64::MAX)) + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PersistedIlmRecoveryControl { + schema: String, + content_sha256: String, + control: IlmRecoveryControl, +} + +pub fn recovery_control_record_object_name(protocol: IlmRecoveryProtocol, control_id: &str) -> Result { + validate_sha256(control_id, "control id is invalid")?; + Ok(format!( + "{}/{}/{}/{}/{}.json", + ILM_RECOVERY_CONTROL_PREFIX, + protocol.as_str(), + &control_id[..2], + &control_id[2..4], + control_id + )) +} + +pub fn recovery_control_id_from_record_object_name(object: &str) -> Result<(IlmRecoveryProtocol, String)> { + let suffix = object + .strip_prefix(ILM_RECOVERY_CONTROL_PREFIX) + .and_then(|suffix| suffix.strip_prefix('/')) + .ok_or(IlmRecoveryControlError::Corrupt("control record path has wrong prefix"))?; + let mut parts = suffix.split('/'); + let protocol = match parts.next() { + Some("transition_transaction") => IlmRecoveryProtocol::TransitionTransaction, + Some("tier_delete_journal") => IlmRecoveryProtocol::TierDeleteJournal, + Some("tier_delete_manifest") => IlmRecoveryProtocol::TierDeleteManifest, + _ => return Err(IlmRecoveryControlError::Corrupt("control record protocol is invalid")), + }; + let shard_a = parts + .next() + .ok_or(IlmRecoveryControlError::Corrupt("control record path is incomplete"))?; + let shard_b = parts + .next() + .ok_or(IlmRecoveryControlError::Corrupt("control record path is incomplete"))?; + let control_id = parts + .next() + .and_then(|name| name.strip_suffix(".json")) + .ok_or(IlmRecoveryControlError::Corrupt("control record suffix is invalid"))?; + if parts.next().is_some() { + return Err(IlmRecoveryControlError::Corrupt("control record path is not canonical")); + } + validate_sha256(control_id, "control id is invalid")?; + if shard_a != &control_id[..2] || shard_b != &control_id[2..4] { + return Err(IlmRecoveryControlError::Corrupt("control record shard does not match control id")); + } + Ok((protocol, control_id.to_string())) +} + +pub async fn save_recovery_control_if_absent(api: Arc, control: &IlmRecoveryControl) -> EcstoreResult<()> { + let control_id = control + .identity + .source_operation_digest() + .map_err(recovery_control_store_error)?; + let object = + recovery_control_record_object_name(control.identity.protocol, &control_id).map_err(recovery_control_store_error)?; + let data = control.encode().map_err(recovery_control_store_error)?; + config_boundary::save_config_with_opts( + api.clone(), + &object, + data.clone(), + &ObjectOptions { + max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_none_match: Some("*".to_string()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await?; + api.record_durable_ilm_decommission_progress(&object, &data).await +} + +pub async fn observe_recovery_source( + api: Arc, + canonical_path: &str, + source_schema: &str, +) -> EcstoreResult { + validate_canonical_source_path(canonical_path).map_err(recovery_control_store_error)?; + if source_schema.trim().is_empty() { + return Err(Error::other("ILM recovery source schema is empty")); + } + + let mut copies = Vec::new(); + let mut observations = Vec::new(); + for set in api.all_set_disks() { + let authority = format!("pool-{}/set-{}", set.pool_index, set.set_index); + match config_boundary::read_config_with_metadata(set, canonical_path, &ObjectOptions::default()).await { + Ok((data, metadata)) => { + let etag = metadata + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("ILM recovery source copy is missing an ETag"))?; + let encoded_len = + u64::try_from(data.len()).map_err(|_| Error::other("ILM recovery source copy length does not fit u64"))?; + let content_sha256 = hex_sha256(&data, ToOwned::to_owned); + copies.push(IlmRecoverySourceCopy { + authority, + canonical_path: canonical_path.to_string(), + etag: etag.clone(), + encoded_len, + content_sha256: content_sha256.clone(), + }); + observations.push((etag, content_sha256, data)); + } + Err(err) if recovery_source_is_missing(&err) => {} + Err(err) => return Err(err), + } + } + let Some((source_etag, content_sha256, first_data)) = observations.first().cloned() else { + return Err(Error::ConfigNotFound); + }; + let consistent = observations + .iter() + .all(|(etag, digest, data)| etag == &source_etag && digest == &content_sha256 && data == &first_data); + let generation = IlmRecoverySourceGeneration::new(source_schema, source_etag, content_sha256, copies) + .map_err(recovery_control_store_error)?; + Ok(ObservedIlmRecoverySource { + generation, + canonical_data: consistent.then_some(first_data), + }) +} + +pub async fn load_recovery_control( + api: Arc, + protocol: IlmRecoveryProtocol, + control_id: &str, +) -> EcstoreResult { + let object = recovery_control_record_object_name(protocol, control_id).map_err(recovery_control_store_error)?; + let (data, metadata) = config_boundary::read_config_with_metadata(api, &object, &ObjectOptions::default()).await?; + let etag = metadata + .etag + .filter(|etag| !etag.trim().is_empty()) + .ok_or_else(|| Error::other("ILM recovery control is missing an ETag"))?; + let control = IlmRecoveryControl::decode(control_id, &data).map_err(recovery_control_store_error)?; + if control.identity.protocol != protocol { + return Err(Error::other("ILM recovery control protocol does not match record path")); + } + Ok(ObservedIlmRecoveryControl { control, etag }) +} + +pub async fn inspect_recovery_control(api: Arc, control_id: &str) -> EcstoreResult { + validate_sha256(control_id, "control id is invalid").map_err(recovery_control_store_error)?; + for protocol in IlmRecoveryProtocol::all() { + match load_recovery_control(api.clone(), protocol, control_id).await { + Ok(observed) => return IlmRecoveryControlView::from_control(control_id.to_string(), &observed.control), + Err(Error::ConfigNotFound) => {} + Err(err) => return Err(err), + } + } + Err(Error::ConfigNotFound) +} + +pub async fn list_recovery_controls( + api: Arc, + protocol: IlmRecoveryProtocol, + classification: Option, + limit: usize, + marker: Option, +) -> EcstoreResult { + if !(1..=1_000).contains(&limit) { + return Err(Error::other("ILM recovery control list limit must be between 1 and 1000")); + } + let prefix = format!("{}/{}/", ILM_RECOVERY_CONTROL_PREFIX, protocol.as_str()); + let page = api + .clone() + .list_objects_v2( + RUSTFS_META_BUCKET, + &prefix, + marker, + None, + i32::try_from(limit).unwrap_or(1_000), + false, + None, + false, + ) + .await?; + if page.is_truncated && page.next_continuation_token.is_none() { + return Err(Error::other( + "ILM recovery control list returned a truncated page without a continuation marker", + )); + } + + let mut records = Vec::new(); + let mut incomplete = false; + for object in page.objects { + let parsed = recovery_control_id_from_record_object_name(&object.name); + let (path_protocol, control_id) = match parsed { + Ok(parsed) if parsed.0 == protocol => parsed, + Ok(_) | Err(_) => { + incomplete = true; + continue; + } + }; + match load_recovery_control(api.clone(), path_protocol, &control_id).await { + Ok(observed) if classification.is_none_or(|filter| observed.control.classification == filter) => { + records.push(IlmRecoveryControlView::from_control(control_id, &observed.control)?); + } + Ok(_) => {} + Err(Error::ConfigNotFound) => {} + Err(_) => incomplete = true, + } + } + + Ok(IlmRecoveryControlPage { + records, + next_marker: page.next_continuation_token, + truncated: page.is_truncated, + incomplete, + }) +} + +pub async fn save_recovery_control_if_current( + api: Arc, + current: &ObservedIlmRecoveryControl, + next: &IlmRecoveryControl, +) -> EcstoreResult<()> { + current + .control + .validate_successor(next) + .map_err(recovery_control_store_error)?; + let control_id = current + .control + .identity + .source_operation_digest() + .map_err(recovery_control_store_error)?; + let authoritative = load_recovery_control(api.clone(), current.control.identity.protocol, &control_id).await?; + if &authoritative != current { + return Err(Error::PreconditionFailed); + } + let object = recovery_control_record_object_name(current.control.identity.protocol, &control_id) + .map_err(recovery_control_store_error)?; + let data = next.encode().map_err(recovery_control_store_error)?; + config_boundary::save_config_with_opts( + api.clone(), + &object, + data.clone(), + &ObjectOptions { + max_parity: true, + write_completion: crate::object_api::WriteCompletion::TailDrained, + http_preconditions: Some(HTTPPreconditions { + if_match: Some(current.etag.clone()), + ..Default::default() + }), + ..Default::default() + }, + ) + .await?; + api.record_durable_ilm_decommission_progress(&object, &data).await +} + +fn retry_delay_nanos(copy_set_sha256: &str, attempt_count: u64, consecutive_failure_count: u32) -> i64 { + let exponent = consecutive_failure_count.saturating_sub(1).min(6); + let base = MIN_RETRY_DELAY_NANOS + .saturating_mul(1_i64 << exponent) + .min(MAX_RETRY_DELAY_NANOS); + let seed = length_delimited_digest(&[copy_set_sha256.as_bytes(), &attempt_count.to_be_bytes()]); + let jitter_bucket = u8::from_str_radix(&seed[..2], 16).unwrap_or(0) % 21; + base.saturating_mul(i64::from(80 + jitter_bucket)) / 100 +} + +fn copy_set_digest(copies: &[IlmRecoverySourceCopy]) -> Result { + let encoded = serde_json::to_vec(copies)?; + Ok(hex_sha256(&encoded, ToOwned::to_owned)) +} + +fn length_delimited_digest(parts: &[&[u8]]) -> String { + let mut encoded = Vec::new(); + for part in parts { + encoded.extend_from_slice(&(part.len() as u64).to_be_bytes()); + encoded.extend_from_slice(part); + } + hex_sha256(&encoded, ToOwned::to_owned) +} + +fn validate_canonical_source_path(path: &str) -> Result<()> { + if path.is_empty() || path.starts_with('/') || path.ends_with('/') || path.split('/').any(|part| part.is_empty()) { + return Err(IlmRecoveryControlError::Corrupt("canonical source path is invalid")); + } + Ok(()) +} + +fn validate_sha256(value: &str, message: &'static str) -> Result<()> { + if !is_sha256_checksum(value) + || value + .bytes() + .any(|byte| byte.is_ascii_hexdigit() && byte.is_ascii_uppercase()) + { + return Err(IlmRecoveryControlError::Corrupt(message)); + } + Ok(()) +} + +fn recovery_control_store_error(err: IlmRecoveryControlError) -> Error { + Error::other(err) +} + +fn recovery_source_is_missing(err: &Error) -> bool { + matches!( + err, + Error::ConfigNotFound | Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::VersionNotFound(_, _, _) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SOURCE_PATH: &str = "ilm/transition-transactions/records/12/34/1234567890abcdef1234567890abcdef.json"; + + fn generation() -> IlmRecoverySourceGeneration { + let content_sha256 = hex_sha256(b"source", ToOwned::to_owned); + IlmRecoverySourceGeneration::new( + "rustfs-transition-transaction-v1", + "etag-a", + content_sha256.clone(), + vec![ + IlmRecoverySourceCopy { + authority: "pool-1/set-0".to_string(), + canonical_path: SOURCE_PATH.to_string(), + etag: "etag-b".to_string(), + encoded_len: 6, + content_sha256: content_sha256.clone(), + }, + IlmRecoverySourceCopy { + authority: "pool-0/set-1".to_string(), + canonical_path: SOURCE_PATH.to_string(), + etag: "etag-a".to_string(), + encoded_len: 6, + content_sha256, + }, + ], + ) + .expect("source generation should build") + } + + fn control() -> IlmRecoveryControl { + IlmRecoveryControl::new( + IlmRecoveryControlIdentity { + protocol: IlmRecoveryProtocol::TransitionTransaction, + canonical_source_path: SOURCE_PATH.to_string(), + stable_operation_identity: "12345678-90ab-cdef-1234-567890abcdef".to_string(), + record_class: "transition_transaction_v1".to_string(), + }, + generation(), + IlmRecoveryClassification::Retrying, + 1_000_000_000, + IlmRecoveryErrorCode::None, + ) + .expect("control should build") + } + + #[test] + fn recovery_control_round_trip_and_canonical_path() { + let control = control(); + let control_id = control.identity.source_operation_digest().expect("control id should derive"); + let path = + recovery_control_record_object_name(control.identity.protocol, &control_id).expect("control path should build"); + assert_eq!( + recovery_control_id_from_record_object_name(&path).expect("control path should parse"), + (control.identity.protocol, control_id.clone()) + ); + let encoded = control.encode().expect("control should encode"); + assert_eq!(IlmRecoveryControl::decode(&control_id, &encoded).expect("control should decode"), control); + } + + #[test] + fn recovery_control_rejects_noncanonical_copy_set_and_tampering() { + let mut noncanonical = control(); + noncanonical.observed_source_generation.copies.swap(0, 1); + assert!(matches!(noncanonical.validate(), Err(IlmRecoveryControlError::Corrupt(_)))); + + let control = control(); + let control_id = control.identity.source_operation_digest().expect("control id should derive"); + let mut persisted: serde_json::Value = + serde_json::from_slice(&control.encode().expect("control should encode")).expect("encoded control should be json"); + persisted["control"]["attempt_count"] = serde_json::json!(9); + let tampered = serde_json::to_vec(&persisted).expect("tampered json should encode"); + assert!(matches!( + IlmRecoveryControl::decode(&control_id, &tampered), + Err(IlmRecoveryControlError::ChecksumMismatch) + )); + } + + #[test] + fn recovery_control_persists_deterministic_bounded_backoff() { + let mut first = control(); + let mut second = first.clone(); + for control in [&mut first, &mut second] { + control + .claim("node-a", Uuid::new_v4(), 2_000_000_000, 300_000_000_000) + .expect("attempt should claim"); + control + .record_retryable_failure(3_000_000_000, IlmRecoveryErrorCode::BackendTimeout) + .expect("failure should schedule retry"); + } + assert_eq!(first.next_attempt_at_unix_nanos, second.next_attempt_at_unix_nanos); + let delay = first.next_attempt_at_unix_nanos.expect("retry time") - 3_000_000_000; + assert!((48_000_000_000..=60_000_000_000).contains(&delay)); + assert!(!first.should_attempt_at(first.next_attempt_at_unix_nanos.expect("retry time") - 1)); + assert!(first.should_attempt_at(first.next_attempt_at_unix_nanos.expect("retry time"))); + } + + #[test] + fn recovery_control_stops_after_bounded_failures() { + let mut control = control(); + let mut now = 2_000_000_000; + for _ in 0..MAX_RECOVERY_ATTEMPTS { + let ready = control.next_attempt_at_unix_nanos.unwrap_or(now); + now = now.max(ready); + control + .claim("node-a", Uuid::new_v4(), now, 300_000_000_000) + .expect("attempt should claim"); + control + .record_retryable_failure(now + 1, IlmRecoveryErrorCode::BackendTimeout) + .expect("failure should persist"); + now += 2; + } + assert_eq!(control.classification, IlmRecoveryClassification::OperatorRequired); + assert_eq!(control.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS)); + assert_eq!(control.next_attempt_at_unix_nanos, None); + } + + #[test] + fn recovery_control_expired_timeout_and_cancellation_attempts_stop_at_bounds() { + let mut active = control(); + active + .claim("node-a", Uuid::new_v4(), 2_000_000_000, 2) + .expect("active attempt should claim"); + assert!(matches!( + active.record_expired_attempt(2_000_000_001), + Err(IlmRecoveryControlError::InvalidSuccessor("attempt owner lease is still active")) + )); + + let mut bounded = control(); + let mut now = 2_000_000_000; + for attempt in 1..=MAX_RECOVERY_ATTEMPTS { + now = now.max(bounded.next_attempt_at_unix_nanos.unwrap_or(now)); + bounded + .claim("node-a", Uuid::new_v4(), now, 1) + .expect("timeout or cancellation attempt should claim"); + let abandonment = if attempt % 2 == 0 { "cancellation" } else { "timeout" }; + bounded + .record_expired_attempt(now + 1) + .unwrap_or_else(|err| panic!("expired {abandonment} attempt should consume its budget: {err}")); + if attempt < MAX_RECOVERY_ATTEMPTS { + assert_eq!(bounded.classification, IlmRecoveryClassification::Retrying); + } + now += 2; + } + + assert_eq!(bounded.classification, IlmRecoveryClassification::OperatorRequired); + assert_eq!(bounded.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS)); + assert_eq!(bounded.consecutive_failure_count, MAX_RECOVERY_ATTEMPTS); + assert_eq!(bounded.last_error_code, IlmRecoveryErrorCode::AttemptLeaseExpired); + assert_eq!(bounded.next_attempt_at_unix_nanos, None); + + let mut younger = control(); + younger + .claim("node-a", Uuid::new_v4(), now, 1) + .expect("younger attempt should claim"); + younger + .record_expired_attempt(now + MAX_RECOVERY_AGE_NANOS - 1) + .expect("younger expired attempt should be recorded"); + assert_eq!(younger.classification, IlmRecoveryClassification::Retrying); + + let mut aged = control(); + aged.claim("node-a", Uuid::new_v4(), now, 1) + .expect("aged attempt should claim"); + aged.record_expired_attempt(now + MAX_RECOVERY_AGE_NANOS) + .expect("seven-day expired attempt should be recorded"); + assert_eq!(aged.classification, IlmRecoveryClassification::OperatorRequired); + assert_eq!(aged.consecutive_failure_count, 1); + } + + #[test] + fn recovery_control_successor_preserves_lineage_and_generation() { + let current = control(); + let mut next = current.clone(); + let mut advanced_generation = generation(); + advanced_generation.source_schema = "rustfs-transition-transaction-v2".to_string(); + next.claim_for_source_generation("node-a", Uuid::new_v4(), 2_000_000_000, 300_000_000_000, advanced_generation) + .expect("attempt should claim"); + current + .validate_successor(&next) + .expect("a claim may adopt a newly proven source generation"); + + let mut changed = next.clone(); + let mut refreshed_generation = changed.observed_source_generation.clone(); + refreshed_generation.source_schema = "rustfs-transition-transaction-v3".to_string(); + changed + .refresh_owned_source_generation(refreshed_generation) + .expect("the current owner may refresh a newly proven source generation"); + next.validate_successor(&changed) + .expect("owned source refresh should be a legal successor"); + + let mut invalid = changed.clone(); + invalid.revision += 1; + invalid.attempt_count += 1; + assert!(matches!( + changed.validate_successor(&invalid), + Err(IlmRecoveryControlError::InvalidSuccessor(_)) + )); + } + + #[test] + fn recovery_control_view_redacts_source_and_owner_details() { + let mut control = control(); + control + .claim("secret-node-id", Uuid::new_v4(), 2_000_000_000, 300_000_000_000) + .expect("attempt should claim"); + let control_id = control.identity.source_operation_digest().expect("control id should derive"); + let view = IlmRecoveryControlView::from_control(control_id, &control).expect("view should build"); + let encoded = serde_json::to_string(&view).expect("view should encode"); + + for secret in [ + SOURCE_PATH, + "etag-a", + "etag-b", + "secret-node-id", + "12345678-90ab-cdef-1234-567890abcdef", + ] { + assert!(!encoded.contains(secret), "redacted view leaked {secret}"); + } + assert!(encoded.contains(ILM_RECOVERY_CONTROL_SCHEMA)); + assert!(encoded.contains("source_generation_sha256")); + } +} diff --git a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs index ad2241629..2d90e9762 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs @@ -575,7 +575,7 @@ pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idemp #[cfg(test)] static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity( obj_name: &str, rv_id: &str, @@ -706,15 +706,16 @@ pub(crate) fn transitioned_delete_journal_entry_for_source( #[cfg(test)] mod test { + #[cfg(feature = "test-util")] + use super::delete_confirmed_transition_candidate_exact_with_manager_and_identity; use rustfs_s3_client::signer_error::invalid_utf8_header_error; use super::{ CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry, RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity, - delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent, - delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error, - is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure, - transitioned_delete_journal_entry, transitioned_force_delete_journal_entry, + delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity, + is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, + should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry, }; use crate::storage_api_contracts::lifecycle::TransitionedObject; use rustfs_filemeta::TransitionVersionState; diff --git a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs index 82e32f598..426956d13 100644 --- a/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs +++ b/crates/ecstore/src/bucket/lifecycle/transition_transaction.rs @@ -23,6 +23,11 @@ use uuid::Uuid; use crate::bucket::lifecycle::config_boundary; use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE; use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; +use crate::bucket::lifecycle::recovery_control::{ + IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol, + ObservedIlmRecoveryControl, load_recovery_control, observe_recovery_source, recovery_control_record_object_name, + save_recovery_control_if_absent, save_recovery_control_if_current, +}; use crate::bucket::lifecycle::tier_sweeper::{ delete_confirmed_transition_candidate_exact_with_lease_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity, @@ -44,6 +49,7 @@ const EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY: &str = "lifecycle_transit pub const DEFAULT_TRANSITION_TRANSACTION_RECOVERY_LIMIT: usize = 1_000; const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(60); const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300); +const TRANSITION_RECOVERY_CONTROL_LEASE_NANOS: i64 = 15 * 60 * 1_000_000_000; pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1"; pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions"; pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix; @@ -737,9 +743,11 @@ pub enum TransitionTransactionRecoveryOutcome { RemoteCandidateDeleted, RecordDeleted, Retained, + RetainedAmbiguous(IlmRecoveryErrorCode), + OperatorRequired(IlmRecoveryErrorCode), } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] #[derive(Default)] struct TransitionRecoveryClaimBarrierState { transaction_id: Uuid, @@ -747,17 +755,17 @@ struct TransitionRecoveryClaimBarrierState { release: tokio::sync::Notify, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) struct TransitionRecoveryClaimBarrier { state: Arc, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] static TRANSITION_RECOVERY_CLAIM_BARRIER: std::sync::OnceLock< std::sync::Mutex>>, > = std::sync::OnceLock::new(); -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl TransitionRecoveryClaimBarrier { pub(crate) fn install(transaction_id: Uuid) -> Self { let state = Arc::new(TransitionRecoveryClaimBarrierState { @@ -788,7 +796,7 @@ impl TransitionRecoveryClaimBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl Drop for TransitionRecoveryClaimBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -802,7 +810,7 @@ impl Drop for TransitionRecoveryClaimBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] async fn pause_before_transition_recovery_claim(transaction_id: Uuid) { let barrier = TRANSITION_RECOVERY_CLAIM_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -817,6 +825,80 @@ async fn pause_before_transition_recovery_claim(transaction_id: Uuid) { } } +#[cfg(all(test, feature = "test-util"))] +#[derive(Default)] +struct TransitionRecoveryTerminalBarrierState { + transaction_id: Uuid, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) struct TransitionRecoveryTerminalBarrier { + state: Arc, +} + +#[cfg(all(test, feature = "test-util"))] +static TRANSITION_RECOVERY_TERMINAL_BARRIER: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(all(test, feature = "test-util"))] +impl TransitionRecoveryTerminalBarrier { + pub(crate) fn install(transaction_id: Uuid) -> Self { + let state = Arc::new(TransitionRecoveryTerminalBarrierState { + transaction_id, + ..Default::default() + }); + let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("transition recovery terminal barrier mutex should not poison"); + assert!( + slot.is_none(), + "transition recovery terminal barrier must be installed by one test at a time" + ); + *slot = Some(Arc::clone(&state)); + drop(slot); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("transition recovery should persist terminal control before source cleanup"); + } +} + +#[cfg(all(test, feature = "test-util"))] +impl Drop for TransitionRecoveryTerminalBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("transition recovery terminal barrier mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(all(test, feature = "test-util"))] +async fn pause_after_transition_recovery_terminal(transaction_id: Uuid) { + let barrier = TRANSITION_RECOVERY_TERMINAL_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("transition recovery terminal barrier mutex should not poison") + .as_ref() + .filter(|barrier| barrier.transaction_id == transaction_id) + .cloned(); + if let Some(barrier) = barrier { + barrier.arrived.notify_one(); + barrier.release.notified().await; + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] pub enum TransitionOperatorProbe { @@ -1020,17 +1102,35 @@ fn transition_transaction_id_from_record_object_name(object: &str) -> Result EcstoreResult { let record_name = transition_transaction_record_object_name(observed.transaction_id).map_err(transition_transaction_store_error)?; + let now_unix_nanos = + i64::try_from(now_unix_nanos).map_err(|_| Error::other("transition transaction recovery timestamp does not fit i64"))?; + let recovery_control_identity = transition_recovery_control_identity(observed, &record_name); + let recovery_control_id = recovery_control_identity + .source_operation_digest() + .map_err(|err| Error::other(err.to_string()))?; + let control_record_name = + recovery_control_record_object_name(IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id) + .map_err(|err| Error::other(err.to_string()))?; + let control_lock = if transition_state_needs_recovery_control(observed, now_unix_nanos) { + Some( + api.new_ns_lock(RUSTFS_META_BUCKET, &format!("{control_record_name}.recovery-lock")) + .await?, + ) + } else { + None + }; + let _control_guard = match &control_lock { + Some(lock) => Some(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?), + None => None, + }; // The synthetic key avoids nesting the recovery lock with the config // object's own I/O lock. Holding it across the bounded source proof and // remote DELETE elects one destructive recovery worker across nodes. @@ -1073,55 +1194,400 @@ async fn process_transition_transaction_record_at( return Ok(TransitionTransactionRecoveryOutcome::Retained); } - match current.state { + let mut recovery_control = if transition_state_needs_recovery_control(¤t, now_unix_nanos) { + if cleanup_terminal_transition_recovery_control( + api.clone(), + ¤t, + &record_name, + &recovery_control_identity, + &recovery_control_id, + ) + .await? + { + return Ok(TransitionTransactionRecoveryOutcome::RecordDeleted); + } + match claim_transition_recovery_control( + api.clone(), + ¤t, + &record_name, + recovery_control_identity, + &recovery_control_id, + now_unix_nanos, + ) + .await? + { + Some(control) => Some(control), + None => return Ok(TransitionTransactionRecoveryOutcome::Retained), + } + } else { + None + }; + + let recovery = match current.state { TransitionTransactionState::Uploaded => { - if transition_transaction_ownership_is_active(¤t, now_unix_nanos) { - return Ok(TransitionTransactionRecoveryOutcome::Retained); - } - let mut cleanup = current.clone(); - cleanup - .mark_cleanup_pending( - current.fence(), - TransitionCleanupProof { - transaction_id: current.transaction_id, - write_id: current.write_id, - remote_object: current.remote_object.clone(), - remote_version: current.remote_version.clone(), - backend_fingerprint: current.backend_fingerprint, - decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit, - }, - ) - .map_err(transition_transaction_store_error)?; - #[cfg(test)] - pause_before_transition_recovery_claim(current.transaction_id).await; - match save_transition_transaction_record_if_current(api.clone(), ¤t, &cleanup).await { - Ok(()) => recover_cleanup_pending(api, &cleanup).await, - Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => Ok(TransitionTransactionRecoveryOutcome::Retained), - Err(err) => Err(err), + if transition_transaction_ownership_is_active(¤t, i128::from(now_unix_nanos)) { + Ok(TransitionTransactionRecoveryOutcome::Retained) + } else { + let mut cleanup = current.clone(); + cleanup + .mark_cleanup_pending( + current.fence(), + TransitionCleanupProof { + transaction_id: current.transaction_id, + write_id: current.write_id, + remote_object: current.remote_object.clone(), + remote_version: current.remote_version.clone(), + backend_fingerprint: current.backend_fingerprint, + decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit, + }, + ) + .map_err(transition_transaction_store_error)?; + #[cfg(all(test, feature = "test-util"))] + pause_before_transition_recovery_claim(current.transaction_id).await; + match save_transition_transaction_record_if_current(api.clone(), ¤t, &cleanup).await { + Ok(()) => recover_cleanup_pending(api.clone(), &cleanup).await, + Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => { + Ok(TransitionTransactionRecoveryOutcome::Retained) + } + Err(err) => Err(err), + } } } - TransitionTransactionState::CleanupPending => recover_cleanup_pending(api, ¤t).await, + TransitionTransactionState::CleanupPending => recover_cleanup_pending(api.clone(), ¤t).await, TransitionTransactionState::LocalCommitStarted => match local_commit_matches_transaction(api.clone(), ¤t).await { - Ok(true) => { - delete_transition_transaction_record(api, ¤t).await?; - Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) - } - Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained), - Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::Retained), + Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted), + Ok(false) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired( + IlmRecoveryErrorCode::LocalCommitAmbiguous, + )), + Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired( + IlmRecoveryErrorCode::LocalCommitAmbiguous, + )), Err(err) => Err(err), }, TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => { - delete_transition_transaction_record(api, ¤t).await?; Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) } TransitionTransactionState::UploadOutcomeUnknown => { - if transition_transaction_ownership_is_active(¤t, now_unix_nanos) { + if transition_transaction_ownership_is_active(¤t, i128::from(now_unix_nanos)) { Ok(TransitionTransactionRecoveryOutcome::Retained) } else { - recover_unknown_upload_outcome(api, ¤t).await + recover_unknown_upload_outcome(api.clone(), ¤t).await } } - TransitionTransactionState::UploadStarted => Ok(TransitionTransactionRecoveryOutcome::Retained), + TransitionTransactionState::UploadStarted => { + if transition_transaction_ownership_is_active(¤t, i128::from(now_unix_nanos)) { + Ok(TransitionTransactionRecoveryOutcome::Retained) + } else { + Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous( + IlmRecoveryErrorCode::RemoteVersionUnknown, + )) + } + } + }; + + if let Some(mut control) = recovery_control.take() { + let source_to_delete = if matches!( + recovery, + Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted + | TransitionTransactionRecoveryOutcome::RecordDeleted) + ) { + let refreshed = + refresh_transition_recovery_control_source(api.clone(), control, &record_name, current.transaction_id).await?; + control = refreshed.0; + refreshed.1 + } else { + None + }; + persist_transition_recovery_result(api.clone(), control, &recovery, now_unix_nanos).await?; + if let Some(source) = source_to_delete { + #[cfg(all(test, feature = "test-util"))] + pause_after_transition_recovery_terminal(source.transaction_id).await; + delete_transition_transaction_record(api, &source).await?; + } + } else if matches!( + recovery, + Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted) + ) { + delete_transition_transaction_record(api, ¤t).await?; + } + recovery +} + +fn transition_recovery_control_identity(transaction: &TransitionTransaction, record_name: &str) -> IlmRecoveryControlIdentity { + IlmRecoveryControlIdentity { + protocol: IlmRecoveryProtocol::TransitionTransaction, + canonical_source_path: record_name.to_string(), + stable_operation_identity: transaction.transaction_id.to_string(), + record_class: "transition_transaction_v1".to_string(), + } +} + +#[cfg(all(test, feature = "test-util"))] +pub(crate) fn transition_recovery_control_id(transaction: &TransitionTransaction) -> Result { + let record_name = transition_transaction_record_object_name(transaction.transaction_id)?; + transition_recovery_control_identity(transaction, &record_name) + .source_operation_digest() + .map_err(|_| TransitionTransactionError::Corrupt("transition recovery control identity is invalid")) +} + +fn transition_state_needs_recovery_control(transaction: &TransitionTransaction, now_unix_nanos: i64) -> bool { + now_unix_nanos >= transaction.not_after_unix_nanos + && !matches!( + transaction.state, + TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed + ) +} + +async fn cleanup_terminal_transition_recovery_control( + api: Arc, + transaction: &TransitionTransaction, + record_name: &str, + identity: &IlmRecoveryControlIdentity, + control_id: &str, +) -> EcstoreResult { + let observed = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await { + Ok(observed) => observed, + Err(Error::ConfigNotFound) => return Ok(false), + Err(err) => return Err(err), + }; + if observed.control.classification != IlmRecoveryClassification::Terminal { + return Ok(false); + } + let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?; + let exact_source = source.is_consistent() + && source.generation == observed.control.observed_source_generation + && source.canonical_data.as_deref().is_some_and(|data| { + TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|decoded| decoded == *transaction) + }); + if observed.control.identity != *identity || !exact_source { + return Ok(false); + } + delete_transition_transaction_record(api, transaction).await?; + Ok(true) +} + +async fn claim_transition_recovery_control( + api: Arc, + transaction: &TransitionTransaction, + record_name: &str, + identity: IlmRecoveryControlIdentity, + control_id: &str, + now_unix_nanos: i64, +) -> EcstoreResult> { + let existing = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await { + Ok(control) => Some(control), + Err(Error::ConfigNotFound) => None, + Err(err) => return Err(err), + }; + if let Some(observed) = existing.as_ref() { + if observed.control.identity != identity { + return Ok(None); + } + if observed + .control + .owner + .as_ref() + .is_some_and(|owner| owner.lease_expires_at_unix_nanos <= now_unix_nanos) + { + let mut expired = observed.control.clone(); + expired + .record_expired_attempt(now_unix_nanos) + .map_err(|err| Error::other(err.to_string()))?; + save_recovery_control_if_current(api, observed, &expired).await?; + return Ok(None); + } + if !observed.control.should_attempt_at(now_unix_nanos) { + return Ok(None); + } + } + + let source = match observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await { + Ok(source) => source, + Err(err) => { + if let Some(observed) = existing { + persist_transition_recovery_source_failure(api, observed, now_unix_nanos).await?; + return Ok(None); + } + return Err(err); + } + }; + let source_matches = source.is_consistent() + && source.canonical_data.as_deref().is_some_and(|data| { + TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|observed| observed == *transaction) + }); + let source_error = if source_matches { + IlmRecoveryErrorCode::None + } else if source.canonical_data.is_some() { + IlmRecoveryErrorCode::SourceGenerationChanged + } else { + IlmRecoveryErrorCode::SourceDivergent + }; + + let mut observed = match existing { + Some(control) => control, + None => { + let candidate = IlmRecoveryControl::new( + identity.clone(), + source.generation.clone(), + if source_matches { + IlmRecoveryClassification::Retrying + } else { + IlmRecoveryClassification::Corrupt + }, + now_unix_nanos, + source_error, + ) + .map_err(|err| Error::other(err.to_string()))?; + match save_recovery_control_if_absent(api.clone(), &candidate).await { + Ok(()) | Err(Error::PreconditionFailed) => {} + Err(err) => return Err(err), + } + load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await? + } + }; + if observed.control.identity != identity || !observed.control.should_attempt_at(now_unix_nanos) { + return Ok(None); + } + + let mut claimed = observed.control.clone(); + claimed + .claim_for_source_generation( + api.id.to_string(), + Uuid::new_v4(), + now_unix_nanos, + TRANSITION_RECOVERY_CONTROL_LEASE_NANOS, + source.generation, + ) + .map_err(|err| Error::other(err.to_string()))?; + save_recovery_control_if_current(api.clone(), &observed, &claimed).await?; + observed = load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?; + if observed.control != claimed { + return Err(Error::PreconditionFailed); + } + if !source_matches { + let mut corrupt = observed.control.clone(); + corrupt + .finish_attempt(IlmRecoveryClassification::Corrupt, source_error) + .map_err(|err| Error::other(err.to_string()))?; + save_recovery_control_if_current(api, &observed, &corrupt).await?; + return Ok(None); + } + Ok(Some(observed)) +} + +async fn persist_transition_recovery_source_failure( + api: Arc, + observed: ObservedIlmRecoveryControl, + now_unix_nanos: i64, +) -> EcstoreResult<()> { + let mut claimed = observed.control.clone(); + claimed + .claim( + api.id.to_string(), + Uuid::new_v4(), + now_unix_nanos, + TRANSITION_RECOVERY_CONTROL_LEASE_NANOS, + ) + .map_err(|err| Error::other(err.to_string()))?; + save_recovery_control_if_current(api.clone(), &observed, &claimed).await?; + let claimed = load_recovery_control( + api.clone(), + IlmRecoveryProtocol::TransitionTransaction, + &claimed + .identity + .source_operation_digest() + .map_err(|err| Error::other(err.to_string()))?, + ) + .await?; + let mut failed = claimed.control.clone(); + failed + .record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceUnavailable) + .map_err(|err| Error::other(err.to_string()))?; + save_recovery_control_if_current(api, &claimed, &failed).await +} + +async fn refresh_transition_recovery_control_source( + api: Arc, + mut observed: ObservedIlmRecoveryControl, + record_name: &str, + transaction_id: Uuid, +) -> EcstoreResult<(ObservedIlmRecoveryControl, Option)> { + let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await { + Ok(transaction) => transaction, + Err(Error::ConfigNotFound) => return Ok((observed, None)), + Err(err) => return Err(err), + }; + let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?; + let exact_source = source.is_consistent() + && source + .canonical_data + .as_deref() + .is_some_and(|data| TransitionTransaction::decode(transaction_id, data).is_ok_and(|decoded| decoded == transaction)); + if !exact_source { + return Err(Error::PreconditionFailed); + } + if observed.control.observed_source_generation != source.generation { + let mut refreshed = observed.control.clone(); + refreshed + .refresh_owned_source_generation(source.generation) + .map_err(|err| Error::other(err.to_string()))?; + save_recovery_control_if_current(api.clone(), &observed, &refreshed).await?; + observed = load_recovery_control( + api, + IlmRecoveryProtocol::TransitionTransaction, + &refreshed + .identity + .source_operation_digest() + .map_err(|err| Error::other(err.to_string()))?, + ) + .await?; + if observed.control != refreshed { + return Err(Error::PreconditionFailed); + } + } + Ok((observed, Some(transaction))) +} + +async fn persist_transition_recovery_result( + api: Arc, + observed: ObservedIlmRecoveryControl, + recovery: &EcstoreResult, + now_unix_nanos: i64, +) -> EcstoreResult<()> { + let mut next = observed.control.clone(); + match recovery { + Ok( + TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted, + ) => next + .finish_attempt(IlmRecoveryClassification::Terminal, IlmRecoveryErrorCode::None) + .map_err(|err| Error::other(err.to_string()))?, + Ok(TransitionTransactionRecoveryOutcome::Retained) => next + .record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceGenerationChanged) + .map_err(|err| Error::other(err.to_string()))?, + Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(code)) => next + .finish_attempt(IlmRecoveryClassification::RetainedAmbiguous, *code) + .map_err(|err| Error::other(err.to_string()))?, + Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(code)) => next + .finish_attempt(IlmRecoveryClassification::OperatorRequired, *code) + .map_err(|err| Error::other(err.to_string()))?, + Err(err) => next + .record_retryable_failure(now_unix_nanos, transition_recovery_error_code(err)) + .map_err(|err| Error::other(err.to_string()))?, + } + save_recovery_control_if_current(api, &observed, &next).await +} + +fn transition_recovery_error_code(err: &Error) -> IlmRecoveryErrorCode { + match err { + Error::PreconditionFailed => IlmRecoveryErrorCode::CasConflict, + Error::ConfigNotFound + | Error::FileNotFound + | Error::FileVersionNotFound + | Error::ObjectNotFound(_, _) + | Error::VersionNotFound(_, _, _) + | Error::BucketNotFound(_) => IlmRecoveryErrorCode::SourceUnavailable, + Error::SlowDown => IlmRecoveryErrorCode::BackendThrottled, + _ => IlmRecoveryErrorCode::Unknown, } } @@ -1134,10 +1600,7 @@ async fn recover_cleanup_pending( transaction: &TransitionTransaction, ) -> EcstoreResult { match local_commit_matches_transaction(api.clone(), transaction).await { - Ok(true) => { - delete_transition_transaction_record(api, transaction).await?; - Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) - } + Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted), Ok(false) => delete_unreferenced_transition_candidate(api, transaction).await, Err(err) if transition_source_is_missing(&err) => delete_unreferenced_transition_candidate(api, transaction).await, Err(err) => Err(err), @@ -1157,7 +1620,6 @@ async fn delete_unreferenced_transition_candidate( return Ok(TransitionTransactionRecoveryOutcome::Retained); } delete_transition_remote_candidate(api.clone(), ¤t).await?; - delete_transition_transaction_record(api, ¤t).await?; Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted) } @@ -1178,24 +1640,26 @@ async fn recover_unknown_upload_outcome( .await .map_err(Error::other)? { - TransitionCandidateProbe::Missing => { - delete_transition_transaction_record(api, transaction).await?; - Ok(TransitionTransactionRecoveryOutcome::RecordDeleted) - } + TransitionCandidateProbe::Missing => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted), TransitionCandidateProbe::UnversionedPresent => { cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await } TransitionCandidateProbe::VersionedPresent(version_id) if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) => { - Ok(TransitionTransactionRecoveryOutcome::Retained) + Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous( + IlmRecoveryErrorCode::RemoteVersionUnknown, + )) } TransitionCandidateProbe::VersionedPresent(version_id) => { cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await } - TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => { - Ok(TransitionTransactionRecoveryOutcome::Retained) - } + TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous( + IlmRecoveryErrorCode::RemoteProbeAmbiguous, + )), + TransitionCandidateProbe::Unsupported => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous( + IlmRecoveryErrorCode::RemoteProbeUnsupported, + )), } } @@ -1289,7 +1753,7 @@ pub async fn recover_transition_transaction_records( recover_transition_transaction_records_with_now(api, limit, marker, None).await } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] pub async fn recover_transition_transaction_records_at( api: Arc, limit: usize, @@ -1323,6 +1787,11 @@ async fn recover_transition_transaction_records_with_now( false, ) .await?; + if list.is_truncated && list.next_continuation_token.is_none() { + return Err(Error::other( + "transition transaction recovery returned a truncated page without a continuation marker", + )); + } let mut stats = TransitionTransactionRecoveryStats { scanned: 0, @@ -1381,7 +1850,11 @@ async fn recover_transition_transaction_records_with_now( ) => { stats.recovered += 1; } - Ok(TransitionTransactionRecoveryOutcome::Retained) => { + Ok( + TransitionTransactionRecoveryOutcome::Retained + | TransitionTransactionRecoveryOutcome::RetainedAmbiguous(_) + | TransitionTransactionRecoveryOutcome::OperatorRequired(_), + ) => { stats.retained += 1; debug!( event = EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY, @@ -1509,11 +1982,74 @@ fn state_requires_known_remote_version(state: TransitionTransactionState) -> boo #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::atomic::{AtomicBool, Ordering}; use super::*; const BACKEND_FINGERPRINT: [u8; 32] = [7; 32]; + struct RecoveryAttemptDropGuard(Arc); + + impl Drop for RecoveryAttemptDropGuard { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } + } + + async fn pending_recovery_attempt(started: Arc, dropped: Arc) -> EcstoreResult<()> { + let _drop_guard = RecoveryAttemptDropGuard(dropped); + started.notify_one(); + std::future::pending().await + } + + #[tokio::test(start_paused = true)] + async fn transition_recovery_timeout_and_cancellation_drop_inflight_attempts() { + let timeout_started = Arc::new(tokio::sync::Notify::new()); + let timeout_dropped = Arc::new(AtomicBool::new(false)); + let timeout_task = tokio::spawn({ + let started = Arc::clone(&timeout_started); + let dropped = Arc::clone(&timeout_dropped); + async move { + await_transition_transaction_recovery( + &CancellationToken::new(), + TRANSITION_TRANSACTION_RECOVERY_TIMEOUT, + pending_recovery_attempt(started, dropped), + ) + .await + } + }); + timeout_started.notified().await; + tokio::time::advance(TRANSITION_TRANSACTION_RECOVERY_TIMEOUT).await; + let timed_out = timeout_task.await.expect("timeout wrapper task should join"); + assert!(matches!(timed_out, Some(Err(_))), "outer timeout should fail the recovery pass"); + assert!(timeout_dropped.load(Ordering::SeqCst), "outer timeout must drop its in-flight attempt"); + + let cancel_token = CancellationToken::new(); + let cancel_started = Arc::new(tokio::sync::Notify::new()); + let cancel_dropped = Arc::new(AtomicBool::new(false)); + let cancel_task = tokio::spawn({ + let cancel_token = cancel_token.clone(); + let started = Arc::clone(&cancel_started); + let dropped = Arc::clone(&cancel_dropped); + async move { + await_transition_transaction_recovery( + &cancel_token, + TRANSITION_TRANSACTION_RECOVERY_TIMEOUT, + pending_recovery_attempt(started, dropped), + ) + .await + } + }); + cancel_started.notified().await; + cancel_token.cancel(); + let cancelled = cancel_task.await.expect("cancellation wrapper task should join"); + assert!(cancelled.is_none(), "outer cancellation should stop the recovery loop"); + assert!( + cancel_dropped.load(Ordering::SeqCst), + "outer cancellation must drop its in-flight attempt" + ); + } + #[derive(Default)] struct MemoryTransactionStore { records: HashMap>, @@ -1968,5 +2504,19 @@ mod tests { transition_transaction_record_object_name(Uuid::nil()), Err(TransitionTransactionError::Corrupt("transaction_id is nil")) )); + assert_eq!( + transition_transaction_id_from_record_object_name(&object).expect("canonical record path should parse"), + transaction_id + ); + for malformed in [ + object.to_ascii_uppercase(), + object.replace("/aa/aa/", "/ff/aa/"), + object.replace("/aa/aa/", "/aa/aa/extra/"), + ] { + assert!(matches!( + transition_transaction_id_from_record_object_name(&malformed), + Err(TransitionTransactionError::Corrupt(_)) + )); + } } } diff --git a/crates/ecstore/src/bucket/metadata.rs b/crates/ecstore/src/bucket/metadata.rs index 09030ba12..04b97f10f 100644 --- a/crates/ecstore/src/bucket/metadata.rs +++ b/crates/ecstore/src/bucket/metadata.rs @@ -1611,6 +1611,34 @@ mod test { assert!(bm.bucket_target_config.is_none()); } + /// rustfs/backlog#2309: the MinIO-origin `.metadata.bin` this repository + /// already carries as a compatibility fixture stores + /// `BucketTargetsConfigJSON` as a bare JSON array, which `BucketTargets` + /// (a `{"targets":[…]}` struct with no array fallback) cannot decode. The + /// bytes below are the exact payload the fixture in + /// `metadata_test.rs::TEST_BUCKET_METADATA_HEX` decodes to, so if RustFS + /// ever grows the array-shaped compatibility parse, this test is where the + /// upgrade break is pinned and where the decision has to be recorded. + #[test] + fn minio_array_shaped_bucket_targets_are_unreadable() { + let minio_array = br#"[{"endpoint":"http://target.example.com","targetBucket":"tb","region":"us-east-1"}]"#.to_vec(); + let mut bm = BucketMetadata::new("minio-array-targets"); + bm.bucket_targets_config_json = minio_array.clone(); + + bm.parse_all_configs() + .expect("a MinIO-shaped targets blob must not fail the whole metadata load"); + + assert!( + bm.bucket_targets_unreadable(), + "an array-shaped MinIO targets blob is unreadable, not an empty target set" + ); + assert!(bm.bucket_target_config.is_none()); + assert_eq!( + bm.bucket_targets_config_json, minio_array, + "the raw MinIO bytes must survive so the configuration stays recoverable" + ); + } + /// The invariant every branch of `parse_all_configs` shares: a stored but /// undecodable payload keeps its raw bytes and leaves the typed field /// `None`, so no branch fabricates a value. What a reader may then do with diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 94f3a6344..c4235bd40 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -3013,7 +3013,7 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result Result { - return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid operation id"))); + "operation_id" | "control_id" if !is_sha256_checksum(id) => { + let id_label = id_kind.trim_end_matches("_id"); + return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid {id_label} id"))); } "transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => { return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID"))); @@ -19300,8 +19301,8 @@ mod pools_tests { load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done, merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh, missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result, - pool_meta_has_active_decommission, publish_pool_meta_updates, read_pool_meta_replica, - reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion, + parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates, + read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion, record_decommission_unresolved_entry, recover_decommission_capacity_reservations, renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers, reserve_decommission_start_target_capacity, resolve_decommission_bucket_state, @@ -20169,6 +20170,26 @@ mod pools_tests { assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token))); } + #[test] + fn decommission_recovery_control_receipt_path_round_trips() { + let run_token = "b".repeat(64); + let control_id = "a".repeat(64); + let source_path = format!( + "ilm/recovery-controls/transition_transaction/{}/{}/{}.json", + &control_id[..2], + &control_id[2..4], + control_id + ); + let path = decommission_durable_ilm_receipt_path(&run_token, &source_path, "control_id", &control_id); + + let locator = parse_decommission_durable_ilm_receipt_path(&path).expect("recovery control receipt path should parse"); + + assert_eq!(locator.run_token, run_token); + assert_eq!(locator.source_path, source_path); + assert_eq!(locator.id_kind, "control_id"); + assert_eq!(locator.id, control_id); + } + #[test] fn decommission_receipt_merge_preserves_terminal_proof() { let operation_id = "a".repeat(64); diff --git a/crates/ecstore/src/diagnostics/get.rs b/crates/ecstore/src/diagnostics/get.rs index 4032773f5..fecb3c701 100644 --- a/crates/ecstore/src/diagnostics/get.rs +++ b/crates/ecstore/src/diagnostics/get.rs @@ -99,11 +99,17 @@ pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_m pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success"; pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream"; pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open"; +#[cfg(unix)] pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map"; pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup"; pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate"; diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index b98b294ca..a11594997 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -324,6 +324,30 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper { } impl LocalDiskWrapper { + pub(in crate::disk) async fn undo_write_with_namespace_owner( + &self, + volume: &str, + path: &str, + fi: FileInfo, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { + self.track_disk_health_mutation( + "delete_version", + DiskMetricMutation::Delete, + || async { + // Preserve the old DiskAPI future's boxing boundary. + Box::pin( + self.disk + .undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner), + ) + .await + }, + get_max_timeout_duration(), + ) + .await + } + pub(in crate::disk) async fn rename_data_observed( &self, src_volume: &str, @@ -333,6 +357,34 @@ impl LocalDiskWrapper { dst_path: &str, external_guard: Option>, ) -> super::RenameDataObservation { + self.rename_data_observed_with_guards( + src_volume, + src_path, + fi, + dst_volume, + dst_path, + super::RenameDataGuards { + external_guard, + ..Default::default() + }, + ) + .await + } + + pub(in crate::disk) async fn rename_data_observed_with_guards( + &self, + src_volume: &str, + src_path: &str, + fi: &FileInfo, + dst_volume: &str, + dst_path: &str, + guards: super::RenameDataGuards, + ) -> super::RenameDataObservation { + let super::RenameDataGuards { + external_guard, + namespace_owner, + .. + } = guards; let operation = self.clone(); let src_volume = src_volume.to_owned(); let src_path = src_path.to_owned(); @@ -357,13 +409,15 @@ impl LocalDiskWrapper { DiskMetricMutation::Write, || async { // Preserve the former DiskAPI future's single boxing boundary. - let observed = - Box::pin( - operation - .disk - .rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path), - ) - .await; + let observed = Box::pin(operation.disk.rename_data_observed( + &src_volume, + &src_path, + &fi, + &dst_volume, + &dst_path, + namespace_owner, + )) + .await; preflight_rejection = observed.preflight_rejection; observed.result }, @@ -1301,6 +1355,7 @@ impl LocalDiskWrapper { self.disk.get_object_path(volume, path) } + #[cfg(unix)] pub(crate) fn get_object_path_for_io(&self, volume: &str, path: &str) -> crate::disk::error::Result { self.disk.get_object_path_for_io(volume, path) } diff --git a/crates/ecstore/src/disk/fs.rs b/crates/ecstore/src/disk/fs.rs index 8612b6474..7dbbaf14d 100644 --- a/crates/ecstore/src/disk/fs.rs +++ b/crates/ecstore/src/disk/fs.rs @@ -218,10 +218,12 @@ pub async fn rename(from: impl AsRef, to: impl AsRef) -> io::Result< fs::rename(from, to).await } +#[cfg(any(not(windows), test))] pub fn rename_std(from: impl AsRef, to: impl AsRef) -> io::Result<()> { std::fs::rename(from, to) } +#[cfg(any(not(windows), test))] #[tracing::instrument(level = "debug", skip_all)] pub async fn read_file(path: impl AsRef) -> io::Result> { fs::read(path.as_ref()).await diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 9cd683578..c24c90724 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -204,15 +204,25 @@ async fn restore_metadata_backup( xl_path: &Path, rollback_dir: Uuid, publication_root: &os::PublicationRoot, +) -> Result<()> { + restore_metadata_backup_with_namespace_owner(object_dir, xl_path, rollback_dir, publication_root, None).await +} + +async fn restore_metadata_backup_with_namespace_owner( + object_dir: &Path, + xl_path: &Path, + rollback_dir: Uuid, + publication_root: &os::PublicationRoot, + namespace_owner: Option>, ) -> Result<()> { let rollback_path = object_dir.join(rollback_dir.to_string()); let backup_path = rollback_path.join(STORAGE_FORMAT_FILE_BACKUP); - rename_all(&backup_path, xl_path, object_dir, publication_root).await?; + os::rename_all_with_owner(&backup_path, xl_path, object_dir, publication_root, namespace_owner.clone()).await?; // A synthetic inline rollback dir held only the backup the rename above // just consumed; reclaim it so the object dir can empty out. A real data // dir still holds its parts, so the non-recursive remove is a benign // no-op there (mirrors restore_delete_rollback). - let _ = fs::remove_dir(&rollback_path).await; + let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; Ok(()) } @@ -222,7 +232,17 @@ async fn restore_delete_rollback( rollback_dir: Uuid, publication_root: &os::PublicationRoot, ) -> Result<()> { - remove_version_delete_markers(object_dir, rollback_dir).await?; + restore_delete_rollback_with_namespace_owner(object_dir, xl_path, rollback_dir, publication_root, None).await +} + +async fn restore_delete_rollback_with_namespace_owner( + object_dir: &Path, + xl_path: &Path, + rollback_dir: Uuid, + publication_root: &os::PublicationRoot, + namespace_owner: Option>, +) -> Result<()> { + remove_version_delete_markers(object_dir, rollback_dir, namespace_owner.clone()).await?; let rollback_path = object_dir.join(rollback_dir.to_string()); let mut staged_paths = Vec::new(); let mut remove_new_metadata = false; @@ -243,34 +263,38 @@ async fn restore_delete_rollback( let had_staged_paths = !staged_paths.is_empty(); for (src, dst) in staged_paths { - rename_all(&src, &dst, object_dir, publication_root).await?; + os::rename_all_with_owner(&src, &dst, object_dir, publication_root, namespace_owner.clone()).await?; } let backup_path = rollback_path.join(STORAGE_FORMAT_FILE_BACKUP); - match rename_all(&backup_path, xl_path, object_dir, publication_root).await { + match os::rename_all_with_owner(&backup_path, xl_path, object_dir, publication_root, namespace_owner.clone()).await { Ok(()) => { - let _ = fs::remove_dir(&rollback_path).await; + let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; Ok(()) } // A missing backup only means "remove the newly-created delete marker" // when the marker proves there was no old metadata to restore. - Err(DiskError::FileNotFound) if remove_new_metadata => match fs::remove_file(xl_path).await { - Ok(()) => { - let _ = fs::remove_file(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE)).await; - let _ = fs::remove_dir(&rollback_path).await; - Ok(()) + Err(DiskError::FileNotFound) if remove_new_metadata => { + match os::remove_file_with_owner(xl_path, namespace_owner.clone()).await { + Ok(()) => { + let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()) + .await; + let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; + Ok(()) + } + Err(err) if err.kind() == ErrorKind::NotFound => { + let _ = os::remove_file_with_owner(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), namespace_owner.clone()) + .await; + let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; + Ok(()) + } + Err(err) => Err(to_file_error(err).into()), } - Err(err) if err.kind() == ErrorKind::NotFound => { - let _ = fs::remove_file(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE)).await; - let _ = fs::remove_dir(&rollback_path).await; - Ok(()) - } - Err(err) => Err(to_file_error(err).into()), - }, + } Err(DiskError::FileNotFound) if had_staged_paths => Err(DiskError::FileNotFound), Err(DiskError::FileNotFound) => match fs::metadata(xl_path).await { Ok(_) => { - let _ = fs::remove_dir(&rollback_path).await; + let _ = os::remove_dir_with_owner(&rollback_path, namespace_owner.clone()).await; Ok(()) } Err(err) if err.kind() == ErrorKind::NotFound => Err(DiskError::FileNotFound), @@ -280,7 +304,11 @@ async fn restore_delete_rollback( } } -async fn remove_version_delete_markers(object_dir: &Path, rollback_dir: Uuid) -> Result<()> { +async fn remove_version_delete_markers( + object_dir: &Path, + rollback_dir: Uuid, + namespace_owner: Option>, +) -> Result<()> { let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); let committed_name = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{rollback_dir}"); let mut entries = match fs::read_dir(object_dir).await { @@ -295,7 +323,7 @@ async fn remove_version_delete_markers(object_dir: &Path, rollback_dir: Uuid) -> continue; } for marker_name in [&reserved_name, &committed_name] { - match fs::remove_file(entry.path().join(marker_name)).await { + match os::remove_file_with_owner(entry.path().join(marker_name), namespace_owner.clone()).await { Ok(()) => {} Err(err) if err.kind() == ErrorKind::NotFound => {} Err(err) => return Err(to_file_error(err).into()), @@ -305,6 +333,12 @@ async fn remove_version_delete_markers(object_dir: &Path, rollback_dir: Uuid) -> Ok(()) } +struct DeleteVersionMutation { + force_del_marker: bool, + opts: DeleteOptions, + namespace_owner: Option>, +} + struct DeleteRollbackFailure { stage: &'static str, error: DiskError, @@ -701,7 +735,9 @@ const EVENT_DISK_LOCAL_FORMAT_DECODE_FAILED: &str = "disk_local_format_decode_fa /// to replace. Best effort — the rename that follows fails closed — but a /// recurring signal means heal is stuck on that drive. const EVENT_DISK_LOCAL_HEAL_PURGE_FAILED: &str = "disk_local_heal_purge_failed"; +#[cfg(unix)] const METRIC_GET_OBJECT_MMAP_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_mmap_page_faults_total"; +#[cfg(unix)] const METRIC_GET_OBJECT_DIRECT_READ_PAGE_FAULTS_TOTAL: &str = "rustfs_io_get_object_direct_read_page_faults_total"; // io_uring read-backend gray-release observability (rustfs/backlog#1172). #[cfg(target_os = "linux")] @@ -898,10 +934,15 @@ const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_ reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)" )] const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false; +#[cfg(any(unix, test))] const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE"; +#[cfg(any(unix, test))] const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false; +#[cfg(any(unix, test))] const ENV_RUSTFS_OBJECT_MMAP_READ_METHOD: &str = "RUSTFS_OBJECT_MMAP_READ_METHOD"; +#[cfg(any(unix, test))] const RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY: &str = "mmap_copy"; +#[cfg(any(unix, test))] const RUSTFS_OBJECT_MMAP_READ_METHOD_DIRECT_READ_COPY: &str = "direct_read_copy"; /// Legacy binary switch for commit-point durability (fsync writes and renames). @@ -917,6 +958,7 @@ const DEFAULT_RUSTFS_DRIVE_SYNC_ENABLE: bool = true; /// See docs/operations/durability-modes.md for the power-loss guarantee matrix. const ENV_RUSTFS_DURABILITY_MODE: &str = "RUSTFS_DURABILITY_MODE"; +#[cfg(any(unix, test))] #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum LocalReadCopyMethod { MmapCopy, @@ -1325,15 +1367,18 @@ cached_read_env! { cached_read_env! { /// Whether mmap reads should fault the mapping in with `MAP_POPULATE`. + #[cfg(any(unix, test))] fn mmap_populate_enabled() -> bool = rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE, DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE); } +#[cfg(any(unix, test))] fn should_populate_mmap_read(length: usize) -> bool { length > 0 && mmap_populate_enabled() } cached_read_env! { + #[cfg(any(unix, test))] fn local_read_copy_method() -> LocalReadCopyMethod = { let method = rustfs_utils::get_env_str(ENV_RUSTFS_OBJECT_MMAP_READ_METHOD, RUSTFS_OBJECT_MMAP_READ_METHOD_MMAP_COPY); match method.as_str() { @@ -1976,7 +2021,7 @@ fn set_inline_preparation_before_backup(dst_path: &str, hook: impl FnOnce() + Se .insert(dst_path.to_string(), Box::new(hook)); } -#[cfg(test)] +#[cfg(all(test, unix))] fn set_inline_before_file_sync_admission(dst_path: &str, hook: impl FnOnce() + Send + 'static) { INLINE_BEFORE_FILE_SYNC_ADMISSION .lock() @@ -2223,7 +2268,7 @@ fn should_remove_staged_meta_before_commit(_dst_path: &str) -> bool { false } -#[cfg(not(test))] +#[cfg(all(not(test), not(windows)))] fn should_fail_local_inline_rollback_hardlink(_dst_path: &Path) -> bool { false } @@ -3724,6 +3769,7 @@ struct FdKey { /// The generation fence and explicit mutation invalidation keep the snapshot /// tied to the inode held by `file`, allowing cache hits to avoid a repeated /// metadata syscall without weakening replacement/heal semantics. +#[cfg(unix)] struct FdCacheEntry { /// An independently cloneable descriptor for the immutable shard inode. file: Arc, @@ -5474,6 +5520,7 @@ impl LocalDisk { local_disk_bucket_path(&self.root, bucket) } + #[cfg(any(unix, test))] pub(crate) fn get_object_path_for_io(&self, bucket: &str, key: &str) -> Result { self.io_get_object_path(bucket, key) } @@ -5607,7 +5654,377 @@ impl LocalDisk { // }) // } + async fn delete_version_inner(&self, volume: &str, path: &str, fi: FileInfo, mutation: DeleteVersionMutation) -> Result<()> { + let DeleteVersionMutation { + force_del_marker, + opts, + namespace_owner, + } = mutation; + if path.starts_with(SLASH_SEPARATOR) { + return self + .delete_with_namespace_owner( + volume, + path, + DeleteOptions { + recursive: false, + immediate: false, + ..Default::default() + }, + namespace_owner, + ) + .await; + } + + let volume_dir = self.io_get_bucket_path(volume)?; + + let file_path = self.io_get_object_path(volume, path)?; + + check_path_length(file_path.to_string_lossy().as_ref())?; + + let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]); + if opts.old_data_dir.is_some() && opts.undo_write { + return self.undo_write(file_path.as_path(), &fi, &opts, namespace_owner).await; + } + + let rollback_dir = opts.old_data_dir; + let buf = match self.read_all_data(volume, &volume_dir, &xl_path).await { + Ok(res) => res, + Err(err) => { + if err != DiskError::FileNotFound { + return Err(err); + } + + if fi.deleted && force_del_marker { + return self + .write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir) + .await; + } + + return if fi.version_id.is_some() { + Err(DiskError::FileVersionNotFound) + } else { + Err(DiskError::FileNotFound) + }; + } + }; + + let mut meta = FileMeta::load(&buf)?; + let old_dir = meta.delete_version(&fi)?; + let mut reserved_version_delete = false; + if let Some(rollback_dir) = rollback_dir { + write_metadata_rollback_backup(file_path.as_path(), rollback_dir, &buf).await?; + } + + if let Some(uuid) = old_dir { + let vid = fi.version_id.unwrap_or_default(); + if let Err(err) = meta.data.remove(vec![vid, uuid]) { + let err: DiskError = err.into(); + return Err(restore_delete_rollback_after_error( + file_path.as_path(), + &xl_path, + rollback_dir, + volume, + path, + DeleteRollbackFailure { + stage: "delete_version_metadata_update", + error: err, + }, + &self.publication_root, + ) + .await); + } + + let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]); + if let Err(err) = check_path_length(old_path.to_string_lossy().as_ref()) { + return Err(restore_delete_rollback_after_error( + file_path.as_path(), + &xl_path, + rollback_dir, + volume, + path, + DeleteRollbackFailure { + stage: "delete_version_data_path", + error: err, + }, + &self.publication_root, + ) + .await); + } + + if let Some(rollback_dir) = rollback_dir { + let rollback_path = file_path.join(rollback_dir.to_string()); + if let Err(err) = fs::create_dir_all(&rollback_path).await { + let err: DiskError = to_file_error(err).into(); + return Err(restore_delete_rollback_after_error( + file_path.as_path(), + &xl_path, + Some(rollback_dir), + volume, + path, + DeleteRollbackFailure { + stage: "delete_version_rollback_dir", + error: err, + }, + &self.publication_root, + ) + .await); + } + reserved_version_delete = match self.reserve_version_delete(volume, path, uuid, rollback_dir).await { + Ok(reserved) => reserved, + Err(err) => { + return Err(restore_delete_rollback_after_error( + file_path.as_path(), + &xl_path, + Some(rollback_dir), + volume, + path, + DeleteRollbackFailure { + stage: "delete_version_reserve_data", + error: err, + }, + &self.publication_root, + ) + .await); + } + }; + let rollback_data_path = rollback_path.join(uuid.to_string()); + if !reserved_version_delete + && let Err(err) = + rename_all_ignore_missing_source(&old_path, &rollback_data_path, &rollback_path, &self.publication_root) + .await + { + return Err(restore_delete_rollback_after_error( + file_path.as_path(), + &xl_path, + Some(rollback_dir), + volume, + path, + DeleteRollbackFailure { + stage: "delete_version_stage_data", + error: err, + }, + &self.publication_root, + ) + .await); + } + if should_fail_after_delete_data_staged(path) { + if reserved_version_delete { + return Err(self + .abort_reserved_version_delete( + file_path.as_path(), + rollback_dir, + volume, + path, + "delete_version_test_after_stage", + DiskError::Unexpected, + ) + .await); + } + return Err(restore_delete_rollback_after_error( + file_path.as_path(), + &xl_path, + Some(rollback_dir), + volume, + path, + DeleteRollbackFailure { + stage: "delete_version_test_after_stage", + error: DiskError::Unexpected, + }, + &self.publication_root, + ) + .await); + } + } else if let Err(err) = self + .move_to_trash_with_namespace_owner(&old_path, true, false, namespace_owner.clone()) + .await + && err != DiskError::FileNotFound + && err != DiskError::VolumeNotFound + { + return Err(err); + } + + // The version's data dir was staged for rollback or trashed, so its + // `part.N` inodes no longer exist for readers. A cached io_uring + // descriptor would keep serving them, so drop every cached fd under + // this data dir (rustfs/backlog#1175). If a later rollback restores + // the dir, the next read simply re-opens it. + self.io_backend.invalidate_cached_fds_under(volume, &format!("{path}/{uuid}")); + } + + let commit_result = if !meta.versions.is_empty() { + let buf = match meta.marshal_msg() { + Ok(buf) => buf, + Err(err) => { + let err: DiskError = err.into(); + if reserved_version_delete && let Some(rollback_dir) = rollback_dir { + return Err(self + .abort_reserved_version_delete( + file_path.as_path(), + rollback_dir, + volume, + path, + "delete_version_metadata_encode", + err, + ) + .await); + } + return Err(restore_delete_rollback_after_error( + file_path.as_path(), + &xl_path, + rollback_dir, + volume, + path, + DeleteRollbackFailure { + stage: "delete_version_metadata_encode", + error: err, + }, + &self.publication_root, + ) + .await); + } + }; + self.write_all_meta_with_namespace_owner( + volume, + format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str(), + &buf, + true, + namespace_owner.clone(), + ) + .await + } else { + self.delete_file_with_namespace_owner(&volume_dir, &xl_path, true, false, namespace_owner.clone()) + .await + }; + + if let Err(err) = commit_result { + if reserved_version_delete && let Some(rollback_dir) = rollback_dir { + return Err(self + .abort_reserved_version_delete(file_path.as_path(), rollback_dir, volume, path, "delete_version_commit", err) + .await); + } + return Err(restore_delete_rollback_after_error( + file_path.as_path(), + &xl_path, + rollback_dir, + volume, + path, + DeleteRollbackFailure { + stage: "delete_version_commit", + error: err, + }, + &self.publication_root, + ) + .await); + } + + if reserved_version_delete + && let Some(rollback_dir) = rollback_dir + && let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await + { + return Err(self + .abort_reserved_version_delete( + file_path.as_path(), + rollback_dir, + volume, + path, + "delete_version_commit_intent", + err, + ) + .await); + } + + if should_fail_after_delete_commit(self.root.as_path(), path) { + return Err(DiskError::Unexpected); + } + + Ok(()) + } + + #[tracing::instrument(name = "delete_version", level = "trace", skip_all)] + pub(in crate::disk) async fn undo_write_with_namespace_owner( + &self, + volume: &str, + path: &str, + fi: FileInfo, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { + // This entry is reserved for rollback, not general version deletion. + if !opts.undo_write { + return Err(DiskError::FileCorrupt); + } + self.delete_version_inner( + volume, + path, + fi, + DeleteVersionMutation { + force_del_marker: false, + opts, + namespace_owner, + }, + ) + .await + } + + async fn undo_write( + &self, + file_path: &Path, + fi: &FileInfo, + opts: &DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { + let old_data_dir = opts.old_data_dir.ok_or(DiskError::FileCorrupt)?; + let xl_path = path_join(&[file_path, Path::new(STORAGE_FORMAT_FILE)]); + if opts.undo_delete { + restore_delete_rollback_with_namespace_owner( + file_path, + &xl_path, + old_data_dir, + &self.publication_root, + namespace_owner.clone(), + ) + .await?; + } else { + restore_metadata_backup_with_namespace_owner( + file_path, + &xl_path, + old_data_dir, + &self.publication_root, + namespace_owner.clone(), + ) + .await?; + } + + if !opts.undo_delete + && let Some(new_data_dir) = fi.data_dir + { + let new_data_path = path_join(&[file_path, Path::new(new_data_dir.to_string().as_str())]); + check_path_length(new_data_path.to_string_lossy().as_ref())?; + if let Err(err) = self + .move_to_trash_with_namespace_owner(&new_data_path, true, false, namespace_owner) + .await + && err != DiskError::FileNotFound + && err != DiskError::VolumeNotFound + { + return Err(err); + } + } + + Ok(()) + } + async fn move_to_trash(&self, delete_path: &PathBuf, recursive: bool, immediate_purge: bool) -> Result<()> { + self.move_to_trash_with_namespace_owner(delete_path, recursive, immediate_purge, None) + .await + } + + async fn move_to_trash_with_namespace_owner( + &self, + delete_path: &PathBuf, + recursive: bool, + immediate_purge: bool, + namespace_owner: Option>, + ) -> Result<()> { // if recursive { // remove_all_std(delete_path).map_err(to_volume_error)?; // } else { @@ -5626,11 +6043,12 @@ impl LocalDisk { // } let err = if recursive { - rename_all_ignore_missing_source( + os::rename_all_ignore_missing_source_with_owner( delete_path, trash_path, self.io_get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?, &self.publication_root, + namespace_owner.clone(), ) .await .err() @@ -5648,11 +6066,12 @@ impl LocalDisk { if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR) { let trash_path2 = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; - let _ = rename_all_ignore_missing_source( + let _ = os::rename_all_ignore_missing_source_with_owner( encode_dir_object(delete_path.to_string_lossy().as_ref()), trash_path2, self.io_get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?, &self.publication_root, + namespace_owner.clone(), ) .await; } @@ -5678,7 +6097,44 @@ impl LocalDisk { Ok(()) } + #[tracing::instrument(name = "delete", level = "trace", skip_all)] + async fn delete_with_namespace_owner( + &self, + volume: &str, + path: &str, + opt: DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { + crate::hp_guard!("LocalDisk::delete"); + let handled_version_delete = if opt.recursive + && opt.immediate + && let Some((object, transaction_id)) = path.rsplit_once('/') + && let Ok(transaction_id) = Uuid::parse_str(transaction_id) + { + self.finish_version_delete(volume, object, transaction_id).await? + } else { + false + }; + match self + .delete_unleased_with_namespace_owner(volume, path, &opt, namespace_owner) + .await + { + Err(DiskError::FileNotFound) if handled_version_delete => Ok(()), + result => result, + } + } + async fn delete_unleased(&self, volume: &str, path: &str, opt: &DeleteOptions) -> Result<()> { + self.delete_unleased_with_namespace_owner(volume, path, opt, None).await + } + + async fn delete_unleased_with_namespace_owner( + &self, + volume: &str, + path: &str, + opt: &DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { let volume_dir = self.io_get_bucket_path(volume)?; if !skip_access_checks(volume) && let Err(e) = cached_access(&volume_dir).await @@ -5688,21 +6144,33 @@ impl LocalDisk { let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; - self.delete_file(&volume_dir, &file_path, opt.recursive, opt.immediate) + self.delete_file_with_namespace_owner(&volume_dir, &file_path, opt.recursive, opt.immediate, namespace_owner) .await?; // A deleted shard must not remain readable through the io_uring fd cache. self.io_backend.invalidate_cached_fds_under(volume, path); Ok(()) } - #[tracing::instrument(level = "trace", skip_all)] - #[async_recursion::async_recursion] async fn delete_file( &self, base_path: &PathBuf, delete_path: &PathBuf, recursive: bool, immediate_purge: bool, + ) -> Result<()> { + self.delete_file_with_namespace_owner(base_path, delete_path, recursive, immediate_purge, None) + .await + } + + #[tracing::instrument(name = "delete_file", level = "trace", skip_all)] + #[async_recursion::async_recursion] + async fn delete_file_with_namespace_owner( + &self, + base_path: &PathBuf, + delete_path: &PathBuf, + recursive: bool, + immediate_purge: bool, + namespace_owner: Option>, ) -> Result<()> { // debug!("delete_file {:?}\n base_path:{:?}", &delete_path, &base_path); @@ -5717,10 +6185,11 @@ impl LocalDisk { } if recursive { - self.move_to_trash(delete_path, recursive, immediate_purge).await?; + self.move_to_trash_with_namespace_owner(delete_path, recursive, immediate_purge, namespace_owner.clone()) + .await?; } else if delete_path.is_dir() { // debug!("delete_file remove_dir {:?}", &delete_path); - if let Err(err) = fs::remove_dir(&delete_path).await { + if let Err(err) = os::remove_dir_with_owner(delete_path, namespace_owner.clone()).await { // debug!("remove_dir err {:?} when {:?}", &err, &delete_path); // A missing or still-populated directory is benign here; see // is_benign_object_rmdir_error (handles the illumos/Solaris EEXIST @@ -5742,7 +6211,7 @@ impl LocalDisk { } } // debug!("delete_file remove_dir done {:?}", &delete_path); - } else if let Err(err) = fs::remove_file(&delete_path).await { + } else if let Err(err) = os::remove_file_with_owner(delete_path, namespace_owner.clone()).await { // debug!("remove_file err {:?} when {:?}", &err, &delete_path); match err.kind() { ErrorKind::NotFound => (), @@ -5765,7 +6234,14 @@ impl LocalDisk { } if let Some(dir_path) = delete_path.parent() { - Box::pin(self.delete_file(base_path, &PathBuf::from(dir_path), false, false)).await?; + Box::pin(self.delete_file_with_namespace_owner( + base_path, + &PathBuf::from(dir_path), + false, + false, + namespace_owner.clone(), + )) + .await?; } // debug!("delete_file done {:?}", &delete_path); @@ -6352,6 +6828,17 @@ impl LocalDisk { } async fn write_all_meta(&self, volume: &str, path: &str, buf: &[u8], sync: bool) -> Result<()> { + self.write_all_meta_with_namespace_owner(volume, path, buf, sync, None).await + } + + async fn write_all_meta_with_namespace_owner( + &self, + volume: &str, + path: &str, + buf: &[u8], + sync: bool, + namespace_owner: Option>, + ) -> Result<()> { let volume_dir = self.io_get_bucket_path(volume)?; let file_path = self.io_get_object_path(volume, path)?; check_path_length(file_path.to_string_lossy().as_ref())?; @@ -6384,13 +6871,15 @@ impl LocalDisk { return Err(DiskError::Unexpected); } - rename_all(tmp_file_path, &file_path, volume_dir, &self.publication_root).await?; + os::rename_all_with_owner(tmp_file_path, &file_path, volume_dir, &self.publication_root, namespace_owner.clone()).await?; if sync && durability.syncs_commit_metadata() && let Some(parent) = file_path.parent() { - os::fsync_dir(parent).await.map_err(to_file_error)?; + os::fsync_dir_with_owner(parent, namespace_owner) + .await + .map_err(to_file_error)?; } Ok(()) @@ -8002,22 +8491,8 @@ impl DiskAPI for LocalDisk { LocalDisk::has_replacement_mount_lease(self) } - #[tracing::instrument(level = "trace", skip_all)] async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - crate::hp_guard!("LocalDisk::delete"); - let handled_version_delete = if opt.recursive - && opt.immediate - && let Some((object, transaction_id)) = path.rsplit_once('/') - && let Ok(transaction_id) = Uuid::parse_str(transaction_id) - { - self.finish_version_delete(volume, object, transaction_id).await? - } else { - false - }; - match self.delete_unleased(volume, path, &opt).await { - Err(DiskError::FileNotFound) if handled_version_delete => Ok(()), - result => result, - } + self.delete_with_namespace_owner(volume, path, opt, None).await } #[tracing::instrument(level = "trace", skip_all)] @@ -8875,7 +9350,7 @@ impl DiskAPI for LocalDisk { dst_volume: &str, dst_path: &str, ) -> Result { - self.rename_data_inner(src_volume, src_path, fi, dst_volume, dst_path, &mut None) + self.rename_data_inner(src_volume, src_path, fi, dst_volume, dst_path, &mut Default::default()) .await } @@ -9392,295 +9867,17 @@ impl DiskAPI for LocalDisk { force_del_marker: bool, opts: DeleteOptions, ) -> Result<()> { - if path.starts_with(SLASH_SEPARATOR) { - return self - .delete( - volume, - path, - DeleteOptions { - recursive: false, - immediate: false, - ..Default::default() - }, - ) - .await; - } - - let volume_dir = self.io_get_bucket_path(volume)?; - - let file_path = self.io_get_object_path(volume, path)?; - - check_path_length(file_path.to_string_lossy().as_ref())?; - - let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]); - if let Some(old_data_dir) = opts.old_data_dir - && opts.undo_write - { - if opts.undo_delete { - restore_delete_rollback(file_path.as_path(), &xl_path, old_data_dir, &self.publication_root).await?; - } else { - restore_metadata_backup(file_path.as_path(), &xl_path, old_data_dir, &self.publication_root).await?; - } - - if !opts.undo_delete - && let Some(new_data_dir) = fi.data_dir - { - let new_data_path = path_join(&[file_path.as_path(), Path::new(new_data_dir.to_string().as_str())]); - check_path_length(new_data_path.to_string_lossy().as_ref())?; - if let Err(err) = self.move_to_trash(&new_data_path, true, false).await - && err != DiskError::FileNotFound - && err != DiskError::VolumeNotFound - { - return Err(err); - } - } - - return Ok(()); - } - - let rollback_dir = opts.old_data_dir; - let buf = match self.read_all_data(volume, &volume_dir, &xl_path).await { - Ok(res) => res, - Err(err) => { - if err != DiskError::FileNotFound { - return Err(err); - } - - if fi.deleted && force_del_marker { - return self - .write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir) - .await; - } - - return if fi.version_id.is_some() { - Err(DiskError::FileVersionNotFound) - } else { - Err(DiskError::FileNotFound) - }; - } - }; - - let mut meta = FileMeta::load(&buf)?; - let old_dir = meta.delete_version(&fi)?; - let mut reserved_version_delete = false; - if let Some(rollback_dir) = rollback_dir { - write_metadata_rollback_backup(file_path.as_path(), rollback_dir, &buf).await?; - } - - if let Some(uuid) = old_dir { - let vid = fi.version_id.unwrap_or_default(); - if let Err(err) = meta.data.remove(vec![vid, uuid]) { - let err: DiskError = err.into(); - return Err(restore_delete_rollback_after_error( - file_path.as_path(), - &xl_path, - rollback_dir, - volume, - path, - DeleteRollbackFailure { - stage: "delete_version_metadata_update", - error: err, - }, - &self.publication_root, - ) - .await); - } - - let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]); - if let Err(err) = check_path_length(old_path.to_string_lossy().as_ref()) { - return Err(restore_delete_rollback_after_error( - file_path.as_path(), - &xl_path, - rollback_dir, - volume, - path, - DeleteRollbackFailure { - stage: "delete_version_data_path", - error: err, - }, - &self.publication_root, - ) - .await); - } - - if let Some(rollback_dir) = rollback_dir { - let rollback_path = file_path.join(rollback_dir.to_string()); - if let Err(err) = fs::create_dir_all(&rollback_path).await { - let err: DiskError = to_file_error(err).into(); - return Err(restore_delete_rollback_after_error( - file_path.as_path(), - &xl_path, - Some(rollback_dir), - volume, - path, - DeleteRollbackFailure { - stage: "delete_version_rollback_dir", - error: err, - }, - &self.publication_root, - ) - .await); - } - reserved_version_delete = match self.reserve_version_delete(volume, path, uuid, rollback_dir).await { - Ok(reserved) => reserved, - Err(err) => { - return Err(restore_delete_rollback_after_error( - file_path.as_path(), - &xl_path, - Some(rollback_dir), - volume, - path, - DeleteRollbackFailure { - stage: "delete_version_reserve_data", - error: err, - }, - &self.publication_root, - ) - .await); - } - }; - let rollback_data_path = rollback_path.join(uuid.to_string()); - if !reserved_version_delete - && let Err(err) = - rename_all_ignore_missing_source(&old_path, &rollback_data_path, &rollback_path, &self.publication_root) - .await - { - return Err(restore_delete_rollback_after_error( - file_path.as_path(), - &xl_path, - Some(rollback_dir), - volume, - path, - DeleteRollbackFailure { - stage: "delete_version_stage_data", - error: err, - }, - &self.publication_root, - ) - .await); - } - if should_fail_after_delete_data_staged(path) { - if reserved_version_delete { - return Err(self - .abort_reserved_version_delete( - file_path.as_path(), - rollback_dir, - volume, - path, - "delete_version_test_after_stage", - DiskError::Unexpected, - ) - .await); - } - return Err(restore_delete_rollback_after_error( - file_path.as_path(), - &xl_path, - Some(rollback_dir), - volume, - path, - DeleteRollbackFailure { - stage: "delete_version_test_after_stage", - error: DiskError::Unexpected, - }, - &self.publication_root, - ) - .await); - } - } else if let Err(err) = self.move_to_trash(&old_path, true, false).await - && err != DiskError::FileNotFound - && err != DiskError::VolumeNotFound - { - return Err(err); - } - - // The version's data dir was staged for rollback or trashed, so its - // `part.N` inodes no longer exist for readers. A cached io_uring - // descriptor would keep serving them, so drop every cached fd under - // this data dir (rustfs/backlog#1175). If a later rollback restores - // the dir, the next read simply re-opens it. - self.io_backend.invalidate_cached_fds_under(volume, &format!("{path}/{uuid}")); - } - - let commit_result = if !meta.versions.is_empty() { - let buf = match meta.marshal_msg() { - Ok(buf) => buf, - Err(err) => { - let err: DiskError = err.into(); - if reserved_version_delete && let Some(rollback_dir) = rollback_dir { - return Err(self - .abort_reserved_version_delete( - file_path.as_path(), - rollback_dir, - volume, - path, - "delete_version_metadata_encode", - err, - ) - .await); - } - return Err(restore_delete_rollback_after_error( - file_path.as_path(), - &xl_path, - rollback_dir, - volume, - path, - DeleteRollbackFailure { - stage: "delete_version_metadata_encode", - error: err, - }, - &self.publication_root, - ) - .await); - } - }; - self.write_all_meta(volume, format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true) - .await - } else { - self.delete_file(&volume_dir, &xl_path, true, false).await - }; - - if let Err(err) = commit_result { - if reserved_version_delete && let Some(rollback_dir) = rollback_dir { - return Err(self - .abort_reserved_version_delete(file_path.as_path(), rollback_dir, volume, path, "delete_version_commit", err) - .await); - } - return Err(restore_delete_rollback_after_error( - file_path.as_path(), - &xl_path, - rollback_dir, - volume, - path, - DeleteRollbackFailure { - stage: "delete_version_commit", - error: err, - }, - &self.publication_root, - ) - .await); - } - - if reserved_version_delete - && let Some(rollback_dir) = rollback_dir - && let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await - { - return Err(self - .abort_reserved_version_delete( - file_path.as_path(), - rollback_dir, - volume, - path, - "delete_version_commit_intent", - err, - ) - .await); - } - - if should_fail_after_delete_commit(self.root.as_path(), path) { - return Err(DiskError::Unexpected); - } - - Ok(()) + self.delete_version_inner( + volume, + path, + fi, + DeleteVersionMutation { + force_del_marker, + opts, + namespace_owner: None, + }, + ) + .await } #[tracing::instrument(level = "trace", skip_all)] async fn delete_versions(&self, volume: &str, versions: Vec, opts: DeleteOptions) -> Vec> { @@ -12971,6 +13168,464 @@ mod test { ); } + #[cfg(not(windows))] + async fn assert_undo_physical_namespace_owner(case: &str, cancel_waiter: bool) { + use crate::disk::{disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks}; + + temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async { + let dir = tempfile::tempdir().expect("fixture directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 fixture path")).expect("endpoint"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk")); + let bucket = "physical-undo"; + let object = format!("object-{case}"); + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_DELETED_BUCKET).await; + let object_dir = disk.io_get_object_path(bucket, &object).expect("object IO path"); + let xl_path = object_dir.join(STORAGE_FORMAT_FILE); + let old_version = Uuid::new_v4(); + let mut old = test_file_info(&object, old_version, None, Some(Bytes::from_static(b"old-payload"))); + old.set_inline_data(); + let old_meta = test_meta(old.clone()); + let new_version = if case == "backup" { old_version } else { Uuid::new_v4() }; + let mut new = test_file_info(&object, new_version, None, Some(Bytes::from_static(b"new-payload"))); + new.set_inline_data(); + let rollback_dir = Uuid::new_v4(); + let mut opts = DeleteOptions { + undo_write: true, + ..Default::default() + }; + fs::create_dir_all(&object_dir).await.expect("object directory"); + let stage = match case { + "backup" => { + fs::write(&xl_path, test_meta(new.clone())).await.expect("current metadata"); + write_metadata_rollback_backup(&object_dir, rollback_dir, &old_meta) + .await + .expect("rollback backup"); + opts.old_data_dir = Some(rollback_dir); + hooks::Stage::Rename + } + "marker" => { + new = FileInfo { + name: object.clone(), + version_id: Some(new_version), + deleted: true, + mark_deleted: true, + mod_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }; + disk.delete_version( + bucket, + &object, + new.clone(), + true, + DeleteOptions { + old_data_dir: Some(rollback_dir), + ..Default::default() + }, + ) + .await + .expect("create the real no-backup delete-marker rollback intent"); + assert!( + object_dir + .join(rollback_dir.to_string()) + .join(DELETE_MARKER_ROLLBACK_FILE) + .exists() + ); + opts.old_data_dir = Some(rollback_dir); + opts.undo_delete = true; + hooks::Stage::Remove + } + "fresh" => { + fs::write(&xl_path, test_meta(new.clone())) + .await + .expect("new version metadata"); + hooks::Stage::Rename + } + "remaining" => { + let mut meta = FileMeta::load(&old_meta).expect("old metadata parses"); + meta.add_version(new.clone()).expect("add distinct new version"); + fs::write(&xl_path, meta.marshal_msg().expect("encode both versions")) + .await + .expect("versioned metadata"); + hooks::Stage::Rename + } + _ => panic!("unknown undo fixture"), + }; + let before = fs::read(&xl_path).await.expect("metadata exists before undo"); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install_at(stage, &xl_path, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let owner = ctx.begin_namespace_commit(); + let owner_probe = Arc::downgrade(&owner); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let mut undo = Box::pin(wrapper.undo_write_with_namespace_owner(bucket, &object, new, opts, Some(owner))); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("physical undo must signal entry"), + _ = undo.as_mut() => panic!("undo returned before its physical publication"), + } + }) + .await + .expect("undo must enter its real filesystem executor"); + assert_eq!(std::fs::read(&xl_path).expect("pre-publication metadata"), before); + assert!(ctx.namespace_commits_pending()); + if cancel_waiter { + drop(undo); + } else { + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(61)).await; + tokio::time::resume(); + let error = tokio::time::timeout(Duration::from_secs(5), undo) + .await + .expect("ordinary undo timeout must not wait for its syscall") + .expect_err("the blocked undo must time out"); + assert_eq!(error, DiskError::Timeout); + } + let pending_while_blocked = ctx.namespace_commits_pending(); + let owner_alive_while_blocked = owner_probe.upgrade().is_some(); + let generation_before_publication = ctx.namespace_commit_generation(); + assert_eq!(std::fs::read(&xl_path).expect("still blocked metadata"), before); + drop(release_tx); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let published = match case { + "backup" => std::fs::read(&xl_path).is_ok_and(|data| data == old_meta), + "marker" | "fresh" => !xl_path.exists(), + "remaining" => std::fs::read(&xl_path) + .ok() + .and_then(|data| FileMeta::load(&data).ok()) + .is_some_and(|meta| { + meta.find_version(Some(old_version)).is_ok() && meta.find_version(Some(new_version)).is_err() + }), + _ => unreachable!(), + }; + if published && owner_probe.upgrade().is_none() && !ctx.namespace_commits_pending() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("released physical undo must publish and release ownership"); + if matches!(case, "backup" | "remaining") { + let restored = disk + .read_version( + "", + bucket, + &object, + &old_version.to_string(), + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("the old version remains readable after physical undo"); + assert_eq!(restored.data.as_deref(), Some(b"old-payload".as_slice())); + } + assert!( + pending_while_blocked && owner_alive_while_blocked, + "undo {case} lost namespace ownership while physical publication was pending" + ); + assert!(!ctx.namespace_commits_pending()); + assert!(ctx.namespace_commit_generation() > generation_before_publication); + }) + .await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn undo_backup_restore_keeps_physical_namespace_owner_after_cancellation() { + assert_undo_physical_namespace_owner("backup", true).await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn undo_delete_marker_removal_keeps_physical_namespace_owner_after_timeout() { + assert_undo_physical_namespace_owner("marker", false).await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn undo_fresh_version_keeps_physical_namespace_owner_after_timeout() { + assert_undo_physical_namespace_owner("fresh", false).await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn undo_version_rewrite_keeps_physical_namespace_owner_after_cancellation() { + assert_undo_physical_namespace_owner("remaining", true).await; + } + + #[cfg(not(windows))] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(capacity_dirty_scope)] + async fn inline_rollback_keeps_namespace_owner_while_cancellation_is_requested() { + use crate::disk::{disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks}; + let dir = tempfile::tempdir().expect("fixture directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 fixture path")).expect("endpoint"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk")); + let bucket = "physical-internal-rollback"; + let object = "physical-owner-inline-internal-rollback-object"; + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + let object_dir = disk.io_get_object_path(bucket, object).expect("object IO path"); + let xl_path = object_dir.join(STORAGE_FORMAT_FILE); + fs::create_dir_all(&object_dir).await.expect("object directory"); + let version_id = Uuid::new_v4(); + let mut old = test_file_info(object, version_id, None, Some(Bytes::from_static(b"old-payload"))); + old.set_inline_data(); + let old_meta = test_meta(old); + fs::write(&xl_path, &old_meta).await.expect("old metadata"); + set_rename_data_fail_after_metadata_commit(object); + let mut new = test_file_info(object, version_id, None, Some(Bytes::from_static(b"new-payload"))); + new.set_inline_data(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install_at(hooks::Stage::Rollback, &xl_path, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let owner = ctx.begin_namespace_commit(); + let owner_probe = Arc::downgrade(&owner); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let mut rename = tokio::spawn(async move { + wrapper + .rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "source", + &new, + bucket, + object, + crate::disk::RenameDataGuards { + namespace_owner: Some(owner), + ..Default::default() + }, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("internal rollback must signal entry"), + _ = &mut rename => panic!("rename returned before the physical internal rollback"), + } + }) + .await + .expect("post-commit fault must reach the physical rollback"); + let published = disk + .read_version( + "", + bucket, + object, + &version_id.to_string(), + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("new metadata was really published before rollback"); + assert_eq!(published.data.as_deref(), Some(b"new-payload".as_slice())); + rename.abort(); + let pending = ctx.namespace_commits_pending(); + let alive = owner_probe.upgrade().is_some(); + let generation = ctx.namespace_commit_generation(); + drop(release_tx); + tokio::time::timeout(Duration::from_secs(5), async { + while !std::fs::read(&xl_path).is_ok_and(|data| data == old_meta) + || owner_probe.upgrade().is_some() + || ctx.namespace_commits_pending() + { + tokio::task::yield_now().await; + } + }) + .await + .expect("physical rollback must restore the old bytes"); + let _cancelled = rename.await; + assert!(pending && alive, "cancelled internal rollback must retain its namespace owner"); + assert!(!ctx.namespace_commits_pending()); + assert!(ctx.namespace_commit_generation() > generation); + } + + #[cfg(not(windows))] + async fn assert_physical_namespace_owner_and_quota_claim(pause_at: &str) { + use crate::disk::{disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks}; + let _group_commit = os::set_dst_dir_fsync_group_commit_for_test(pause_at == "fsync"); + let _durability = durability_mode_override::set(DurabilityMode::Strict); + temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async { + let dir = tempfile::tempdir().expect("fixture directory"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 fixture path")).expect("endpoint"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk")); + let bucket = "physical-quota-coexistence"; + let object = "object"; + ensure_test_volume(&disk, bucket).await; + ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await; + let object_dir = disk.io_get_object_path(bucket, object).expect("object IO path"); + let xl_path = object_dir.join(STORAGE_FORMAT_FILE); + let fence_path = quota_mutation_fence_path(bucket, object); + let token = disk + .acquire_snapshot_lease(RUSTFS_META_BUCKET, &fence_path) + .await + .expect("quota fence"); + let fence = disk + .snapshot_leases + .lock() + .await + .entries + .get(&SnapshotLeaseKey { + volume: RUSTFS_META_BUCKET.to_string(), + path: fence_path.clone(), + }) + .and_then(|entry| entry.mutation_fence.clone()) + .expect("real quota fence state"); + let version_id = Uuid::new_v4(); + let staged_backup = disk + .io_get_object_path(RUSTFS_META_TMP_BUCKET, "source/xl.meta.bkp") + .expect("staged backup IO path"); + if pause_at != "prepared" { + let mut old = test_file_info(object, version_id, None, Some(Bytes::from_static(b"old-payload"))); + old.set_inline_data(); + fs::create_dir_all(&object_dir).await.expect("existing object directory"); + fs::write(&xl_path, test_meta(old)) + .await + .expect("old metadata requiring a rollback backup"); + } + let mut new = test_file_info(object, version_id, None, Some(Bytes::from_static(b"new-payload"))); + new.set_inline_data(); + rustfs_utils::http::metadata_compat::insert_str( + &mut new.metadata, + super::super::QUOTA_MUTATION_FENCE_METADATA_SUFFIX, + token.as_uuid().to_string(), + ); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let (stage, pause_path) = match pause_at { + "backup" => (hooks::Stage::Rename, staged_backup.clone()), + // Group fsync keys use canonical paths, including on Linux mount-FD IO paths. + "fsync" => (hooks::Stage::DirFsync, object_dir.canonicalize().expect("canonical group fsync path")), + "prepared" => (hooks::Stage::PreparedRename, xl_path.clone()), + _ => panic!("unknown physical quota pause"), + }; + let _hook = hooks::install_at(stage, &pause_path, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let owner = ctx.begin_namespace_commit(); + let owner_probe = Arc::downgrade(&owner); + let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false); + let mut rename = Box::pin(wrapper.rename_data_observed_with_guards( + RUSTFS_META_TMP_BUCKET, + "source", + &new, + bucket, + object, + crate::disk::RenameDataGuards { + namespace_owner: Some(owner), + ..Default::default() + }, + )); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + entered = entered_rx => entered.expect("physical publication entry"), + _ = rename.as_mut() => panic!("rename returned before publication"), + } + }) + .await + .expect("publication must enter with a real quota claim"); + assert_eq!(fence.running.load(Ordering::Acquire), 1); + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(61)).await; + tokio::time::resume(); + let observed = tokio::time::timeout(Duration::from_secs(5), rename) + .await + .expect("namespace owner must not select the external-guard unlimited deadline"); + assert!(!observed.rejected_before_publication()); + assert_eq!(observed.result.expect_err("ordinary timeout"), DiskError::Timeout); + assert_eq!( + fence.running.load(Ordering::Acquire), + 1, + "namespace owner must not replace the quota claim" + ); + assert!(ctx.namespace_commits_pending()); + assert!(owner_probe.upgrade().is_some()); + let generation = ctx.namespace_commit_generation(); + let mut revoke = + Box::pin(disk.release_snapshot_lease(RUSTFS_META_BUCKET, &fence_path, SnapshotLeaseToken::revoke_all())); + assert!(futures::poll!(tokio::task::unconstrained(revoke.as_mut())).is_pending()); + assert!(fence.revoked.load(Ordering::Acquire), "revoke must actually enter its claim-drain wait"); + drop(release_tx); + tokio::time::timeout(Duration::from_secs(5), revoke) + .await + .expect("quota claim drains after syscall") + .expect("revoke"); + tokio::time::timeout(Duration::from_secs(5), async { + while owner_probe.upgrade().is_some() || ctx.namespace_commits_pending() { + tokio::task::yield_now().await; + } + }) + .await + .expect("namespace owner drains alongside quota claim"); + assert_eq!(fence.running.load(Ordering::Acquire), 0); + assert!(!ctx.namespace_commits_pending()); + assert!(ctx.namespace_commit_generation() > generation); + let stored = disk + .read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("late publication remains readable"); + let expected = if pause_at == "backup" { + b"old-payload".as_slice() + } else { + b"new-payload".as_slice() + }; + assert_eq!(stored.data.as_deref(), Some(expected)); + if pause_at == "backup" { + assert!(!staged_backup.exists(), "the detached sibling lease must really publish the backup"); + } + }) + .await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope, dst_dir_fsync_group_commit)] + async fn physical_namespace_owner_and_quota_claim_both_survive_rename_timeout() { + assert_physical_namespace_owner_and_quota_claim("prepared").await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope, dst_dir_fsync_group_commit)] + async fn rollback_backup_sibling_lease_keeps_namespace_owner_and_quota_after_timeout() { + assert_physical_namespace_owner_and_quota_claim("backup").await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope, dst_dir_fsync_group_commit)] + async fn grouped_fsync_keeps_physical_namespace_owner_and_quota_after_timeout() { + assert_physical_namespace_owner_and_quota_claim("fsync").await; + } + #[tokio::test] async fn observed_rename_timeout_has_no_preflight_proof_and_retains_namespace_lease() { use crate::disk::disk_store::LocalDiskWrapper; @@ -18709,11 +19364,17 @@ mod test { path_resolve_stage: "path", metadata_lookup_stage: "metadata_lookup", metadata_validate_stage: "metadata_validate", + #[cfg(unix)] blocking_wait_stage: "blocking_wait", + #[cfg(unix)] blocking_task_stage: "blocking_task", + #[cfg(unix)] file_open_stage: "file_open", + #[cfg(unix)] mmap_map_stage: "mmap_map", + #[cfg(unix)] mmap_copy_stage: "mmap_copy", + #[cfg(unix)] direct_read_copy_stage: "direct_read_copy", }; diff --git a/crates/ecstore/src/disk/local/commit.rs b/crates/ecstore/src/disk/local/commit.rs index 9d51bc1cd..ccb950c27 100644 --- a/crates/ecstore/src/disk/local/commit.rs +++ b/crates/ecstore/src/disk/local/commit.rs @@ -17,13 +17,14 @@ #[cfg(all(test, windows))] use super::run_destination_commit_directory_preparation; +#[cfg(any(not(windows), test))] +use super::should_fail_local_inline_rollback_hardlink; use super::{ EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size, remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature, run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup, - should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit, - skip_access_checks, + should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks, }; #[cfg(test)] use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication}; @@ -33,7 +34,7 @@ use crate::disk::{ error::{DiskError, Result}, error_conv::{to_access_error, to_file_error}, os, - os::{check_path_length, rename_all}, + os::check_path_length, }; use bytes::Bytes; use rustfs_filemeta::{FileInfo, FileMeta}; @@ -73,6 +74,8 @@ fn rollback_inline_metadata_commit_std( rollback_data_dir: Option, local_rollback_path: Option<&Path>, ) -> std::io::Result<()> { + #[cfg(all(test, not(windows)))] + os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, dst_file_path); if let Some(backup_path) = local_rollback_path { // The commit immediately before this rollback renamed the staged // xl.meta from the same directory as `backup_path` onto @@ -86,6 +89,7 @@ fn rollback_inline_metadata_commit_std( Ok(()) } +#[cfg(any(not(windows), test))] pub(super) fn create_local_inline_rollback_backup( dst_file_path: &Path, staging_file_path: &Path, @@ -231,6 +235,12 @@ async fn restore_published_data_source( #[derive(Debug)] pub(in crate::disk) struct LocalRenamePreflightRejection(()); +#[derive(Default)] +pub(super) struct RenameDataState { + namespace_owner: Option>, + preflight_rejection: Option, +} + impl LocalDisk { #[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)] pub(super) async fn rename_data_inner( @@ -240,7 +250,7 @@ impl LocalDisk { fi: FileInfo, dst_volume: &str, dst_path: &str, - preflight_rejection: &mut Option, + state: &mut RenameDataState, ) -> Result { crate::hp_guard!("LocalDisk::rename_data"); let mut fi = fi; @@ -269,7 +279,13 @@ impl LocalDisk { Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?), None => None, }; - let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await; + let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner( + &self.root, + dst_volume, + &destination_object_path, + state.namespace_owner.take(), + ) + .await; if let Some(claim) = quota_fence_claim { mutation_lease.attach_external_guard(claim); } @@ -302,7 +318,7 @@ impl LocalDisk { error = %e, "Disk local access check failed" ); - *preflight_rejection = Some(LocalRenamePreflightRejection(())); + state.preflight_rejection = Some(LocalRenamePreflightRejection(())); return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); } @@ -320,7 +336,7 @@ impl LocalDisk { error = %e, "Disk local access check failed" ); - *preflight_rejection = Some(LocalRenamePreflightRejection(())); + state.preflight_rejection = Some(LocalRenamePreflightRejection(())); return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); } @@ -528,7 +544,9 @@ impl LocalDisk { // rename below. if fi_healing && let Some((_, dst_data_path)) = has_data_dir_path.as_ref() - && let Err(err) = self.move_to_trash(dst_data_path, true, false).await + && let Err(err) = self + .move_to_trash_with_namespace_owner(dst_data_path, true, false, Some(mutation_lease.clone())) + .await { warn!( target: "rustfs_ecstore::disk::local", @@ -755,7 +773,7 @@ impl LocalDisk { && let Some(parent) = dst_file_path.parent() { let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dst_dir_group_commit(parent).await { + if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(mutation_lease.clone())).await { rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC, fsync_started, @@ -793,7 +811,7 @@ impl LocalDisk { break; } let fsync_started = rustfs_io_metrics::put_stage_timer(); - if let Err(err) = os::fsync_dir(dir).await { + if let Err(err) = os::fsync_dir_with_owner(dir, Some(mutation_lease.clone())).await { rustfs_io_metrics::record_put_object_stage_duration_from( rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC, fsync_started, @@ -1024,7 +1042,15 @@ impl LocalDisk { // rename_all acquires the backup path's namespace lease. Do not // hold a disk admission while acquiring another namespace lock. drop(file_sync_admission.take()); - if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await { + if let Err(err) = os::rename_all_with_owner( + staged_backup, + &backup_path, + &dst_volume_dir, + &self.publication_root, + Some(mutation_lease.clone()), + ) + .await + { let _ = remove_file_if_exists(staged_backup); return Err(err); } @@ -1220,14 +1246,18 @@ impl LocalDisk { fi: &FileInfo, dst_volume: &str, dst_path: &str, + namespace_owner: Option>, ) -> super::super::RenameDataObservation { - let mut preflight_rejection = None; + let mut state = RenameDataState { + namespace_owner, + ..Default::default() + }; let result = self - .rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection) + .rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut state) .await; super::super::RenameDataObservation { result, - preflight_rejection, + preflight_rejection: state.preflight_rejection, } } } diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index c2f2c52b4..5e6ce9786 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -75,6 +75,14 @@ use time::OffsetDateTime; use tokio::io::{AsyncRead, AsyncWrite}; use uuid::Uuid; +/// Independent admission and physical ownership for one disk rename. +#[derive(Default)] +pub(crate) struct RenameDataGuards { + pub(crate) scanner_publication_lease_token: Option, + pub(crate) external_guard: Option>, + pub(crate) namespace_owner: Option>, +} + /// Local preflight evidence stays outside DiskAPI and the RPC response format. pub(crate) struct RenameDataObservation { pub(crate) result: Result, @@ -190,11 +198,17 @@ pub struct MmapCopyStageMetrics { pub(crate) path_resolve_stage: &'static str, pub(crate) metadata_lookup_stage: &'static str, pub(crate) metadata_validate_stage: &'static str, + #[cfg(unix)] pub(crate) blocking_wait_stage: &'static str, + #[cfg(unix)] pub(crate) blocking_task_stage: &'static str, + #[cfg(unix)] pub(crate) file_open_stage: &'static str, + #[cfg(unix)] pub(crate) mmap_map_stage: &'static str, + #[cfg(unix)] pub(crate) mmap_copy_stage: &'static str, + #[cfg(unix)] pub(crate) direct_read_copy_stage: &'static str, } @@ -718,6 +732,25 @@ impl Disk { } } + /// Keep local undo publication owned independently of the wrapper deadline. + /// Remote undo retains its existing RPC contract; this is not a remote drain proof. + pub(crate) async fn undo_write_with_namespace_owner( + &self, + volume: &str, + path: &str, + fi: FileInfo, + opts: DeleteOptions, + namespace_owner: Option>, + ) -> Result<()> { + match self { + Self::Local(disk) => { + disk.undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner) + .await + } + Self::Remote(disk) => disk.delete_version(volume, path, fi, false, opts).await, + } + } + pub(crate) async fn rename_data_borrowed( &self, src_volume: &str, @@ -737,12 +770,12 @@ impl Disk { fi: &FileInfo, dst_volume: &str, dst_path: &str, - scanner_publication_lease_token: Option, + guards: RenameDataGuards, ) -> RenameDataObservation { match self { Disk::Local(local_disk) => { local_disk - .rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None) + .rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards) .await } Disk::Remote(remote_disk) => RenameDataObservation::unknown( @@ -753,7 +786,7 @@ impl Disk { fi, dst_volume, dst_path, - scanner_publication_lease_token, + guards.scanner_publication_lease_token, ) .await, ), @@ -922,6 +955,7 @@ impl Disk { } } + #[cfg(unix)] pub(crate) fn get_object_path_for_io_if_local( &self, volume: &str, diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index e68a51d99..9940d36ba 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -91,6 +91,7 @@ pub(crate) mod fsync_dir_recorder { static RECORDED: Mutex> = Mutex::new(Vec::new()); static LIMITED: Mutex> = Mutex::new(Vec::new()); static GROUPED: Mutex> = Mutex::new(Vec::new()); + #[cfg(unix)] static BEFORE_LIMITED: std::sync::LazyLock>> = std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); static BEFORE_GROUP_BATCH: std::sync::LazyLock>> = @@ -150,6 +151,7 @@ pub(crate) mod fsync_dir_recorder { contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir) } + #[cfg(unix)] pub(crate) fn record_limited(dir: &Path) { record_path(&LIMITED, dir, "limited fsync dir recorder"); let hook = remove_hook(&BEFORE_LIMITED, dir, "limited fsync hook poisoned"); @@ -162,6 +164,7 @@ pub(crate) mod fsync_dir_recorder { contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir) } + #[cfg(unix)] pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) { BEFORE_LIMITED .lock() @@ -237,11 +240,56 @@ pub(crate) mod fsync_dir_recorder { .insert(dir.to_path_buf(), kind); } + #[cfg(unix)] pub(crate) fn take_grouped_failure(dir: &Path) -> Option { remove_path_keyed(&GROUPED_FAILURES, dir, "grouped fsync failure hook poisoned") } } +/// Pause a real namespace mutation inside its physical executor. +#[cfg(all(test, not(windows)))] +pub(crate) mod prepared_publication_test_hooks { + use super::*; + + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] + pub(crate) enum Stage { + PreparedRename, + Rename, + Remove, + Rollback, + DirFsync, + } + + type Hook = Box; + type Key = (Stage, PathBuf); + static BEFORE_PUBLICATION: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); + + pub(crate) struct Guard(Key); + + impl Drop for Guard { + fn drop(&mut self) { + BEFORE_PUBLICATION.lock().remove(&self.0); + } + } + + pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard { + install_at(Stage::PreparedRename, path, hook) + } + + pub(crate) fn install_at(stage: Stage, path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard { + let key = (stage, path.to_path_buf()); + assert!(BEFORE_PUBLICATION.lock().insert(key.clone(), Box::new(hook)).is_none()); + Guard(key) + } + + pub(crate) fn run(stage: Stage, path: &Path) { + let hook = BEFORE_PUBLICATION.lock().remove(&(stage, path.to_path_buf())); + if let Some(hook) = hook { + hook(); + } + } +} + #[cfg(all(test, windows))] pub(crate) mod windows_rename_test_hooks { use super::*; @@ -576,6 +624,7 @@ impl OpenedDstDirFsyncGroup { } struct DstDirFsyncWaiter { + namespace_owner: Option>, result_tx: oneshot::Sender, } @@ -634,6 +683,7 @@ impl DstDirFsyncGroupCommit { fn enqueue_opened( &self, opened: OpenedDstDirFsyncGroup, + namespace_owner: Option>, ) -> io::Result<(oneshot::Receiver, Option>)> { let (result_tx, result_rx) = oneshot::channel(); let mut registry = self.inner.lock(); @@ -664,7 +714,10 @@ impl DstDirFsyncGroupCommit { group }; let mut group_state = group.inner.lock(); - group_state.pending.push_back(DstDirFsyncWaiter { result_tx }); + group_state.pending.push_back(DstDirFsyncWaiter { + result_tx, + namespace_owner, + }); let start_worker = !group_state.worker_running; if start_worker { group_state.worker_running = true; @@ -686,7 +739,13 @@ impl DstDirFsyncGroupCommit { fn remove_idle_group(&self, group: &Arc) { let mut registry = self.inner.lock(); let group_state = group.inner.lock(); - if !group_state.worker_running && group_state.pending.is_empty() { + if !group_state.worker_running + && group_state.pending.is_empty() + && registry + .groups + .get(&group.key) + .is_some_and(|registered| Arc::ptr_eq(registered, group)) + { registry.groups.remove(&group.key); } } @@ -709,16 +768,20 @@ impl DstDirFsyncGroupCommit { &self, dir: &Path, ) -> io::Result<(oneshot::Receiver, Option>)> { - self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?) + self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?, None) } } #[cfg(unix)] -async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> { +async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec>) -> io::Result<()> { #[cfg(test)] let dir = group.dir.clone(); let dir_file = group.dir_file.clone(); fsync_spawn_blocking(move || { + // The batch worker may be cancelled while this syscall is still running. + let _namespace_owners = namespace_owners; + #[cfg(all(test, not(windows)))] + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &dir); #[cfg(test)] { if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) { @@ -733,66 +796,118 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> { } #[cfg(not(unix))] -async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> { +async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec>) -> io::Result<()> { + let _namespace_owners = namespace_owners; fsync_dir(&group.dir).await } -async fn run_dst_dir_fsync_group_worker(group: Arc) { - loop { - #[cfg(test)] - fsync_dir_recorder::run_before_group_batch(&group.dir); - tokio::task::yield_now().await; - let batch: Vec = { - let mut group_state = group.inner.lock(); - group_state.pending.drain(..).collect() - }; - if batch.is_empty() { - let mut group_state = group.inner.lock(); +struct DstDirFsyncWorkerGuard { + group: Arc, + in_flight: usize, + armed: bool, +} + +impl Drop for DstDirFsyncWorkerGuard { + fn drop(&mut self) { + if !self.armed { + return; + } + // Cancellation must release queued owners, but the physical batch keeps + // its own owners until its blocking syscall returns. + let pending = { + let mut registry = DST_DIR_FSYNC_GROUP_COMMIT.inner.lock(); + let mut group_state = self.group.inner.lock(); + let pending = std::mem::take(&mut group_state.pending); group_state.worker_running = false; - drop(group_state); - DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group); - return; - } - - #[cfg(test)] - fsync_dir_recorder::record_grouped(&group.dir, batch.len()); - let result = fsync_open_dst_dir_group(&group) - .await - .map_err(SharedDstDirFsyncError::from_error); - let batch_len = batch.len(); - DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len); - - let should_stop = { - let mut group_state = group.inner.lock(); - if group_state.pending.is_empty() { - group_state.worker_running = false; - true - } else { - false + if registry + .groups + .get(&self.group.key) + .is_some_and(|group| Arc::ptr_eq(group, &self.group)) + { + registry.total_waiters = registry.total_waiters.saturating_sub(pending.len() + self.in_flight); + registry.groups.remove(&self.group.key); } + pending }; - if should_stop { - DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group); - } - for waiter in batch { - let _ = waiter.result_tx.send(result.clone()); - } - if should_stop { - return; + // Lease and channel destructors must run outside the registry locks. + drop(pending); + } +} + +fn run_dst_dir_fsync_group_worker(group: Arc) -> impl std::future::Future { + // Capture before spawning: shutdown may drop the future without polling it. + let worker_guard = DstDirFsyncWorkerGuard { + group: group.clone(), + in_flight: 0, + armed: true, + }; + async move { + let mut worker_guard = worker_guard; + loop { + #[cfg(test)] + fsync_dir_recorder::run_before_group_batch(&group.dir); + tokio::task::yield_now().await; + let mut batch: Vec = { + let mut group_state = group.inner.lock(); + group_state.pending.drain(..).collect() + }; + if batch.is_empty() { + let mut group_state = group.inner.lock(); + worker_guard.armed = false; + group_state.worker_running = false; + drop(group_state); + DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group); + return; + } + worker_guard.in_flight = batch.len(); + + #[cfg(test)] + fsync_dir_recorder::record_grouped(&group.dir, batch.len()); + let namespace_owners = batch.iter_mut().filter_map(|waiter| waiter.namespace_owner.take()).collect(); + let result = fsync_open_dst_dir_group(&group, namespace_owners) + .await + .map_err(SharedDstDirFsyncError::from_error); + let batch_len = batch.len(); + DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len); + worker_guard.in_flight = 0; + + let should_stop = { + let mut group_state = group.inner.lock(); + if group_state.pending.is_empty() { + worker_guard.armed = false; + group_state.worker_running = false; + true + } else { + false + } + }; + if should_stop { + DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group); + } + for waiter in batch { + let _ = waiter.result_tx.send(result.clone()); + } + if should_stop { + return; + } } } } -async fn fsync_dst_dir_group_commit_with_enabled(dir: impl AsRef, enabled: bool) -> io::Result<()> { +async fn fsync_dst_dir_group_commit_with_enabled( + dir: impl AsRef, + enabled: bool, + namespace_owner: Option>, +) -> io::Result<()> { if !enabled { - return fsync_dir(dir).await; + return fsync_dir_with_owner(dir.as_ref(), namespace_owner).await; } let dir = dir.as_ref().to_path_buf(); let opened = tokio::task::spawn_blocking(move || OpenedDstDirFsyncGroup::open(&dir)) .await .map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??; - let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened)?; + let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened, namespace_owner)?; if let Some(group) = worker { tokio::spawn(run_dst_dir_fsync_group_worker(group)); } @@ -804,8 +919,11 @@ async fn fsync_dst_dir_group_commit_with_enabled(dir: impl AsRef, enabled: } } -pub(crate) async fn fsync_dst_dir_group_commit(dir: impl AsRef) -> io::Result<()> { - fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled()).await +pub(crate) async fn fsync_dst_dir_group_commit( + dir: impl AsRef, + namespace_owner: Option>, +) -> io::Result<()> { + fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled(), namespace_owner).await } pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit( @@ -814,7 +932,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit( admission: &FileSyncAdmission, ) -> io::Result<()> { if dst_dir_fsync_group_commit_enabled() { - fsync_dst_dir_group_commit_with_enabled(dir, true).await + fsync_dst_dir_group_commit_with_enabled(dir, true, Some(lease)).await } else { fsync_dir_with_namespace_file_sync_limit(dir, lease, admission).await } @@ -822,7 +940,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit( #[cfg(test)] pub(crate) async fn fsync_dst_dir_group_commit_for_test(dir: impl AsRef, enabled: bool) -> io::Result<()> { - fsync_dst_dir_group_commit_with_enabled(dir, enabled).await + fsync_dst_dir_group_commit_with_enabled(dir, enabled, None).await } #[cfg(test)] @@ -1229,6 +1347,8 @@ pub(crate) struct NamespaceMutationLease { _namespace_guard: OwnedMutexGuard<()>, _volume_guard: Option>, external_guard: Mutex>>, + // Independent of the quota claim; both survive cancellation of the waiter. + _namespace_owner: Option>, } impl NamespaceMutationLease { @@ -1238,10 +1358,18 @@ impl NamespaceMutationLease { } async fn acquire_namespace_mutation_lease(path: &Path) -> Arc { + acquire_namespace_mutation_lease_with_owner(path, None).await +} + +async fn acquire_namespace_mutation_lease_with_owner( + path: &Path, + namespace_owner: Option>, +) -> Arc { Arc::new(NamespaceMutationLease { _namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await, _volume_guard: None, external_guard: Mutex::new(None), + _namespace_owner: namespace_owner, }) } @@ -1251,6 +1379,15 @@ pub(crate) async fn acquire_rename_data_mutation_lease( root: &Path, volume: &str, destination_object: &Path, +) -> Arc { + acquire_rename_data_mutation_lease_with_owner(root, volume, destination_object, None).await +} + +pub(crate) async fn acquire_rename_data_mutation_lease_with_owner( + root: &Path, + volume: &str, + destination_object: &Path, + namespace_owner: Option>, ) -> Arc { let namespace_guard = disk_namespace_mutation_lock(destination_object).lock_owned().await; let volume_guard = disk_volume_mutation_lock(root, volume).read_owned().await; @@ -1258,6 +1395,7 @@ pub(crate) async fn acquire_rename_data_mutation_lease( _namespace_guard: namespace_guard, _volume_guard: Some(volume_guard), external_guard: Mutex::new(None), + _namespace_owner: namespace_owner, }) } @@ -1747,6 +1885,69 @@ pub async fn rename_all( Ok(()) } +pub(crate) async fn fsync_dir_with_owner(path: &Path, namespace_owner: Option>) -> io::Result<()> { + #[cfg(unix)] + { + if namespace_owner.is_none() { + return fsync_dir(path).await; + } + let path = path.to_path_buf(); + fsync_spawn_blocking(move || { + let _namespace_owner = namespace_owner; + fsync_dir_std(path) + }) + .await? + } + #[cfg(not(unix))] + { + let _ = namespace_owner; + fsync_dir(path).await + } +} + +/// Retain namespace ownership in the actual filesystem executor after timeout. +pub(crate) async fn remove_file_with_owner( + path: impl AsRef, + namespace_owner: Option>, +) -> io::Result<()> { + if namespace_owner.is_none() { + return tokio::fs::remove_file(path).await; + } + let path = path.as_ref().to_path_buf(); + let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await; + run_blocking_namespace_operation(lease, move || { + #[cfg(all(test, not(windows)))] + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path); + std::fs::remove_file(path) + }) + .await +} + +/// Retain namespace ownership in the actual filesystem executor after timeout. +pub(crate) async fn remove_dir_with_owner( + path: impl AsRef, + namespace_owner: Option>, +) -> io::Result<()> { + if namespace_owner.is_none() { + return tokio::fs::remove_dir(path).await; + } + let path = path.as_ref().to_path_buf(); + let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await; + run_blocking_namespace_operation(lease, move || std::fs::remove_dir(path)).await +} + +#[tracing::instrument(name = "rename_all", level = "debug", skip_all)] +pub(crate) async fn rename_all_with_owner( + src_file_path: impl AsRef, + dst_file_path: impl AsRef, + base_dir: impl AsRef, + publication_root: &PublicationRoot, + namespace_owner: Option>, +) -> Result<()> { + let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await; + rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await +} + pub(crate) async fn rename_all_with_lease( src_file_path: impl AsRef, dst_file_path: impl AsRef, @@ -1939,6 +2140,8 @@ pub(crate) async fn rename_all_with_prepared_source( move || { validate_prepared_rename_source(&prepared_source, &src_file_path)?; let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?; + #[cfg(test)] + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path); rename_prepared(&src_file_path, &dst_file_path, &preparation) } }; @@ -1977,6 +2180,32 @@ pub async fn rename_all_ignore_missing_source( } } +#[tracing::instrument(name = "rename_all_ignore_missing_source", level = "debug", skip_all)] +pub(crate) async fn rename_all_ignore_missing_source_with_owner( + src_file_path: impl AsRef, + dst_file_path: impl AsRef, + base_dir: impl AsRef, + publication_root: &PublicationRoot, + namespace_owner: Option>, +) -> Result<()> { + let src_file_path = src_file_path.as_ref(); + let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await; + match reliable_rename_inner_with_lease( + src_file_path.to_path_buf(), + dst_file_path.as_ref().to_path_buf(), + base_dir.as_ref().to_path_buf(), + publication_root.clone(), + false, + lease, + ) + .await + { + Ok(()) => Ok(()), + Err(err) if err.kind() == io::ErrorKind::NotFound && rename_source_is_missing(src_file_path, publication_root) => Ok(()), + Err(err) => Err(to_file_error(err).into()), + } +} + #[cfg(windows)] pub(crate) fn rename_source_is_missing(src_file_path: &Path, publication_root: &PublicationRoot) -> bool { let Some(source_parent) = src_file_path.parent() else { @@ -2042,6 +2271,11 @@ async fn reliable_rename_inner_with_lease( let base_dir = base_dir.clone(); move || { let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?; + #[cfg(all(test, not(windows)))] + { + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path); + prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path); + } rename_prepared(&src_file_path, &dst_file_path, &preparation) } }; @@ -6136,6 +6370,245 @@ mod tests { wait_for_dst_dir_fsync_group_commit_idle().await; } + #[cfg(unix)] + #[tokio::test] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn grouped_fsync_physical_batch_keeps_all_owners_after_worker_cancellation() { + let temp_dir = tempdir().expect("fixture directory"); + let dir = temp_dir.path().canonicalize().expect("canonical fsync path"); + let first_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let second_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let first_owner = first_ctx.begin_namespace_commit(); + let second_owner = second_ctx.begin_namespace_commit(); + let first_probe = Arc::downgrade(&first_owner); + let second_probe = Arc::downgrade(&second_owner); + let (first_rx, group) = DST_DIR_FSYNC_GROUP_COMMIT + .enqueue_opened( + OpenedDstDirFsyncGroup::open(&dir).expect("open first waiter directory"), + Some(first_owner), + ) + .expect("queue first real waiter"); + let group = group.expect("first waiter starts the group"); + let (second_rx, second_worker) = DST_DIR_FSYNC_GROUP_COMMIT + .enqueue_opened( + OpenedDstDirFsyncGroup::open(&dir).expect("open second waiter directory"), + Some(second_owner), + ) + .expect("queue second real waiter"); + assert!(second_worker.is_none(), "same directory must join the same batch"); + assert_eq!(group.inner.lock().pending.len(), 2); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = + prepared_publication_test_hooks::install_at(prepared_publication_test_hooks::Stage::DirFsync, &dir, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let worker = tokio::spawn(run_dst_dir_fsync_group_worker(group.clone())); + tokio::time::timeout(Duration::from_secs(5), entered_rx) + .await + .expect("batch must reach its physical fsync") + .expect("physical fsync entry"); + assert_eq!(fsync_dir_recorder::grouped_batch_sizes(&dir), vec![2]); + assert!( + group.inner.lock().pending.is_empty(), + "both waiters were transferred into the physical batch" + ); + let queued_ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let queued_owner = queued_ctx.begin_namespace_commit(); + let queued_probe = Arc::downgrade(&queued_owner); + let queued_generation = queued_ctx.namespace_commit_generation(); + let (queued_rx, queued_worker) = DST_DIR_FSYNC_GROUP_COMMIT + .enqueue_opened( + OpenedDstDirFsyncGroup::open(&dir).expect("open queued waiter directory"), + Some(queued_owner), + ) + .expect("queue a waiter after the physical batch was frozen"); + assert!(queued_worker.is_none()); + assert_eq!(group.inner.lock().pending.len(), 1); + drop((first_rx, second_rx)); + worker.abort(); + assert!(worker.await.expect_err("cancel the async batch owner").is_cancelled()); + assert!(queued_rx.await.is_err(), "an undispatched waiter must observe worker cancellation"); + assert!(queued_probe.upgrade().is_none()); + assert!(!queued_ctx.namespace_commits_pending()); + assert!(queued_ctx.namespace_commit_generation() > queued_generation); + assert!(group.inner.lock().pending.is_empty()); + assert!(!group.inner.lock().worker_running); + assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0)); + let first_pending = first_ctx.namespace_commits_pending() && first_probe.upgrade().is_some(); + let second_pending = second_ctx.namespace_commits_pending() && second_probe.upgrade().is_some(); + let generations = (first_ctx.namespace_commit_generation(), second_ctx.namespace_commit_generation()); + drop(release_tx); + tokio::time::timeout(Duration::from_secs(5), async { + while Arc::strong_count(&group.dir_file) != 1 + || first_probe.upgrade().is_some() + || second_probe.upgrade().is_some() + || first_ctx.namespace_commits_pending() + || second_ctx.namespace_commits_pending() + { + tokio::task::yield_now().await; + } + }) + .await + .expect("physical fsync must release every batch owner"); + assert!(fsync_dir_recorder::was_fsynced(&dir), "the detached syscall must really execute"); + assert!( + first_pending && second_pending, + "one physical batch must preserve both independent namespace owners" + ); + assert!(!first_ctx.namespace_commits_pending()); + assert!(!second_ctx.namespace_commits_pending()); + assert!(first_ctx.namespace_commit_generation() > generations.0); + assert!(second_ctx.namespace_commit_generation() > generations.1); + } + + #[cfg(unix)] + #[tokio::test] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn grouped_fsync_unpolled_worker_releases_queued_owner() { + let temp_dir = tempdir().expect("fixture directory"); + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let owner = ctx.begin_namespace_commit(); + let probe = Arc::downgrade(&owner); + let generation = ctx.namespace_commit_generation(); + let (rx, group) = DST_DIR_FSYNC_GROUP_COMMIT + .enqueue_opened( + OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open queued waiter directory"), + Some(owner), + ) + .expect("queue a real waiter"); + let group = group.expect("first waiter starts the group"); + let worker = run_dst_dir_fsync_group_worker(group.clone()); + assert!(ctx.namespace_commits_pending()); + drop(worker); + assert!(rx.await.is_err(), "shutdown before first poll must release the waiter"); + assert!(probe.upgrade().is_none()); + assert!(!ctx.namespace_commits_pending()); + assert!(ctx.namespace_commit_generation() > generation); + assert!(group.inner.lock().pending.is_empty()); + assert!(!group.inner.lock().worker_running); + assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0)); + assert!( + fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()).is_empty(), + "the dropped future must not dispatch a physical batch" + ); + } + + #[cfg(unix)] + #[test] + fn stale_idle_group_cleanup_preserves_successor_registration() { + let temp_dir = tempdir().expect("fixture directory"); + let registry = DstDirFsyncGroupCommit::default(); + let (mut first_rx, first_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue first worker"); + let old_group = first_worker.expect("first waiter starts a worker"); + // W1 has completed its batch and marked G idle, but has not cleaned G up. + let first_waiter = old_group.inner.lock().pending.pop_front().expect("first batch waiter"); + registry.complete_batch(1); + old_group.inner.lock().worker_running = false; + + let (mut second_rx, second_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue second worker"); + let reused_group = second_worker.expect("idle G starts another worker"); + assert!(Arc::ptr_eq(&old_group, &reused_group)); + let second_waiter = reused_group.inner.lock().pending.pop_front().expect("second batch waiter"); + registry.complete_batch(1); + reused_group.inner.lock().worker_running = false; + registry.remove_idle_group(&reused_group); + assert_eq!(registry.counts_for_test(), (0, 0), "normal idle cleanup must remove G"); + assert!(second_waiter.result_tx.send(Ok(())).is_ok()); + assert!(second_rx.try_recv().expect("second worker reports completion").is_ok()); + + let (mut successor_rx, successor_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue successor"); + let successor = successor_worker.expect("successor starts a new group"); + assert!(!Arc::ptr_eq(&old_group, &successor)); + assert_eq!(registry.counts_for_test(), (1, 1)); + // W1 resumes with its old Arc after W2 removed G and W3 installed G2. + registry.remove_idle_group(&old_group); + assert!(first_waiter.result_tx.send(Ok(())).is_ok()); + assert!(first_rx.try_recv().expect("first worker reports completion").is_ok()); + assert!( + registry + .inner + .lock() + .groups + .get(&successor.key) + .is_some_and(|registered| Arc::ptr_eq(registered, &successor)), + "stale cleanup must retain the exact successor Arc" + ); + assert_eq!(registry.counts_for_test(), (1, 1)); + assert!(successor.inner.lock().worker_running); + assert_eq!(successor.inner.lock().pending.len(), 1); + assert!(matches!(successor_rx.try_recv(), Err(oneshot::error::TryRecvError::Empty))); + + let (_joined_rx, new_worker) = registry.enqueue_for_test(temp_dir.path()).expect("join successor"); + assert!(new_worker.is_none(), "a later waiter must join G2 instead of creating G3"); + assert_eq!(successor.inner.lock().pending.len(), 2); + assert_eq!(registry.counts_for_test(), (1, 2)); + } + + #[cfg(unix)] + #[tokio::test] + #[serial_test::serial(dst_dir_fsync_group_commit)] + async fn stale_idle_cleanup_then_unpolled_worker_drop_releases_waiter_budget() { + wait_for_dst_dir_fsync_group_commit_idle().await; + let temp_dir = tempdir().expect("fixture directory"); + let (old_rx, old_worker) = DST_DIR_FSYNC_GROUP_COMMIT + .enqueue_for_test(temp_dir.path()) + .expect("enqueue old group"); + let old_group = old_worker.expect("old group starts a worker"); + tokio::time::timeout(Duration::from_secs(5), run_dst_dir_fsync_group_worker(old_group.clone())) + .await + .expect("old worker must finish its actual fsync"); + assert!(old_rx.await.expect("old worker reports completion").is_ok()); + assert!(fsync_dir_recorder::was_fsynced(temp_dir.path())); + assert_eq!(fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()), vec![1]); + assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0)); + + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let owner = ctx.begin_namespace_commit(); + let probe = Arc::downgrade(&owner); + let generation = ctx.namespace_commit_generation(); + let (rx, successor_worker) = DST_DIR_FSYNC_GROUP_COMMIT + .enqueue_opened( + OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open successor directory"), + Some(owner), + ) + .expect("enqueue successor owner"); + let successor = successor_worker.expect("successor starts a new group"); + assert!(!Arc::ptr_eq(&old_group, &successor)); + let worker = run_dst_dir_fsync_group_worker(successor.clone()); + // The stale Arc represents W1 resuming after another worker removed G. + DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&old_group); + assert!(ctx.namespace_commits_pending()); + assert!(probe.upgrade().is_some()); + drop(worker); + let channel_closed = tokio::time::timeout(Duration::from_secs(5), rx) + .await + .expect("dropping the unpolled worker must release its channel") + .is_err(); + let counts_after_drop = DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(); + let owner_released = probe.upgrade().is_none(); + let namespace_pending = ctx.namespace_commits_pending(); + let generation_after_drop = ctx.namespace_commit_generation(); + let successor_pending = successor.inner.lock().pending.len(); + let worker_running = successor.inner.lock().worker_running; + // Preserve the observed result before cleanup, so a RED run cannot leak + // its phantom count into unrelated tests in the same process. + clear_dst_dir_fsync_group_commit_for_test(); + assert!(channel_closed); + assert!(owner_released); + assert!(!namespace_pending); + assert!(generation_after_drop > generation); + assert_eq!(successor_pending, 0); + assert!(!worker_running); + assert_eq!( + fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()), + vec![1], + "dropping the successor before its first poll must not dispatch another fsync" + ); + assert_eq!(counts_after_drop, (0, 0), "stale cleanup must not strand a phantom waiter"); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial_test::serial(dst_dir_fsync_group_commit)] async fn dst_dir_fsync_group_commit_cancellation_releases_waiter_state() { diff --git a/crates/ecstore/src/io_support/bitrot.rs b/crates/ecstore/src/io_support/bitrot.rs index 371a93560..d5d5f06de 100644 --- a/crates/ecstore/src/io_support/bitrot.rs +++ b/crates/ecstore/src/io_support/bitrot.rs @@ -13,12 +13,15 @@ // limitations under the License. use crate::diagnostics::get::{ - GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, - GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, - GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE, + GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE, GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS, GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled, }; +#[cfg(unix)] +use crate::diagnostics::get::{ + GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, GET_STAGE_READER_MMAP_COPY_BUFFER, + GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, GET_STAGE_READER_MMAP_MAP, +}; #[cfg(feature = "hotpath")] use crate::disk::FileWriter; use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError}; @@ -406,11 +409,17 @@ async fn open_disk_reader( path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE, metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP, metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE, + #[cfg(unix)] blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT, + #[cfg(unix)] blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK, + #[cfg(unix)] file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN, + #[cfg(unix)] mmap_map_stage: GET_STAGE_READER_MMAP_MAP, + #[cfg(unix)] mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER, + #[cfg(unix)] direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY, }); let mmap_result = { diff --git a/crates/ecstore/src/services/tier/test_util.rs b/crates/ecstore/src/services/tier/test_util.rs index 59b4d6f4e..09c6dee81 100644 --- a/crates/ecstore/src/services/tier/test_util.rs +++ b/crates/ecstore/src/services/tier/test_util.rs @@ -56,6 +56,7 @@ use std::collections::HashMap; use std::io::Cursor; +#[cfg(feature = "test-util")] use std::path::Path; use std::sync::{ Arc, @@ -68,21 +69,28 @@ use tokio::io::AsyncReadExt; use tokio::sync::{Mutex, Notify, RwLock}; use uuid::Uuid; +#[cfg(feature = "test-util")] use crate::disk::endpoint::Endpoint; +#[cfg(feature = "test-util")] use crate::disk::format::FormatV3; +#[cfg(feature = "test-util")] use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk}; use crate::services::tier::tier::TierConfigMgr; use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType}; use crate::services::tier::warm_backend::{ TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options, }; +#[cfg(feature = "test-util")] use rustfs_filemeta::FileMeta; use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl}; +#[cfg(feature = "test-util")] use rustfs_utils::path::path_join_buf; /// One-shot barrier before rejected transition cleanup resolves its ECStore. +#[cfg(feature = "test-util")] pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier); +#[cfg(feature = "test-util")] impl TransitionCleanupStoreBarrier { /// Install the barrier for the next rejected transition cleanup. pub fn install() -> Self { @@ -96,6 +104,7 @@ impl TransitionCleanupStoreBarrier { } /// Default polling cadence used by the `wait_for_*` helpers. +#[cfg(feature = "test-util")] const POLL_INTERVAL: Duration = Duration::from_millis(50); /// A fault to inject into [`MockWarmBackend`] operations. @@ -208,10 +217,12 @@ impl Drop for MockRemoveOperationGuard { } /// One-shot barrier that pauses a mock tier PUT after storing its remote body. +#[cfg(feature = "test-util")] pub struct MockPutBarrier { state: Arc, } +#[cfg(feature = "test-util")] impl MockPutBarrier { /// Wait until the remote body is stored and the PUT is paused before returning. pub async fn wait_until_paused(&self) { @@ -226,6 +237,7 @@ impl MockPutBarrier { } } +#[cfg(feature = "test-util")] impl Drop for MockPutBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -258,10 +270,12 @@ impl Drop for MockGetBarrier { } /// One-shot barrier that pauses and then fails a mock tier DELETE. +#[cfg(feature = "test-util")] pub struct MockRemoveBarrier { state: Arc, } +#[cfg(feature = "test-util")] impl MockRemoveBarrier { /// Wait until DELETE reaches the deterministic failure point. pub async fn wait_until_paused(&self) { @@ -283,6 +297,7 @@ impl MockRemoveBarrier { } } +#[cfg(feature = "test-util")] impl Drop for MockRemoveBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -306,6 +321,7 @@ impl MockWarmBackend { } /// Arm a one-shot pause after the next tier PUT stores its remote body. + #[cfg(feature = "test-util")] pub async fn arm_put_barrier(&self) -> MockPutBarrier { let state = Arc::new(MockPutBarrierState::default()); *self.inner.put_barrier.lock().await = Some(Arc::clone(&state)); @@ -313,6 +329,7 @@ impl MockWarmBackend { } /// Pause and then fail the next DELETE after it reaches the backend. + #[cfg(feature = "test-util")] pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier { let state = Arc::new(MockRemoveBarrierState::default()); let mut barrier = self.inner.remove_barrier.lock().await; @@ -323,6 +340,7 @@ impl MockWarmBackend { /// Arm a one-shot pause before the next tier GET, then return an error /// after the test releases it. + #[cfg(feature = "test-util")] pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier { let state = Arc::new(MockGetBarrierState { fail_after_release: true, @@ -343,6 +361,7 @@ impl MockWarmBackend { // ---- fault injection ------------------------------------------------- /// Replace the entire fault configuration. + #[cfg(feature = "test-util")] pub async fn set_faults(&self, faults: FaultConfig) { *self.inner.faults.lock().await = faults; } @@ -353,6 +372,7 @@ impl MockWarmBackend { } /// Toggle "HTTP 5xx" server errors on every operation. + #[cfg(feature = "test-util")] pub async fn set_server_error(&self, server_error: bool) { self.inner.faults.lock().await.server_error = server_error; } @@ -363,11 +383,13 @@ impl MockWarmBackend { } /// Set (or clear, with `None`) injected latency applied before each op. + #[cfg(feature = "test-util")] pub async fn set_latency(&self, latency: Option) { self.inner.faults.lock().await.latency = latency; } /// Clear all injected faults, restoring healthy behaviour. + #[cfg(feature = "test-util")] pub async fn clear_faults(&self) { *self.inner.faults.lock().await = FaultConfig::default(); } @@ -375,6 +397,7 @@ impl MockWarmBackend { /// Limit how many body bytes a successful mock PUT consumes. `None` drains /// the complete body. This models a backend that incorrectly accepts a /// truncated stream while still returning success. + #[cfg(feature = "test-util")] pub async fn set_put_read_limit(&self, limit: Option) { *self.inner.put_read_limit.lock().await = limit; } @@ -395,12 +418,14 @@ impl MockWarmBackend { } /// Reject non-empty remote versions before transition metadata is committed. + #[cfg(feature = "test-util")] pub fn set_reject_non_empty_remote_versions(&self, reject: bool) { self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release); } /// Reject the next non-empty remote version validation without changing /// subsequent exact-version backend cleanup behavior. + #[cfg(feature = "test-util")] pub fn reject_next_non_empty_remote_version_validation(&self) { self.inner .reject_non_empty_remote_version_validations @@ -438,6 +463,7 @@ impl MockWarmBackend { } /// Clear the operation log without touching stored objects or faults. + #[cfg(feature = "test-util")] pub async fn clear_op_log(&self) { self.inner.op_log.lock().await.clear(); } @@ -459,11 +485,13 @@ impl MockWarmBackend { } /// Return the exact object/version pairs produced by successful tier PUTs. + #[cfg(feature = "test-util")] pub async fn put_versions(&self) -> Vec<(String, String)> { self.inner.put_versions.lock().await.clone() } /// Return the exact object/version pairs passed to successful tier removes. + #[cfg(feature = "test-util")] pub async fn remove_versions(&self) -> Vec<(String, String)> { self.inner.remove_versions.lock().await.clone() } @@ -475,6 +503,7 @@ impl MockWarmBackend { /// Number of `get` calls recorded — useful to assert restore reads hit the /// local copy rather than the remote tier. + #[cfg(feature = "test-util")] pub async fn get_count(&self) -> usize { self.inner .op_log @@ -486,6 +515,7 @@ impl MockWarmBackend { } /// Number of `put` calls recorded. + #[cfg(feature = "test-util")] pub async fn put_count(&self) -> usize { self.inner .op_log @@ -499,6 +529,7 @@ impl MockWarmBackend { // ---- storage inspection --------------------------------------------- /// Whether the backend currently stores `object`. + #[cfg(feature = "test-util")] pub async fn contains(&self, object: &str) -> bool { self.inner.objects.lock().await.contains_key(object) } @@ -509,11 +540,13 @@ impl MockWarmBackend { } /// A clone of the stored object, if present. + #[cfg(feature = "test-util")] pub async fn stored(&self, object: &str) -> Option { self.inner.objects.lock().await.get(object).cloned() } /// A clone of the raw bytes stored for `object`, if present. + #[cfg(feature = "test-util")] pub async fn bytes(&self, object: &str) -> Option> { self.inner.objects.lock().await.get(object).map(|o| o.bytes.clone()) } @@ -538,6 +571,7 @@ impl MockWarmBackend { /// Poll until `object` is absent from the backend, or `timeout` elapses. /// Returns `true` if the object disappeared within the budget. + #[cfg(feature = "test-util")] pub async fn wait_for_remote_absence(&self, object: &str, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; loop { @@ -553,6 +587,7 @@ impl MockWarmBackend { /// Poll until the backend holds exactly `expected` objects, or `timeout` /// elapses. Returns `true` if the count was reached within the budget. + #[cfg(feature = "test-util")] pub async fn wait_for_object_count(&self, expected: usize, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; loop { @@ -847,6 +882,7 @@ pub async fn register_mock_tier_backend(handle: &Arc>, tie /// The transition-state tuple read from an on-disk `xl.meta`, plus the object's /// free-version count. #[derive(Clone, Debug, PartialEq, Eq)] +#[cfg(feature = "test-util")] pub struct TransitionMeta { /// `transition_status` (e.g. `"complete"`), empty when not transitioned. pub status: String, @@ -860,6 +896,7 @@ pub struct TransitionMeta { pub free_version_count: usize, } +#[cfg(feature = "test-util")] async fn open_disk(disk_path: &Path) -> Option { // `LocalDisk::new` rejects an endpoint whose (set_idx, disk_idx) disagrees // with the position recorded in the disk's own format.json, so derive the @@ -890,6 +927,7 @@ async fn open_disk(disk_path: &Path) -> Option { /// The free-version metadata removal lands asynchronously after the remote /// object disappears, so callers typically poll via /// [`wait_for_free_version_absence`] instead of asserting a single read. +#[cfg(feature = "test-util")] pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize { let Some(disk) = open_disk(disk_path).await else { return 0; @@ -914,6 +952,7 @@ pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> /// fields are taken from the newest version that carries a transition record; /// if no version is transitioned, they are taken from the current version (and /// will be empty). +#[cfg(feature = "test-util")] pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) -> Option { let disk = open_disk(disk_path).await?; let data = disk @@ -947,6 +986,7 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) /// disk is missing the object or disagrees — this is the shard-consistency /// check required by ilm-6 (the `(status, tier, remote key, remote version id)` /// four-tuple plus free-version count must match across all erasure shards). +#[cfg(feature = "test-util")] pub async fn assert_transition_meta_consistent>(disk_paths: &[P], bucket: &str, object: &str) -> TransitionMeta { assert!(!disk_paths.is_empty(), "assert_transition_meta_consistent needs at least one disk"); @@ -972,6 +1012,7 @@ pub async fn assert_transition_meta_consistent>(disk_paths: &[P], /// Poll until `object` retains no free versions on `disk_path`, or `timeout` /// elapses. Returns `true` if the free versions drained within the budget. +#[cfg(feature = "test-util")] pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, object: &str, timeout: Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; loop { @@ -1040,6 +1081,44 @@ mod tests { ); } + #[tokio::test] + async fn mock_metadata_survives_put_and_external_delete_is_distinct() { + let backend = MockWarmBackend::new(); + let metadata = HashMap::from([ + ("content-type".to_string(), "text/plain".to_string()), + ("project".to_string(), "archive".to_string()), + ]); + let version = backend + .put_with_meta("object", ReaderImpl::Body(Bytes::from_static(b"body")), 4, metadata.clone()) + .await + .expect("mock PUT should preserve remote metadata"); + assert_eq!(backend.metadata("object").await, Some(metadata)); + assert_eq!( + backend + .probe_transition_candidate_state("object") + .await + .expect("probe stored object"), + TransitionCandidateProbe::VersionedPresent(version) + ); + + backend.external_remove("object").await; + assert_eq!(backend.metadata("object").await, None); + assert_eq!( + backend + .probe_transition_candidate_state("object") + .await + .expect("probe removed object"), + TransitionCandidateProbe::Missing + ); + let operations = backend.op_log().await; + assert!( + operations + .iter() + .any(|op| matches!(op, MockWarmOp::ExternalRemove { object } if object == "object")) + ); + assert!(!operations.iter().any(|op| matches!(op, MockWarmOp::Remove { .. }))); + } + #[tokio::test] async fn mock_probe_preserves_fault_fail_closed_behavior() { let backend = MockWarmBackend::new(); diff --git a/crates/ecstore/src/services/tier/tier.rs b/crates/ecstore/src/services/tier/tier.rs index 5bb35f956..00117a907 100644 --- a/crates/ecstore/src/services/tier/tier.rs +++ b/crates/ecstore/src/services/tier/tier.rs @@ -143,11 +143,12 @@ struct TierDriverBuildBarrier { static TIER_DRIVER_BUILD_BARRIER: LazyLock>>> = LazyLock::new(|| Mutex::new(None)); #[cfg(test)] -type TierDriverTestFactory = Arc std::result::Result + Send + Sync + 'static>; +pub(crate) type TierDriverTestFactory = + Arc std::result::Result + Send + Sync + 'static>; #[cfg(test)] tokio::task_local! { - static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory; + pub(crate) static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory; } #[cfg(test)] @@ -1377,27 +1378,42 @@ async fn ensure_no_authoritative_persisted_references( where S: TierReferenceProofStore, { - ensure_no_authoritative_persisted_references_with(api.clone(), TIER_DELETE_JOURNAL_PREFIX, |_object, data| { - let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?; - Ok(( - journal.tier_name.clone(), - tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets), - )) - }) + ensure_no_authoritative_persisted_references_with( + api.clone(), + TIER_DELETE_JOURNAL_PREFIX, + "tier-delete journal", + |_object, data| { + let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?; + Ok(( + journal.tier_name.clone(), + tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets), + )) + }, + ) .await?; - ensure_no_authoritative_persisted_references_with(api, TRANSITION_TRANSACTION_RECORD_PREFIX, |object, data| { - let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?; - Ok(( - transaction.tier_name.clone(), - tier_persisted_reference_blocks_any_target(&transaction.tier_name, Some(transaction.backend_fingerprint), targets), - )) - }) + ensure_no_authoritative_persisted_references_with( + api, + TRANSITION_TRANSACTION_RECORD_PREFIX, + "transition transaction", + |object, data| { + let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?; + Ok(( + transaction.tier_name.clone(), + tier_persisted_reference_blocks_any_target( + &transaction.tier_name, + Some(transaction.backend_fingerprint), + targets, + ), + )) + }, + ) .await } async fn ensure_no_authoritative_persisted_references_with( api: Arc, prefix: &str, + reference_kind: &str, blocks_target: F, ) -> std::result::Result<(), AdminError> where @@ -1426,7 +1442,7 @@ where .map_err(tier_reference_proof_admin_error)?; let (tier_name, blocks) = blocks_target(&object.name, &data).map_err(tier_reference_proof_admin_error)?; if blocks { - return Err(tier_reference_proof_persisted_in_use_error(&tier_name, &object.name)); + return Err(tier_reference_proof_persisted_in_use_error(&tier_name, reference_kind, &object.name)); } } if !page.is_truncated { @@ -1488,16 +1504,21 @@ fn tier_persisted_reference_blocks_target( fn tier_reference_proof_in_use_error(tier_name: &str, object: &ObjectInfo) -> AdminError { let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + let reference_kind = if object.transitioned_object.free_version { + "free-version ownership" + } else { + "transitioned-object" + }; err.message = format!( - "Remote tier {tier_name} still has object references, for example {}/{}", + "Remote tier {tier_name} still has a {reference_kind} reference, for example {}/{}", object.bucket, object.name ); err } -fn tier_reference_proof_persisted_in_use_error(tier_name: &str, object: &str) -> AdminError { +fn tier_reference_proof_persisted_in_use_error(tier_name: &str, reference_kind: &str, object: &str) -> AdminError { let mut err = ERR_TIER_BACKEND_IN_USE.clone(); - err.message = format!("Remote tier {tier_name} still has a persisted reference, for example {object}"); + err.message = format!("Remote tier {tier_name} still has a {reference_kind} reference, for example {object}"); err } @@ -3685,13 +3706,17 @@ impl TierConfigMgr { let manager = handle.read().await; let runtime = tier_driver_runtime(handle, &manager); let runtime = lock_unpoisoned(&runtime); - if !runtime + let prepared = runtime + .prepared_mutation_blocks + .values() + .any(|blocked_mutation_id| *blocked_mutation_id == mutation_id); + let committed = runtime .committed_mutation_blocks .values() - .any(|mutation_ids| mutation_ids.contains(&mutation_id)) - { + .any(|mutation_ids| mutation_ids.contains(&mutation_id)); + if !prepared && !committed { let mut err = ERR_TIER_INVALID_CONFIG.clone(); - err.message = "Remote tier committed mutation fence was not installed".to_string(); + err.message = "Remote tier mutation fence was not installed".to_string(); return Err(err); } Ok(MutationBlockAllowance { @@ -3898,14 +3923,6 @@ impl TierConfigMgr { Self::begin_tier_transition_with_destinations(handle, manager, changed, replaced_destinations, mutation_block_allowance) } - fn begin_tier_transition( - handle: &Arc>, - manager: &mut Self, - changed: HashSet, - ) -> std::result::Result { - Self::begin_tier_transition_with_destinations(handle, manager, changed, HashMap::new(), None) - } - fn begin_tier_transition_with_destinations( handle: &Arc>, manager: &mut Self, @@ -4176,7 +4193,7 @@ impl TierConfigMgr { let mut config_lock = config_lock; let coordinated_config_update = config_lock.is_some(); let mut update = Some(update); - let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, mut transition) = + let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, target_tiers) = match mutation { TierCandidateMutation::Prevalidated(prepared) => { if version != prepared.version { @@ -4192,9 +4209,8 @@ impl TierConfigMgr { ))); } candidate = prepared.candidate; - let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; - let mut transition = { - let mut manager = handle.write().await; + let target_tiers = { + let manager = handle.read().await; let mut target_tiers = changed_tier_names(&manager, &candidate); if let Some(tier_name) = prepared.explicit_tier_name.as_ref() && (manager.tiers.contains_key(tier_name) @@ -4203,8 +4219,7 @@ impl TierConfigMgr { { target_tiers.insert(tier_name.clone()); } - Self::begin_tier_transition(&handle, &mut manager, target_tiers) - .map_err(TierConfigUpdateError::Publish)? + target_tiers }; ( prepared.kind, @@ -4212,7 +4227,7 @@ impl TierConfigMgr { prepared.force, prepared.current, prepared.driver_tier, - transition, + target_tiers, ) } mutation => { @@ -4237,11 +4252,9 @@ impl TierConfigMgr { last_refreshed_at: candidate.last_refreshed_at, }; let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; - let mut transition = { - let mut manager = handle.write().await; - let target_tiers = mutation.target_tiers(&manager, &candidate); - Self::begin_tier_transition(&handle, &mut manager, target_tiers) - .map_err(TierConfigUpdateError::Publish)? + let target_tiers = { + let manager = handle.read().await; + mutation.target_tiers(&manager, &candidate) }; let driver_tier = apply_tier_candidate_mutation(mutation, &mut candidate, validation_deadline) .await @@ -4252,7 +4265,7 @@ impl TierConfigMgr { mutation_force, current_for_targets, driver_tier, - transition, + target_tiers, ) } }; @@ -4275,28 +4288,83 @@ impl TierConfigMgr { save_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref()) .await .map_err(TierConfigUpdateError::Save)?; + let mut blocked_target_tiers = target_tiers.clone(); if let Some(intent) = coordinator_intent.as_ref() { - TierConfigMgr::apply_prepared_mutation_intent_block(&handle, intent) + blocked_target_tiers.extend(intent.affected_targets.iter().map(|target| target.tier_name.clone())); + } + if let Some(intent) = coordinator_intent.as_ref() { + // `target_tiers` may include a stale local-only manager + // entry that is absent from the persisted proof + // snapshot. Fence that local transition under the same + // mutation ID as well; recovery may discard this + // process-local superset, which advances the revision + // and makes the deferred transition fail closed. + TierConfigMgr::apply_prepared_mutation_intent_block_for_tiers(&handle, intent, &blocked_target_tiers) .await .map_err(TierConfigUpdateError::Publish)?; } + let prepared_mutation_block_allowance = match coordinator_intent.as_ref() { + Some(intent) => Some( + TierConfigMgr::mutation_block_allowance_for(&handle, intent.mutation_id) + .await + .map_err(TierConfigUpdateError::Publish)?, + ), + None => None, + }; + // A durable coordinator intent supplies the admission + // fence that lets us defer generation revocation. Keep the + // original early transition for no-intent paths (for + // example, reconciling a stale local manager to an + // idempotently removed persisted tier), where there is no + // Prepared record capable of blocking a new lease. + let (mut transition, deferred_target_tiers) = if coordinator_intent.is_some() { + (None, Some(target_tiers)) + } else { + let transition = { + let mut manager = handle.write().await; + Self::begin_tier_transition_with_destinations( + &handle, + &mut manager, + target_tiers, + HashMap::new(), + None, + ) + .map_err(TierConfigUpdateError::Publish)? + }; + (Some(transition), None) + }; if coordinated_config_update { drop(update.take()); drop(config_lock.take()); } - let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; - if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await { - if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), Vec::new()).await { + // The durable Prepared block closes admission before the + // zero-reference proof, but deliberately leaves already + // issued generations current. In particular, an exact + // free-version cleanup that has completed remote DELETE + // must still be able to remove its local ownership marker; + // revoking its generation here would strand that marker + // and make this mutation reject its own interrupted work. + if let Some(intent) = coordinator_intent.as_ref() + && let Err(drain_error) = + TierConfigMgr::wait_for_blocked_tier_operation_leases_for_tiers(&handle, &blocked_target_tiers).await + { + if !abort_prepared_tier_mutation(&handle, api.clone(), Some(intent), Vec::new()).await { warn!( event = "tier_mutation_abort", component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_TIER, result = "prepared_intent_retained", - coordinator_intent = coordinator_intent.is_some(), + mutation_id = %intent.mutation_id, "tier mutation lease drain failed and abort was incomplete" ); } return Err(TierConfigUpdateError::Publish(drain_error)); + } else if let Some(transition) = transition.as_ref() { + let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; + transition + .wait_for_active_leases_until(drain_deadline) + .await + .map_err(TierConfigUpdateError::Publish)?; } let prepared_peers = if let Some(intent) = coordinator_intent.as_ref() { let peers = match remote_tier_mutation_peers().await { @@ -4363,6 +4431,71 @@ impl TierConfigMgr { } return Err(TierConfigUpdateError::Publish(proof_error)); } + // No affected-tier lease can start after Prepared, and the + // existing set was drained above. It is now safe to revoke + // the generation for publication without invalidating a + // cleanup between its remote and local commit boundaries. + if transition.is_none() { + let target_tiers = deferred_target_tiers.ok_or_else(|| { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier mutation lost its deferred transition targets".to_string(); + TierConfigUpdateError::Publish(err) + })?; + let mut manager = handle.write().await; + transition = Some( + match Self::begin_tier_transition_with_destinations( + &handle, + &mut manager, + target_tiers, + HashMap::new(), + prepared_mutation_block_allowance.as_ref(), + ) { + Ok(transition) => transition, + Err(transition_error) => { + drop(manager); + if !abort_prepared_tier_mutation( + &handle, + api.clone(), + coordinator_intent.as_ref(), + prepared_peers, + ) + .await + { + warn!( + event = "tier_mutation_abort", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_TIER, + result = "prepared_intent_retained", + coordinator_intent = coordinator_intent.is_some(), + "tier mutation publish transition failed and abort was incomplete" + ); + } + return Err(TierConfigUpdateError::Publish(transition_error)); + } + }, + ); + } + let mut transition = transition.ok_or_else(|| { + let mut err = ERR_TIER_INVALID_CONFIG.clone(); + err.message = "Remote tier mutation lost its publish transition".to_string(); + TierConfigUpdateError::Publish(err) + })?; + let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT; + if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await { + drop(transition); + if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), prepared_peers).await + { + warn!( + event = "tier_mutation_abort", + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_TIER, + result = "prepared_intent_retained", + coordinator_intent = coordinator_intent.is_some(), + "tier mutation publish drain failed and abort was incomplete" + ); + } + return Err(TierConfigUpdateError::Publish(drain_error)); + } let candidate_digest = tier_config_candidate_digest(&candidate).map_err(TierConfigUpdateError::Save)?; if coordinated_config_update { config_lock = match Self::acquire_tier_config_write_lock(api.clone()).await { @@ -5414,11 +5547,40 @@ impl TierConfigMgr { handle: &Arc>, intent: &TierMutationIntent, ) -> std::result::Result<(), AdminError> { + let target_tiers = intent + .affected_targets + .iter() + .map(|target| target.tier_name.clone()) + .collect(); + Self::apply_prepared_mutation_intent_block_for_tiers(handle, intent, &target_tiers).await + } + + async fn apply_prepared_mutation_intent_block_for_tiers( + handle: &Arc>, + intent: &TierMutationIntent, + target_tiers: &HashSet, + ) -> std::result::Result<(), AdminError> { + if intent.state != TierMutationIntentState::Prepared { + return Ok(()); + } let manager = handle.read().await; let runtime = tier_driver_runtime(handle, &manager); let mut runtime = lock_unpoisoned(&runtime); let mut prepared_mutation_blocks = runtime.prepared_mutation_blocks.clone(); Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, intent)?; + for tier_name in target_tiers { + match prepared_mutation_blocks.entry(tier_name.clone()) { + Entry::Vacant(entry) => { + entry.insert(intent.mutation_id); + } + Entry::Occupied(entry) if *entry.get() == intent.mutation_id => {} + Entry::Occupied(_) => { + let mut err = ERR_TIER_BACKEND_IN_USE.clone(); + err.message = format!("Remote tier {tier_name} already has another prepared mutation"); + return Err(err); + } + } + } if prepared_mutation_blocks == runtime.prepared_mutation_blocks { return Ok(()); } @@ -5434,6 +5596,18 @@ impl TierConfigMgr { pub(crate) async fn wait_for_blocked_tier_operation_leases( handle: &Arc>, intent: &TierMutationIntent, + ) -> std::result::Result<(), AdminError> { + let target_tiers = intent + .affected_targets + .iter() + .map(|target| target.tier_name.clone()) + .collect(); + Self::wait_for_blocked_tier_operation_leases_for_tiers(handle, &target_tiers).await + } + + async fn wait_for_blocked_tier_operation_leases_for_tiers( + handle: &Arc>, + target_tiers: &HashSet, ) -> std::result::Result<(), AdminError> { let generations = { let manager = handle.read().await; @@ -5441,10 +5615,9 @@ impl TierConfigMgr { return Ok(()); }; let runtime = lock_unpoisoned(&runtime); - intent - .affected_targets + target_tiers .iter() - .filter_map(|target| runtime.generations.get(&target.tier_name).cloned()) + .filter_map(|tier_name| runtime.generations.get(tier_name).cloned()) .collect::>() }; let drain = async { @@ -14414,6 +14587,13 @@ mod tests { .push(object); } + fn remove_listed_version(&self, bucket: &str, object: &str) { + self.listed_versions + .lock() + .expect("tier reference fixture should not poison") + .retain(|version| version.bucket != bucket || version.name != object); + } + fn add_lifecycle_config(&self, bucket: &str, config: BucketLifecycleConfiguration) { self.lifecycle_configs .lock() @@ -15682,6 +15862,152 @@ mod tests { ); } + #[tokio::test] + async fn tier_remove_prepared_fence_allows_inflight_free_version_cleanup_to_finish() { + let store = Arc::new(CasConfigStore::default()); + let tier = build_rustfs_tier("COLD-A"); + let identity = tier_backend_identity(&tier).expect("test tier identity should encode"); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), tier.clone_with_credentials()); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("free-version drain fixture should persist"); + + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + guard.tiers.insert("COLD-A".to_string(), tier); + guard.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B")); + guard + .replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("cleanup"))) + .expect("cleanup driver generation should install"); + guard + .replace_driver("COLD-B", Box::new(LeaseTestBackend::ready("stale-local"))) + .expect("stale local driver generation should install"); + } + let cleanup_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("in-flight cleanup lease should be available"); + let stale_local_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B") + .await + .expect("stale local tier lease should be available"); + let mut free_version = transitioned_tier_object("photos", "2026/free-version.jpg", "COLD-A", Some(identity)); + free_version.transitioned_object.status = "pending".to_string(); + free_version.transitioned_object.free_version = true; + store.add_listed_version(free_version); + + let remove_manager = manager.clone(); + let remove_store = store.clone(); + let remove = tokio::spawn(async move { + TIER_MUTATION_TEST_PEERS + .scope( + Vec::new(), + TierConfigMgr::remove_and_save_with(&remove_manager, remove_store, "COLD-A", true), + ) + .await + }); + + tokio::time::timeout(Duration::from_secs(1), async { + loop { + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + let prepared = { + let runtime = lock_unpoisoned(&runtime); + runtime.prepared_mutation_blocks.contains_key("COLD-A") + && runtime.prepared_mutation_blocks.contains_key("COLD-B") + }; + if prepared { + break; + } + drop(guard); + tokio::task::yield_now().await; + } + }) + .await + .expect("tier remove should install its durable prepared fence"); + + assert!( + cleanup_lease.is_current(&manager).await, + "the prepared fence must let the already leased cleanup finish its exact local marker deletion" + ); + assert!( + stale_local_lease.is_current(&manager).await, + "the local superset fence must also let an already leased stale-manager operation finish" + ); + let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await { + Ok(_) => panic!("the prepared fence must reject new tier operations"), + Err(err) => err, + }; + assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked)); + let stale_blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-B").await { + Ok(_) => panic!("the local superset fence must reject new stale-manager operations"), + Err(err) => err, + }; + assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&stale_blocked)); + + store.remove_listed_version("photos", "2026/free-version.jpg"); + drop(cleanup_lease); + drop(stale_local_lease); + + tokio::time::timeout(Duration::from_secs(5), remove) + .await + .expect("tier remove should finish after both in-flight operations release their leases") + .expect("tier remove task should join") + .expect("tier remove should pass once the in-flight cleanup removes its marker"); + assert!(!manager.read().await.tiers.contains_key("COLD-A")); + assert!(!manager.read().await.tiers.contains_key("COLD-B")); + assert!( + !load_tier_config_for_update(store) + .await + .expect("removed tier config should reload") + .0 + .tiers + .contains_key("COLD-A") + ); + } + + #[tokio::test] + async fn no_intent_stale_manager_removal_keeps_early_generation_drain() { + let store = Arc::new(CasConfigStore::default()); + empty_mgr() + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("empty persisted tier config should exist"); + let manager = TierConfigMgr::new(); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("stale")); + } + let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A") + .await + .expect("stale manager lease should be available"); + + let remove_manager = manager.clone(); + let remove_store = store.clone(); + let remove = + tokio::spawn(async move { TierConfigMgr::remove_and_save_with(&remove_manager, remove_store, "COLD-A", true).await }); + tokio::time::timeout(Duration::from_secs(1), async { + while old.is_current(&manager).await { + tokio::task::yield_now().await; + } + }) + .await + .expect("no-intent stale-manager reconciliation should revoke before its proof"); + let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await { + Ok(_) => panic!("stale-manager reconciliation must not admit a new operation"), + Err(err) => err, + }; + assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked)); + + drop(old); + remove + .await + .expect("stale-manager removal task should join") + .expect("stale-manager removal should converge to the persisted empty config"); + assert!(!manager.read().await.tiers.contains_key("COLD-A")); + } + async fn assert_lifecycle_only_reference_obeys_force(clear: bool, force: bool) { let store = Arc::new(CasConfigStore::default()); let tier = build_rustfs_tier("COLD-A"); @@ -16585,12 +16911,23 @@ mod tests { .await }); tokio::time::timeout(Duration::from_secs(1), async { - while old.is_current(&manager).await { + loop { + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + let prepared = lock_unpoisoned(&runtime).prepared_mutation_blocks.contains_key("COLD-A"); + if prepared { + break; + } + drop(guard); tokio::task::yield_now().await; } }) .await - .expect("owned update should revoke before caller cancellation"); + .expect("owned update should install its prepared fence before caller cancellation"); + assert!( + old.is_current(&manager).await, + "an already leased operation must remain current until it can finish" + ); caller.abort(); drop(old); @@ -16635,12 +16972,23 @@ mod tests { .await }); tokio::time::timeout(Duration::from_secs(1), async { - while old.is_current(&manager).await { + loop { + let guard = manager.read().await; + let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered"); + let prepared = lock_unpoisoned(&runtime).prepared_mutation_blocks.contains_key("COLD-A"); + if prepared { + break; + } + drop(guard); tokio::task::yield_now().await; } }) .await - .expect("owned update should revoke before caller cancellation"); + .expect("owned update should install its prepared fence before caller cancellation"); + assert!( + old.is_current(&manager).await, + "the prepared fence must not invalidate an already leased operation" + ); caller.abort(); let config_file = tier_config_lock_path(); @@ -17026,6 +17374,71 @@ mod tests { assert!(current.tiers.contains_key("COLD-B")); } + #[tokio::test] + #[serial_test::serial] + async fn reference_proof_rejects_a_changed_prepared_fence_revision_before_publish() { + let manager = TierConfigMgr::new(); + let store = Arc::new(CasConfigStore::default()); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A")); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("prepared-fence revision fixture should persist"); + { + let mut guard = manager.write().await; + install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old")); + } + + let barrier = tier_reference_proof_test_barrier(); + let scoped_barrier = barrier.clone(); + let update_manager = manager.clone(); + let update_store = store.clone(); + let update = tokio::spawn(async move { + TIER_REFERENCE_PROOF_TEST_BARRIER + .scope( + scoped_barrier, + TIER_MUTATION_TEST_PEERS.scope( + Vec::new(), + TierConfigMgr::update_candidate_with_config_lock( + &update_manager, + update_store, + TierCandidateMutation::Remove("COLD-A".to_string(), true), + ), + ), + ) + .await + }); + barrier.arrived.notified().await; + + let unrelated = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(0x2237)); + TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &unrelated) + .await + .expect("an unrelated prepared fence should advance the runtime revision"); + barrier.release.add_permits(1); + + let err = update + .await + .expect("tier update task should join") + .expect_err("a reference proof cannot authorize publication across a fence revision change"); + let TierConfigUpdateError::Publish(err) = err else { + panic!("the stale prepared-fence allowance should fail publication: {err:?}"); + }; + assert!(err.message.contains("changed before replacement"), "{err}"); + assert!(manager.read().await.tiers.contains_key("COLD-A")); + assert!( + load_tier_config_for_update(store) + .await + .expect("rejected tier config should remain readable") + .0 + .tiers + .contains_key("COLD-A") + ); + TierConfigMgr::clear_prepared_mutation_intent_block(&manager, unrelated.mutation_id) + .await + .expect("unrelated test fence should clear"); + } + #[tokio::test] #[serial_test::serial] async fn caller_cancellation_after_durable_prepare_does_not_hide_the_mutation() { diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 14da6fc97..4dc17ad34 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -111,10 +111,11 @@ use crate::disk::{ use crate::erasure::coding::BitrotReader; use crate::io_support::bitrot::ShardReader; use crate::io_support::bitrot::{ - BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params, - create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle, - object_mmap_read_max_length, + BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_from_bytes_with_stage_metrics, + create_deferred_bitrot_reader_with_stripe_handle, }; +#[cfg(unix)] +use crate::io_support::bitrot::{adjust_shard_read_params, object_mmap_read_max_length}; use crate::set_disk::runtime_sources; use crate::set_disk::shard_source::ShardReadCost; use crate::storage_api_contracts::object::ObjectOperations; @@ -3662,22 +3663,22 @@ async fn rollback_failed_rename( let object = object.to_string(); let disk_namespace_commit_guard = namespace_commit_guard.clone(); let task = tokio::spawn(async move { - let _namespace_commit_guard = disk_namespace_commit_guard; + let _namespace_commit_guard = disk_namespace_commit_guard.clone(); #[allow(clippy::let_unit_value)] let _task_guard = SetDisks::rename_fanout_task_guard(&object); SetDisks::rename_fanout_barrier(&object, disk_index, rename_fanout_barrier_phase::ROLLBACK).await; #[cfg(test)] rollback_fault_injection::before_undo(&object, disk_index)?; - disk.delete_version( + disk.undo_write_with_namespace_owner( &bucket, &object, fi, - false, DeleteOptions { undo_write: true, old_data_dir: rollback_dir, ..Default::default() }, + disk_namespace_commit_guard.map(|owner| owner as Arc), ) .await }); @@ -4237,7 +4238,7 @@ impl SetDisks { let successful_rename_completion_rank = successful_rename_completion_rank.clone(); let namespace_commit_guard = namespace_commit_guard.clone(); tasks.spawn(async move { - let _namespace_commit_guard = namespace_commit_guard; + let _namespace_commit_guard = namespace_commit_guard.clone(); let mut dispatch_state = RenameDispatchState::NotDispatched; let result = std::panic::AssertUnwindSafe(async { #[allow(clippy::let_unit_value)] @@ -4272,7 +4273,13 @@ impl SetDisks { &file_info, &dst_bucket, &dst_object, - scanner_publication_lease_token, + crate::disk::RenameDataGuards { + scanner_publication_lease_token, + namespace_owner: namespace_commit_guard + .clone() + .map(|owner| owner as Arc), + ..Default::default() + }, ) .await; let rejected_before_publication = observed.rejected_before_publication(); @@ -4601,7 +4608,7 @@ impl SetDisks { // Keep the storage-owned movement permit attached to the actual // fan-out owner, even if the caller future is cancelled. let _fanout_publication_scope = fanout_publication_scope; - let _namespace_commit_guard = fanout_namespace_commit_guard; + let _namespace_commit_guard = fanout_namespace_commit_guard.clone(); let successful_rename_completion_rank = rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0))); let futures = fanout_disks @@ -4616,6 +4623,7 @@ impl SetDisks { let dst_bucket = fanout_dst_bucket.clone(); let successful_rename_completion_rank = successful_rename_completion_rank.clone(); let publication_scope = scanner_publication_commit_scope.clone(); + let namespace_commit_guard = fanout_namespace_commit_guard.clone(); async move { let mut dispatch_state = RenameDispatchState::NotDispatched; @@ -4668,7 +4676,13 @@ impl SetDisks { file_info, &dst_bucket, &dst_object, - scanner_publication_lease_token, + crate::disk::RenameDataGuards { + scanner_publication_lease_token, + namespace_owner: namespace_commit_guard + .clone() + .map(|owner| owner as Arc), + ..Default::default() + }, ) .await; let rejected_before_publication = observed.rejected_before_publication(); @@ -10859,6 +10873,358 @@ mod tests { .await; } + #[cfg(not(windows))] + async fn assert_namespace_owner_survives_physical_publication_timeout(allow_early_ack: bool) { + use crate::disk::os; + use futures::FutureExt; + + temp_env::async_with_vars( + [ + (rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")), + (ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")), + ], + async { + const DISKS: usize = 4; + let bucket = "namespace-physical-tail"; + let object = "inline-overwrite"; + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let mut old = metadata_test_fileinfo(object); + old.mod_time = Some(OffsetDateTime::now_utc()); + old.size = 15; + old.parts.clear(); + old.add_object_part(1, "old-etag".to_string(), 15, None, 15, None, None); + old.data = Some(Bytes::from_static(b"old-inline-body")); + old.set_inline_data(); + old.metadata.insert("etag".to_string(), "old-etag".to_string()); + let mut infos = rename_commit_fileinfos(object, DISKS, "new-etag"); + let mut hooks = Vec::new(); + let mut entered = Vec::new(); + let mut releases = Vec::new(); + let mut publication_paths = Vec::new(); + for (disk, info) in disks.iter().flatten().zip(&mut infos) { + disk.write_metadata(bucket, bucket, object, old.clone()) + .await + .expect("the old inline version must be readable before overwrite"); + info.size = 11; + info.parts.clear(); + info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None); + let crate::disk::Disk::Local(local) = disk.as_ref() else { + panic!("physical publication fixture requires local disks"); + }; + // Linux IO paths use a mount FD, which is also the namespace lock key. + let destination = local + .get_disk() + .get_object_path_for_io(bucket, object) + .expect("the publication path must resolve through the disk's mount lease"); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + hooks.push(os::prepared_publication_test_hooks::install( + &destination.join(STORAGE_FORMAT_FILE), + move || { + let _ = entered_tx.send(()); + // Sender drop also releases the syscall when an earlier assertion fails. + let _ = release_rx.recv(); + }, + )); + entered.push(entered_rx); + releases.push(release_tx); + publication_paths.push(destination); + } + let namespace_owner = ctx.begin_namespace_commit(); + let namespace_probe = Arc::downgrade(&namespace_owner); + let receipt = RenameRollbackReceipt::default(); + let mut rename = Box::pin(SetDisks::rename_data_owned_with_fence( + &disks, + (RUSTFS_META_TMP_BUCKET, "source"), + infos, + (bucket, object), + allow_early_ack, + RenameDataFenceOptions::new(3, None) + .with_rollback_receipt(receipt.clone()) + .with_namespace_commit_guard(Some(namespace_owner)), + )); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::select! { + signals = join_all(entered) => { + assert!(signals.into_iter().all(|signal| signal.is_ok()), "all physical publishers must enter"); + } + _ = rename.as_mut() => panic!("rename must not finish before physical publication is paused"), + } + }) + .await + .expect("all four prepared metadata renames must reach their blocking syscall"); + assert!(ctx.namespace_commits_pending()); + assert_eq!(ctx.namespace_commit_generation(), 1); + + // Every wrapper timer exists before advancing; the physical closures stay blocked. + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(61)).await; + tokio::time::resume(); + let result = tokio::time::timeout(Duration::from_secs(5), rename) + .await + .expect("ordinary disk timeout must not wait for the physical rename"); + assert!(result.is_err(), "four timed-out disks cannot satisfy write quorum"); + let report = receipt.0.get().expect("failed fanout must finish rollback accounting"); + assert_eq!(report.disks.len(), DISKS); + assert!( + report + .disks + .iter() + .all(|disk| matches!(disk.outcome, RenameRollbackOutcome::Indeterminate(DiskError::Timeout))) + ); + let pending_before_release = ctx.namespace_commits_pending(); + let owner_alive_before_release = namespace_probe.upgrade().is_some(); + let old_snapshot_generation = ctx.namespace_commit_generation(); + for (disk, destination) in disks.iter().flatten().zip(&publication_paths) { + let root = disk.path(); + assert!( + os::acquire_rename_data_mutation_lease(&root, bucket, destination) + .now_or_never() + .is_none(), + "the physical publication must still own object serialization after the async timeout" + ); + assert!( + root.join(RUSTFS_META_TMP_BUCKET) + .join("source") + .join(STORAGE_FORMAT_FILE) + .exists() + ); + let stored = disk + .read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("a scanner can still read the complete old metadata while publication is paused"); + assert_eq!(stored.size, 15); + assert_eq!(stored.data.as_deref(), Some(b"old-inline-body".as_slice())); + } + assert_eq!(ctx.namespace_commit_generation(), old_snapshot_generation); + + // Drain real syscalls before checking the regression, including on the RED run. + drop(releases); + for (disk, destination) in disks.iter().flatten().zip(&publication_paths) { + let root = disk.path(); + let lease = tokio::time::timeout( + Duration::from_secs(5), + os::acquire_rename_data_mutation_lease(&root, bucket, destination), + ) + .await + .expect("released physical publishers must drain"); + drop(lease); + } + for dir in &dirs { + let reopened = reopen_local_disk(dir).await; + let stored = reopened + .read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("the detached prepared rename must actually publish after timeout"); + assert_eq!(stored.size, 11); + assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice())); + } + // The lease releases its locks before dropping the namespace owner, and the + // owner's `Drop` runs after its `Weak` probe stops upgrading, so wait for the + // pending counter itself instead of asserting it right after the drain. + tokio::time::timeout(Duration::from_secs(5), async { + while ctx.namespace_commits_pending() || namespace_probe.upgrade().is_some() { + tokio::task::yield_now().await; + } + }) + .await + .expect("released physical publishers must release namespace ownership"); + let generation_after_publication = ctx.namespace_commit_generation(); + assert!(!ctx.namespace_commits_pending()); + assert!(namespace_probe.upgrade().is_none()); + assert!(receipt.is_incomplete(), "late publication must not erase failed-write recovery evidence"); + assert!( + pending_before_release && owner_alive_before_release, + "physical publication outlived namespace accounting: early_ack={allow_early_ack}, \ + pending={pending_before_release}, owner_alive={owner_alive_before_release}, \ + old_snapshot_generation={old_snapshot_generation}, after_late_publication={generation_after_publication}" + ); + assert!( + generation_after_publication > old_snapshot_generation, + "physical completion must invalidate the scanner's old metadata snapshot" + ); + }, + ) + .await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_full_wait_timeout_keeps_namespace_owner_until_physical_publication() { + assert_namespace_owner_survives_physical_publication_timeout(false).await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn rename_early_ack_timeout_keeps_namespace_owner_until_physical_publication() { + assert_namespace_owner_survives_physical_publication_timeout(true).await; + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial(capacity_dirty_scope)] + async fn successful_rename_ack_keeps_physical_tail_owner_after_caller_cancellation() { + use crate::disk::os; + temp_env::async_with_vars( + [ + (rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60")), + (ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")), + ], + async { + let bucket = "physical-ack-tail"; + let object = "ack-object"; + let (dirs, disks) = call_counter_local_disks(bucket, 4).await; + prepare_rename_source_dirs(&dirs, &disks, "source").await; + let mut infos = rename_commit_fileinfos(object, 4, "new-etag"); + for info in &mut infos { + info.size = 11; + info.parts.clear(); + info.add_object_part(1, "new-etag".to_string(), 11, None, 11, None, None); + } + let disk = disks[3].as_ref().expect("tail disk"); + let crate::disk::Disk::Local(local) = disk.as_ref() else { + panic!("local fixture"); + }; + let destination = local.get_disk().get_object_path_for_io(bucket, object).expect("tail IO path"); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = os::prepared_publication_test_hooks::install(&destination.join(STORAGE_FORMAT_FILE), move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let owner = ctx.begin_namespace_commit(); + let owner_probe = Arc::downgrade(&owner); + let receipt = RenameRollbackReceipt::default(); + let caller_receipt = receipt.clone(); + let caller_disks = disks.clone(); + let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); + let caller = tokio::spawn(async move { + let commit = SetDisks::rename_data_owned_with_fence( + &caller_disks, + (RUSTFS_META_TMP_BUCKET, "source"), + infos, + (bucket, object), + true, + RenameDataFenceOptions::new(3, None) + .with_namespace_commit_guard(Some(owner)) + .with_rollback_receipt(caller_receipt), + ) + .await + .expect("three real disk publications must produce a successful ACK"); + assert!(ack_tx.send(commit).is_ok(), "deliver successful ACK"); + std::future::pending::<()>().await; + }); + let mut commit = tokio::time::timeout(Duration::from_secs(10), async { + entered_rx.await.expect("physical tail entry"); + ack_rx + .await + .expect("ACK must arrive while the fourth disk is physically paused") + }) + .await + .expect("successful quorum ACK must not wait for its physical tail"); + assert_eq!(commit.online_disks.iter().flatten().count(), 3); + assert!(!destination.join(STORAGE_FORMAT_FILE).exists(), "tail has not published at ACK"); + let tail_drain = commit.tail_drain.take().expect("early ACK transfers a real tail handle"); + drop(commit); + caller.abort(); + assert!(caller.await.expect_err("cancel caller after it delivered ACK").is_cancelled()); + tokio::time::pause(); + tokio::time::advance(Duration::from_secs(61)).await; + tokio::time::resume(); + let tail = tokio::time::timeout(Duration::from_secs(5), tail_drain) + .await + .expect("ordinary tail timeout stays bounded after ACK") + .expect("tail owner must not panic") + .expect("successful ACK keeps its convergence result"); + assert_eq!(tail.convergence, RenameConvergence::PartialCommit); + assert!(receipt.0.get().is_none(), "an acknowledged write must never enter rollback"); + let pending = ctx.namespace_commits_pending(); + let alive = owner_probe.upgrade().is_some(); + let generation = ctx.namespace_commit_generation(); + for disk in disks.iter().flatten().take(3) { + let stored = disk + .read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("all ACK voters keep the new object after caller cancellation"); + assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice())); + } + drop(release_tx); + let lease = tokio::time::timeout( + Duration::from_secs(5), + os::acquire_rename_data_mutation_lease(&disk.path(), bucket, &destination), + ) + .await + .expect("late physical tail drains"); + drop(lease); + tokio::time::timeout(Duration::from_secs(5), async { + while ctx.namespace_commits_pending() || owner_probe.upgrade().is_some() { + tokio::task::yield_now().await; + } + }) + .await + .expect("late physical tail must release namespace ownership"); + for dir in &dirs { + let stored = reopen_local_disk(dir) + .await + .read_version( + "", + bucket, + object, + "", + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("successful ACK remains committed on every disk after late publication"); + assert_eq!(stored.data.as_deref(), Some(b"inline-body".as_slice())); + } + assert!( + pending && alive, + "physical ACK tail must retain namespace ownership after the coordinator exits" + ); + assert!(!ctx.namespace_commits_pending()); + assert!(owner_probe.upgrade().is_none()); + assert!(ctx.namespace_commit_generation() > generation); + assert!(receipt.0.get().is_none(), "late publication cannot change success into rollback"); + }, + ) + .await; + } + #[tokio::test] #[serial_test::serial(capacity_dirty_scope)] async fn rename_rollback_incomplete_receipt_waits_for_undo_barrier() { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 2c076efea..179c4fd38 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -874,7 +874,7 @@ pub(crate) use ops::multipart::NewMultipartUploadCommitObservation; pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; #[cfg(test)] pub(crate) use ops::object::DeleteObjectCommitBarrier; -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; #[cfg(all(test, feature = "test-util"))] pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier; diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 3e09515d2..4fcb450bd 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -5785,7 +5785,7 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably( } async fn transition_cleanup_store(ctx: &Arc) -> Option> { - #[cfg(any(test, feature = "test-util"))] + #[cfg(feature = "test-util")] pause_transition_cleanup_store().await; transition_object_store(ctx).await @@ -6031,24 +6031,24 @@ async fn delete_transition_transaction_after_remote_cleanup( } } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] #[derive(Default)] struct TransitionCleanupStoreBarrierState { arrived: tokio::sync::Notify, release: tokio::sync::Notify, } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] /// One-shot test barrier placed before transition cleanup resolves its ECStore. pub(crate) struct TransitionCleanupStoreBarrier { state: Arc, } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] static TRANSITION_CLEANUP_STORE_BARRIER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] impl TransitionCleanupStoreBarrier { /// Install the process-local barrier for the next cleanup-store resolution. pub(crate) fn install() -> Self { @@ -6071,7 +6071,7 @@ impl TransitionCleanupStoreBarrier { } } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] impl Drop for TransitionCleanupStoreBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -6085,7 +6085,7 @@ impl Drop for TransitionCleanupStoreBarrier { } } -#[cfg(any(test, feature = "test-util"))] +#[cfg(feature = "test-util")] async fn pause_transition_cleanup_store() { let barrier = TRANSITION_CLEANUP_STORE_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -6159,7 +6159,7 @@ async fn pause_after_transition_upload_candidate_recorded() { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] struct TransitionUploadedCommitBarrierState { bucket: String, object: String, @@ -6167,17 +6167,17 @@ struct TransitionUploadedCommitBarrierState { release: tokio::sync::Notify, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) struct TransitionUploadedCommitBarrier { state: Arc, } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] static TRANSITION_UPLOADED_COMMIT_BARRIER: std::sync::OnceLock< std::sync::Mutex>>, > = std::sync::OnceLock::new(); -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl TransitionUploadedCommitBarrier { pub(crate) fn install(bucket: &str, object: &str) -> Self { let state = Arc::new(TransitionUploadedCommitBarrierState { @@ -6210,7 +6210,7 @@ impl TransitionUploadedCommitBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] impl Drop for TransitionUploadedCommitBarrier { fn drop(&mut self) { self.state.release.notify_one(); @@ -6224,7 +6224,7 @@ impl Drop for TransitionUploadedCommitBarrier { } } -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] async fn pause_after_transition_uploaded_persisted(bucket: &str, object: &str) { let barrier = TRANSITION_UPLOADED_COMMIT_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -9154,7 +9154,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { } upload_cleanup.update_cleanup_transaction(&transaction); - #[cfg(test)] + #[cfg(all(test, feature = "test-util"))] pause_after_transition_uploaded_persisted(bucket, object).await; let commit_opts = opts.as_commit_opts(); @@ -12672,6 +12672,65 @@ mod metadata_mutation_generation_tests { set_disks.invalidate_get_object_metadata_cache(bucket, object).await; } + #[tokio::test] + #[serial_test::serial(metadata_cache_invalidation_probe)] + async fn segment_observation_equal_size_mutations_retire_metadata_generation() { + let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; + let bucket = "segment-observation-bucket"; + let object = "hot/object"; + for disk in &disk_stores { + disk.make_volume(bucket).await.expect("create segment fixture bucket"); + } + let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"before").await; + let probe = MetadataCacheInvalidationProbe::install(bucket, object); + let mut replacement = PutObjReader::from_vec(b"after!".to_vec()); + set_disks + .put_object(bucket, object, &mut replacement, &ObjectOptions::default()) + .await + .expect("commit same-length replacement with normal owner locking"); + assert_eq!(probe.count(), 2, "same-length PUT must retire its metadata generation"); + assert_retired(&set_disks, &old_key).await; + drop(probe); + let after = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("read replacement metadata"); + assert_eq!(before.size, after.size); + assert_ne!(before.etag, after.etag, "equal size is not equal content"); + let mut reader = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read replacement body through the owner"); + let mut body = Vec::new(); + reader.stream.read_to_end(&mut body).await.expect("drain replacement body"); + assert_eq!(body, b"after!"); + drop(reader); + + let (before, old_key) = put_and_prime(&set_disks, bucket, object, b"after!").await; + let probe = MetadataCacheInvalidationProbe::install(bucket, object); + set_disks + .put_object_metadata( + bucket, + object, + &ObjectOptions { + eval_metadata: Some(HashMap::from([("x-amz-meta-segment".to_string(), "changed".to_string())])), + ..Default::default() + }, + ) + .await + .expect("commit metadata-only mutation with normal owner locking"); + assert_eq!(probe.count(), 4, "metadata-only mutation must retire both owner fences"); + assert_retired(&set_disks, &old_key).await; + let after = set_disks + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("read committed metadata-only mutation"); + assert_eq!(before.size, after.size); + assert_eq!(before.etag, after.etag); + assert!(!before.user_defined.contains_key("x-amz-meta-segment")); + assert_eq!(after.user_defined.get("x-amz-meta-segment").map(String::as_str), Some("changed")); + } + #[tokio::test] #[serial_test::serial(metadata_cache_invalidation_probe)] async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 764e48391..239cb20cc 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -825,6 +825,11 @@ mod tests { manual_transition_scope_record_object_name, manual_transition_task_object_name, manual_transition_worker_result_object_name, manual_transition_worker_result_task_key, }, + recovery_control::{ + IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, + IlmRecoveryProtocol, MAX_RECOVERY_ATTEMPTS, load_recovery_control, observe_recovery_source, + save_recovery_control_if_absent, + }, tier_delete_journal::{ DecommissionCheckpointTargetFailureHook, TIER_DELETE_DISPATCH_MANIFEST_PREFIX, TIER_DELETE_JOURNAL_PREFIX, TierDeleteChunkTestBarrier, TierDeleteChunkTestStage, TierDeleteDispatchBatchLimitGuard, @@ -844,12 +849,13 @@ mod tests { }, transition_transaction::{ TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError, - TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRemoteVersion, TransitionSourceIdentity, - TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit, TransitionTransactionState, - delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator, - inspect_transition_transaction_for_operator, load_transition_transaction_record, - recover_transition_transaction_records, recover_transition_transaction_records_at, - save_transition_transaction_record, save_transition_transaction_record_if_current, + TransitionOperatorProbe, TransitionRecoveryClaimBarrier, TransitionRecoveryTerminalBarrier, + TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, + TransitionTransactionInit, TransitionTransactionState, delete_transition_candidate_for_operator, + finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator, + load_transition_transaction_record, recover_transition_transaction_records, + recover_transition_transaction_records_at, save_transition_transaction_record, + save_transition_transaction_record_if_current, transition_recovery_control_id, transition_transaction_record_object_name, }, validate_durable_ilm_record, @@ -19239,6 +19245,105 @@ mod tests { assert!(!Arc::ptr_eq(&ctx_a, &ctx_b), "the regression requires two distinct instance contexts"); } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn transition_transaction_recovery_expires_abandoned_attempt_at_budget_bound() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (ctx, store, _shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "transition-transaction-expired-attempt-budget", + &[4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let transaction = TransitionTransaction::new(TransitionTransactionInit { + deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"), + transaction_id: uuid::Uuid::new_v4(), + owner_epoch: uuid::Uuid::new_v4(), + write_id: uuid::Uuid::new_v4(), + source: TransitionSourceIdentity { + bucket: "source-bucket".to_string(), + object: "source-object".to_string(), + version_id: Some(uuid::Uuid::new_v4()), + data_dir: uuid::Uuid::new_v4(), + mod_time_unix_nanos: 1_770_000_000_000_000_000, + size: 42, + etag: "source-etag".to_string(), + version_mode: TransitionSourceVersionMode::Versioned, + }, + tier_name: "UNUSEDABANDONEDTIER".to_string(), + backend_fingerprint: [7; 32], + not_after_unix_nanos: 1, + }) + .expect("transaction should build"); + save_transition_transaction_record(store.clone(), &transaction) + .await + .expect("transaction record should persist"); + + let record_name = + transition_transaction_record_object_name(transaction.transaction_id).expect("transaction record name should derive"); + let source = observe_recovery_source( + store.clone(), + &record_name, + crate::bucket::lifecycle::transition_transaction::TRANSITION_TRANSACTION_SCHEMA, + ) + .await + .expect("transaction source generation should be observable"); + let mut control = IlmRecoveryControl::new( + IlmRecoveryControlIdentity { + protocol: IlmRecoveryProtocol::TransitionTransaction, + canonical_source_path: record_name, + stable_operation_identity: transaction.transaction_id.to_string(), + record_class: "transition_transaction_v1".to_string(), + }, + source.generation, + IlmRecoveryClassification::Retrying, + 2_000_000_000, + IlmRecoveryErrorCode::None, + ) + .expect("recovery control should build"); + let mut now = 3_000_000_000; + for _ in 1..MAX_RECOVERY_ATTEMPTS { + now = now.max(control.next_attempt_at_unix_nanos.unwrap_or(now)); + control + .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1) + .expect("abandoned attempt should claim"); + control + .record_expired_attempt(now + 1) + .expect("expired attempt should consume retry budget"); + now += 2; + } + now = now.max(control.next_attempt_at_unix_nanos.expect("last retry should have a backoff")); + control + .claim("cancelled-or-timed-out-owner", uuid::Uuid::new_v4(), now, 1) + .expect("final abandoned attempt should claim"); + save_recovery_control_if_absent(store.clone(), &control) + .await + .expect("claimed recovery control should persist"); + + let stats = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(now + 1)) + .await + .expect("recovery should account for the expired attempt"); + assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0)); + + let control_id = transition_recovery_control_id(&transaction).expect("control id should derive"); + let persisted = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id) + .await + .expect("expired recovery control should remain inspectable"); + assert_eq!(persisted.control.classification, IlmRecoveryClassification::OperatorRequired); + assert_eq!(persisted.control.attempt_count, u64::from(MAX_RECOVERY_ATTEMPTS)); + assert_eq!(persisted.control.consecutive_failure_count, MAX_RECOVERY_ATTEMPTS); + assert_eq!(persisted.control.last_error_code, IlmRecoveryErrorCode::AttemptLeaseExpired); + assert!(persisted.control.owner.is_none()); + assert_eq!( + transition_transaction_record_count(store).await, + 1, + "budget exhaustion must retain the source record" + ); + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] @@ -19351,6 +19456,7 @@ mod tests { ), ]; let mut expected_removes = Vec::new(); + let mut recovery_control_ids = Vec::new(); for (case, put_version, remote_version, source_mode) in cases { let mut transaction = TransitionTransaction::new(TransitionTransactionInit { deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"), @@ -19388,6 +19494,8 @@ mod tests { save_transition_transaction_record(store.clone(), &transaction) .await .expect("transaction record should persist"); + recovery_control_ids + .push(transition_recovery_control_id(&transaction).expect("transition recovery control id should derive")); expected_removes.push((transaction.remote_object, put_version)); } @@ -19403,6 +19511,14 @@ mod tests { assert_eq!(actual_removes, expected_removes, "recovery must preserve each remote version shape"); assert_eq!(backend.exact_remove_count(), 2); assert_eq!(backend.object_count().await, 0); + for control_id in recovery_control_ids { + let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id) + .await + .expect("completed recovery control should remain inspectable"); + assert_eq!(control.control.classification, IlmRecoveryClassification::Terminal); + assert_eq!(control.control.attempt_count, 1); + assert!(control.control.owner.is_none()); + } let replay = recover_transition_transaction_records(store, 100, None) .await @@ -19415,6 +19531,94 @@ mod tests { ); } + #[cfg(feature = "test-util")] + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn transition_transaction_recovery_resumes_source_cleanup_after_terminal_crash() { + let temp_dir = tempfile::tempdir().expect("create temp store dir"); + let (ctx, store, _shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-transaction-terminal-crash", &[4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let tier_name = "TXTERMINALCRASH"; + let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await; + let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name) + .await + .expect("tier lease should resolve") + .backend_identity(); + let remote_version = uuid::Uuid::new_v4().to_string(); + let mut transaction = TransitionTransaction::new(TransitionTransactionInit { + deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"), + transaction_id: uuid::Uuid::new_v4(), + owner_epoch: uuid::Uuid::new_v4(), + write_id: uuid::Uuid::new_v4(), + source: TransitionSourceIdentity { + bucket: "source-bucket".to_string(), + object: "source-object".to_string(), + version_id: Some(uuid::Uuid::new_v4()), + data_dir: uuid::Uuid::new_v4(), + mod_time_unix_nanos: 1_770_000_000_000_000_000, + size: 42, + etag: "source-etag".to_string(), + version_mode: TransitionSourceVersionMode::Versioned, + }, + tier_name: tier_name.to_string(), + backend_fingerprint: backend_identity, + not_after_unix_nanos: 1, + }) + .expect("transaction should build"); + transaction + .advance( + transaction.fence(), + TransitionTransactionState::Uploaded, + Some(TransitionRemoteVersion::versioned(remote_version.clone())), + ) + .expect("transaction should enter uploaded state"); + backend.set_put_remote_version(Some(remote_version)).await; + let candidate = bytes::Bytes::from_static(b"terminal crash candidate"); + backend + .put( + &transaction.remote_object, + ReaderImpl::Body(candidate.clone()), + i64::try_from(candidate.len()).expect("test candidate length should fit i64"), + ) + .await + .expect("mock backend should accept candidate"); + save_transition_transaction_record(store.clone(), &transaction) + .await + .expect("transaction record should persist"); + let control_id = transition_recovery_control_id(&transaction).expect("control id should derive"); + + let barrier = TransitionRecoveryTerminalBarrier::install(transaction.transaction_id); + let recovery_store = store.clone(); + let recovery = tokio::spawn(async move { recover_transition_transaction_records(recovery_store, 100, None).await }); + barrier.wait_until_paused().await; + let terminal = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &control_id) + .await + .expect("terminal control should persist before source cleanup"); + assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal); + assert_eq!(transition_transaction_record_count(store.clone()).await, 1); + assert_eq!(backend.object_count().await, 0); + assert_eq!(backend.exact_remove_count(), 1); + + recovery.abort(); + assert!( + recovery + .await + .expect_err("recovery should be cancelled at the crash boundary") + .is_cancelled() + ); + drop(barrier); + + let replay = recover_transition_transaction_records(store.clone(), 100, None) + .await + .expect("terminal control should resume source cleanup without another remote delete"); + assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 1, 0, 0)); + assert_eq!(transition_transaction_record_count(store).await, 0); + assert_eq!(backend.exact_remove_count(), 1, "terminal replay must not repeat the remote delete"); + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] @@ -19472,6 +19676,8 @@ mod tests { save_transition_transaction_record(store.clone(), &uploaded) .await .expect("transaction record should persist"); + let recovery_control_id = + transition_recovery_control_id(&uploaded).expect("transition recovery control id should derive"); let barrier = TransitionRecoveryClaimBarrier::install(uploaded.transaction_id); let recovery_store = store.clone(); @@ -19493,11 +19699,17 @@ mod tests { .expect("recovery should treat the lost CAS as a retained transaction"); assert_eq!((stats.scanned, stats.recovered, stats.retained, stats.failed), (1, 0, 1, 0)); assert_eq!( - load_transition_transaction_record(store, uploaded.transaction_id) + load_transition_transaction_record(store.clone(), uploaded.transaction_id) .await .expect("newer transaction revision must remain"), active ); + let control = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id) + .await + .expect("lost source CAS should retain a retryable recovery control"); + assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying); + assert_eq!(control.control.consecutive_failure_count, 1); + assert_eq!(control.control.last_error_code, IlmRecoveryErrorCode::SourceGenerationChanged); assert_eq!(backend.object_count().await, 1, "a stale recovery must not delete the candidate"); assert_eq!(backend.remove_count().await, 0); } @@ -19799,27 +20011,15 @@ mod tests { not_after_unix_nanos: 1_780_000_000_000_000_000, }) .expect("transaction should build"); - let uploaded_fence = transaction + transaction .advance( transaction.fence(), TransitionTransactionState::Uploaded, - Some(TransitionRemoteVersion::versioned(remote_version)), + Some(TransitionRemoteVersion::versioned(remote_version.clone())), ) .expect("transaction should enter uploaded state"); - transaction - .mark_cleanup_pending( - uploaded_fence, - TransitionCleanupProof { - transaction_id: transaction.transaction_id, - write_id: transaction.write_id, - remote_object: transaction.remote_object.clone(), - remote_version: transaction.remote_version.clone(), - backend_fingerprint: transaction.backend_fingerprint, - decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit, - }, - ) - .expect("transaction should enter cleanup pending state"); let candidate = bytes::Bytes::from_static(b"cleanup pending candidate retained after failure"); + backend.set_put_remote_version(Some(remote_version)).await; backend .put( &transaction.remote_object, @@ -19831,6 +20031,8 @@ mod tests { save_transition_transaction_record(store.clone(), &transaction) .await .expect("transaction record should persist"); + let recovery_control_id = + transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"); backend.set_remove_failure(true); let stats = recover_transition_transaction_records(store.clone(), 100, None) @@ -19846,6 +20048,42 @@ mod tests { assert_eq!(backend.remove_versions().await, Vec::<(String, String)>::new()); assert_eq!(backend.exact_remove_count(), 1); assert_eq!(backend.object_count().await, 1); + let control = load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id) + .await + .expect("failed recovery control should persist"); + assert_eq!(control.control.classification, IlmRecoveryClassification::Retrying); + assert_eq!(control.control.attempt_count, 1); + assert_eq!(control.control.consecutive_failure_count, 1); + assert!( + control + .control + .next_attempt_at_unix_nanos + .is_some_and(|next| next > OffsetDateTime::now_utc().unix_timestamp_nanos() as i64) + ); + + backend.set_remove_failure(false); + let replay = recover_transition_transaction_records(store.clone(), 100, None) + .await + .expect("recovery before the persisted deadline should be skipped"); + assert_eq!((replay.scanned, replay.recovered, replay.retained, replay.failed), (1, 0, 1, 0)); + assert_eq!(backend.exact_remove_count(), 1, "persisted backoff must prevent an immediate retry"); + + let retry_at = control + .control + .next_attempt_at_unix_nanos + .expect("retry deadline should persist"); + let retried = recover_transition_transaction_records_at(store.clone(), 100, None, i128::from(retry_at) + 1) + .await + .expect("recovery at the persisted deadline should retry the advanced source generation"); + assert_eq!((retried.scanned, retried.recovered, retried.retained, retried.failed), (1, 1, 0, 0)); + assert_eq!(backend.exact_remove_count(), 2); + assert_eq!(backend.object_count().await, 0); + assert_eq!(transition_transaction_record_count(store.clone()).await, 0); + let terminal = load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id) + .await + .expect("completed retry control should remain inspectable"); + assert_eq!(terminal.control.classification, IlmRecoveryClassification::Terminal); + assert_eq!(terminal.control.attempt_count, 2); } #[cfg(feature = "test-util")] @@ -20146,6 +20384,10 @@ mod tests { local_commit_started .advance(local_commit_started.fence(), TransitionTransactionState::LocalCommitStarted, None) .expect("transaction should enter local commit state"); + let upload_started_control_id = + transition_recovery_control_id(&upload_started).expect("upload-started control id should derive"); + let local_commit_control_id = + transition_recovery_control_id(&local_commit_started).expect("local-commit control id should derive"); backend.set_put_remote_version(Some(remote_version)).await; for transaction in [&upload_started, &local_commit_started] { @@ -20176,6 +20418,19 @@ mod tests { assert_eq!(backend.object_count().await, 2, "recovery must not delete an unproven remote candidate"); assert_eq!(backend.remove_count().await, 0); assert_eq!(backend.exact_remove_count(), 0); + let upload_started_control = + load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &upload_started_control_id) + .await + .expect("upload-started control should persist"); + assert_eq!( + upload_started_control.control.classification, + IlmRecoveryClassification::RetainedAmbiguous + ); + let local_commit_control = + load_recovery_control(store, IlmRecoveryProtocol::TransitionTransaction, &local_commit_control_id) + .await + .expect("local-commit control should persist"); + assert_eq!(local_commit_control.control.classification, IlmRecoveryClassification::OperatorRequired); } #[cfg(feature = "test-util")] @@ -20514,7 +20769,7 @@ mod tests { .await; let unsupported_stats = recover_transition_transaction_records(store.clone(), 100, None) .await - .expect("unsupported provider recovery should fail closed"); + .expect("active unknown ownership should remain fenced before provider recovery"); assert_eq!( ( unsupported_stats.scanned, @@ -20523,14 +20778,28 @@ mod tests { unsupported_stats.failed ), (1, 0, 1, 0), - "an unsupported provider probe must retain the unknown upload" + "active unknown ownership must retain the upload before the recovery deadline" ); assert_eq!(transition_transaction_record_count(store.clone()).await, 1); + let recovery_control_id = + transition_recovery_control_id(&transaction).expect("transition recovery control id should derive"); + assert!(matches!( + load_recovery_control(store.clone(), IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id).await, + Err(Error::ConfigNotFound) + )); assert!( backend.contains(&transaction.remote_object).await, - "unsupported recovery must not delete the candidate" + "active ownership must not delete the candidate" + ); + assert_eq!(backend.remove_count().await, 0, "active ownership must not attempt cleanup"); + assert!( + !backend + .op_log() + .await + .iter() + .any(|operation| matches!(operation, MockWarmOp::Probe { .. })), + "active ownership must not probe the provider" ); - assert_eq!(backend.remove_count().await, 0, "unsupported recovery must not attempt cleanup"); backend.set_transition_candidate_probe_override(None).await; let stats = diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index 908572998..469a5e365 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -238,7 +238,7 @@ async fn list_pool_multipart_uploads_for_incarnation( } impl ECStore { - #[cfg(test)] + #[cfg(all(test, feature = "test-util"))] pub(crate) fn reset_data_movement_multipart_discovery_count_for_test(&self) { data_movement_multipart_discovery_counts() .lock() @@ -246,7 +246,7 @@ impl ECStore { .insert(self.id, 0); } - #[cfg(test)] + #[cfg(all(test, feature = "test-util"))] pub(crate) fn data_movement_multipart_discovery_count_for_test(&self) -> usize { data_movement_multipart_discovery_counts() .lock() diff --git a/crates/heal/src/error.rs b/crates/heal/src/error.rs index 336ba00ad..7fa8d7717 100644 --- a/crates/heal/src/error.rs +++ b/crates/heal/src/error.rs @@ -54,6 +54,15 @@ pub enum Error { #[error("Heal task execution failed: {message}")] TaskExecutionFailed { message: String }, + /// The current page already exhausted its local retry budget. Retrying + /// the enclosing bucket would replay pages whose results were counted. + #[error("Heal listing failed for bucket {bucket}: {source}")] + HealListingFailed { + bucket: String, + #[source] + source: Box, + }, + #[error("Invalid heal type: {heal_type}")] InvalidHealType { heal_type: String }, diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 33ac33c38..580b01246 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -447,6 +447,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "running".to_string(), None, @@ -463,6 +464,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "running".to_string(), Some(format!("heal task retrying after recoverable failure, attempt {retry_attempt}: {error}")), @@ -479,6 +481,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "finished".to_string(), None, @@ -495,6 +498,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "stopped".to_string(), Some("heal task cancelled".to_string()), @@ -511,6 +515,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "stopped".to_string(), Some("heal task timed out".to_string()), @@ -527,6 +532,7 @@ impl HealChannelProcessor { progress, next_seq, min_seq, + .. }) => ( "stopped".to_string(), Some(error), diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 971590fdb..c90a18729 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::heal::{ + outcome::HealTaskOutcome, progress::{HealProgress, HealStatistics}, resume::{ReplacementPhase, ResumeGc, ResumeManager, ResumeState, ResumeUtils}, storage::HealStorageAPI, @@ -185,6 +186,7 @@ fn record_displaced_terminal( request: &HealRequest, ) -> Arc { let terminal = Arc::new(CompletedHealStatus { + outcome: None, progress: None, retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type.clone(), @@ -268,6 +270,7 @@ async fn publish_completed_heal( #[derive(Debug, Clone)] pub struct HealTaskReport { + pub outcome: Option>, pub status: HealTaskStatus, pub result_items: Vec, pub result_items_truncated: bool, @@ -285,6 +288,7 @@ async fn active_task_report(task: &HealTask, since: Option) -> HealTaskRepo let window = task.get_result_items_since(since).await; HealTaskReport { status: task.get_status().await, + outcome: Some(Arc::new(task.get_outcome().await)), result_items: window.items, // The legacy flag stays set once anything was evicted; a lagging // incremental cursor additionally marks this response truncated so @@ -298,6 +302,7 @@ async fn active_task_report(task: &HealTask, since: Option) -> HealTaskRepo fn empty_task_report(status: HealTaskStatus) -> HealTaskReport { HealTaskReport { + outcome: None, status, result_items: Vec::new(), result_items_truncated: false, @@ -325,6 +330,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option) -> }; HealTaskReport { status: completed.status.clone(), + outcome: completed.outcome.clone(), result_items, result_items_truncated: completed.result_items_truncated || lagged, progress: completed.progress.clone(), diff --git a/crates/heal/src/heal/manager/queue.rs b/crates/heal/src/heal/manager/queue.rs index aceabe42a..a47cd43e2 100644 --- a/crates/heal/src/heal/manager/queue.rs +++ b/crates/heal/src/heal/manager/queue.rs @@ -83,6 +83,7 @@ pub(super) struct CompletedHealStatus { pub(super) heal_type: HealType, pub(super) status: HealTaskStatus, pub(super) progress: Option, + pub(super) outcome: Option>, pub(super) retained_bytes: std::sync::OnceLock, pub(super) result_items_truncated: bool, pub(super) completed_at: SystemTime, @@ -105,6 +106,7 @@ impl CompletedHealStatus { fn measure_retained_bytes(&self) -> usize { let mut bytes = size_of::(); let mut add = |amount: usize| bytes = bytes.saturating_add(amount); + add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes())); match &self.heal_type { HealType::Cluster => {} HealType::Bucket { bucket } => add(bucket.capacity()), @@ -209,6 +211,7 @@ impl CompletedHealStatus { heal_type: task.heal_type.clone(), status, progress: Some(task.get_progress().await), + outcome: Some(Arc::new(task.get_outcome().await)), retained_bytes: std::sync::OnceLock::new(), result_items_truncated: task.result_items_truncated(), completed_at: SystemTime::now(), diff --git a/crates/heal/src/heal/manager/scheduler.rs b/crates/heal/src/heal/manager/scheduler.rs index feb0c1231..704a7cbeb 100644 --- a/crates/heal/src/heal/manager/scheduler.rs +++ b/crates/heal/src/heal/manager/scheduler.rs @@ -298,6 +298,7 @@ impl HealManager { if cancelled_completion { completed_status = HealTaskStatus::Cancelled; completed_status_entry.status = HealTaskStatus::Cancelled; + completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await)); } let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. }); let successful_completion = matches!(completed_status, HealTaskStatus::Completed); diff --git a/crates/heal/src/heal/manager/tests.rs b/crates/heal/src/heal/manager/tests.rs index aa1c1293f..d8b3e3b16 100644 --- a/crates/heal/src/heal/manager/tests.rs +++ b/crates/heal/src/heal/manager/tests.rs @@ -103,6 +103,7 @@ struct MockStorage; fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus { CompletedHealStatus { + outcome: None, heal_type: HealType::Cluster, status: HealTaskStatus::Completed, progress: Some(HealProgress { @@ -287,6 +288,59 @@ pub(super) async fn pause_completed_retention_before_publish(task_id: &str, stat } } +#[tokio::test] +async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() { + use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome}; + use crate::heal::task::{OUTCOME_FINISH_TEST_HOOK, OutcomeFinishTestHook}; + let bucket = "canonical-outcome-cancel-before-finish"; + let manager = HealManager::new(Arc::new(MockStorage), None); + let request = HealRequest::object(bucket.to_string(), "object".to_string(), None); + let task_id = request.id.clone(); + let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None); + let alias = duplicate.id.clone(); + let retention_hook = Arc::new(CompletedRetentionHook::default()); + { + let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await; + hooks.insert(bucket.to_string(), retention_hook.clone()); + hooks.insert(task_id.clone(), retention_hook.clone()); + } + let finish_hook = Arc::new(OutcomeFinishTestHook { + task_id: task_id.clone(), + reached: Notify::new(), + release: Notify::new(), + }); + *OUTCOME_FINISH_TEST_HOOK.lock().await = Some(finish_hook.clone()); + manager.submit_heal_request(request).await.expect("admit original"); + manager.submit_heal_request(duplicate).await.expect("admit alias"); + process_manager_queue_once(&manager).await; + tokio::time::timeout(Duration::from_secs(5), retention_hook.started.notified()) + .await + .expect("storage started"); + retention_hook.execute.notify_one(); + tokio::time::timeout(Duration::from_secs(5), finish_hook.reached.notified()) + .await + .expect("storage returned before outcome finalization"); + manager.cancel_task(&alias).await.expect("cancel wins publication"); + finish_hook.release.notify_one(); + tokio::time::timeout(Duration::from_secs(5), retention_hook.handoff.notified()) + .await + .expect("scheduler completes cancelled handoff"); + for token in [&task_id, &alias] { + let report = manager.get_task_report(token).await.expect("cancelled token retained"); + assert_eq!(report.status, HealTaskStatus::Cancelled); + assert_eq!( + report.outcome.as_ref().expect("frozen outcome").execution, + HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) + ); + } + retention_hook.finish.notify_one(); + *OUTCOME_FINISH_TEST_HOOK.lock().await = None; + COMPLETED_RETENTION_HOOKS + .lock() + .await + .retain(|key, _| key != bucket && key != &task_id); +} + #[tokio::test] async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() { let bucket = "completed-retention-retry-cancel"; @@ -325,6 +379,10 @@ async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() { for token in [&task_id, &alias] { let report = manager.get_task_report(token).await.expect("cancelled token retained"); assert_eq!(report.status, HealTaskStatus::Cancelled); + assert_eq!( + report.outcome.as_ref().expect("cancelled outcome retained").execution, + crate::heal::outcome::HealExecutionOutcome::Aborted(crate::heal::outcome::HealAbortReason::Cancelled) + ); assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1); } assert!(!manager.retrying_heals.lock().await.contains_key(&task_id)); @@ -391,6 +449,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han .expect("scheduler archives terminal"); assert!(!manager.active_heals.lock().await.contains_key(&task_id)); let expected = task.get_progress().await; + let expected_outcome = task.get_outcome().await; for token in [&task_id, &alias] { assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected); let report = manager @@ -398,6 +457,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han .await .expect("terminal token remains queryable at handoff"); assert_eq!(report.progress.as_ref(), Some(&expected)); + assert_eq!(report.outcome.as_deref(), Some(&expected_outcome)); assert!(report.result_items.is_empty()); match outcome { "success" => assert_eq!(report.status, HealTaskStatus::Completed), @@ -1976,6 +2036,7 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) -> task_id, Arc::new(CompletedHealStatus { progress: None, + outcome: None, retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type, status: HealTaskStatus::Retrying { @@ -2700,6 +2761,7 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() { task_id.clone(), Arc::new(CompletedHealStatus { progress: None, + outcome: None, retained_bytes: std::sync::OnceLock::new(), heal_type: request.heal_type.clone(), status: HealTaskStatus::Retrying { @@ -2737,6 +2799,7 @@ async fn test_get_task_status_reads_recent_completed_status() { "completed-token".to_string(), Arc::new(CompletedHealStatus { progress: None, + outcome: None, retained_bytes: std::sync::OnceLock::new(), heal_type: HealType::Bucket { bucket: "bucket".to_string(), @@ -2768,6 +2831,7 @@ async fn test_get_task_report_for_path_reads_completed_items() { "completed-token".to_string(), Arc::new(CompletedHealStatus { progress: None, + outcome: None, retained_bytes: std::sync::OnceLock::new(), heal_type: HealType::Object { bucket: "bucket".to_string(), diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index c918bea49..5f17c8cd8 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -16,6 +16,7 @@ pub mod channel; pub mod erasure_healer; pub mod manager; pub mod mrf_queue; +pub mod outcome; pub mod progress; pub(crate) mod replacement_readiness; pub mod resume; diff --git a/crates/heal/src/heal/mrf_queue/snapshot.rs b/crates/heal/src/heal/mrf_queue/snapshot.rs index e8d51ed9e..d39e0fafa 100644 --- a/crates/heal/src/heal/mrf_queue/snapshot.rs +++ b/crates/heal/src/heal/mrf_queue/snapshot.rs @@ -33,6 +33,9 @@ use std::collections::HashMap; use tokio::io::AsyncReadExt; use uuid::Uuid; +/// Explicit pending migration; never activates the production writer or GC. +pub mod migration; + // Root-level control files avoid requiring a new directory before the first // atomic commit. They remain inside the storage owner's metadata volume. const PAYLOAD_PATHS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"]; @@ -66,6 +69,23 @@ struct Manifest { } impl Manifest { + fn encode(owner: Uuid, sequence: u64, payload: &[u8]) -> Result, SnapshotError> { + let mut bytes = Vec::with_capacity(MANIFEST_LEN); + bytes.extend_from_slice(MAGIC); + bytes.push(VERSION); + bytes.extend_from_slice(owner.as_bytes()); + bytes.extend_from_slice(&sequence.to_le_bytes()); + bytes.extend_from_slice( + &u64::try_from(payload.len()) + .map_err(|_| SnapshotError::TooLarge)? + .to_le_bytes(), + ); + bytes.extend_from_slice(&Sha256::digest(payload)); + bytes.extend_from_slice(&Sha256::digest(&bytes)); + Self::decode(&bytes, payload.len())?; + Ok(bytes) + } + fn decode(bytes: &[u8], limit: usize) -> Result { if bytes.len() != MANIFEST_LEN || &bytes[..8] != MAGIC { return Err(SnapshotError::Corrupt); diff --git a/crates/heal/src/heal/mrf_queue/snapshot/migration.rs b/crates/heal/src/heal/mrf_queue/snapshot/migration.rs new file mode 100644 index 000000000..97dedaa59 --- /dev/null +++ b/crates/heal/src/heal/mrf_queue/snapshot/migration.rs @@ -0,0 +1,1450 @@ +// Copyright 2026 RustFS Team +// Licensed under the Apache License, Version 2.0. + +//! Pending, owner-local legacy import. These paths are deliberately invisible +//! to the active snapshot reader and legacy consumer. Source revalidation is +//! not a writer freeze: no result here grants activation or reclamation rights. +//! All source bytes and inherited responsibilities survive admission/replay. + +use super::{MANIFEST_LEN, Manifest, SnapshotError, read_bounded}; +use crate::heal::RUSTFS_META_BUCKET; +use crate::heal::mrf_queue::{MRF_JOURNAL_PATH, MRF_SCOPED_JOURNAL_PATH, decode_one}; +use crate::heal::storage_api::owner::{ + EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskStore, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use tokio::io::AsyncReadExt; +use uuid::Uuid; + +const PAYLOADS: [&str; 2] = [".heal-mrf-import-pending.0.bin", ".heal-mrf-import-pending.1.bin"]; +const COMMITS: [&str; 2] = [".heal-mrf-import-commit.0.bin", ".heal-mrf-import-commit.1.bin"]; +const MAX_DISKS: usize = 64; +const CLAIM: &str = ".heal-mrf-import-claim.bin"; + +/// Limits apply to the complete encoded candidate and all distinct raw records, +/// including inherited sources. Exceeding either preserves previous anchors. +#[derive(Clone, Copy, Debug)] +pub struct MigrationLimits { + pub max_bytes: usize, + pub max_records: usize, + pub max_sources: usize, +} + +#[derive(Debug, thiserror::Error)] +pub enum MigrationError { + #[error(transparent)] + Snapshot(#[from] SnapshotError), + #[error("MRF migration requires every configured, formatted local disk")] + CoverageGap, + #[error("MRF migration source changed; all recovery anchors are retained")] + SourceChanged, + #[error("MRF migration candidate is invalid")] + Invalid, + #[error("MRF migration has no responsibility evidence")] + Empty, + #[error("MRF migration conditional publication conflicted")] + Conflict, + #[error("MRF migration staging is claimed; interrupted claims require separately fenced recovery")] + Claimed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +enum LegacyPath { + Scoped, + Mirror, +} + +impl LegacyPath { + fn path(self) -> &'static str { + match self { + Self::Scoped => MRF_SCOPED_JOURNAL_PATH, + Self::Mirror => MRF_JOURNAL_PATH, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Source { + disk_id: Uuid, + path: LegacyPath, + digest: [u8; 32], + // None proves an observed absent path, distinct from a present empty file. + bytes: Option>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PendingMigration { + version: u8, + sources: Vec, + inherited: Vec, +} + +impl PendingMigration { + /// Raw, complete records. No scope/version normalization, attempts pruning, + /// incarnation inference or task-success interpretation is performed. + pub fn replay_records(&self, limits: MigrationLimits) -> Result>, MigrationError> { + self.validate_limits(limits)?; + let mut records = BTreeSet::new(); + for source in self.sources.iter().chain(&self.inherited) { + if source.disk_id.is_nil() { + return Err(MigrationError::Invalid); + } + let bytes = source.bytes.as_deref().unwrap_or_default(); + if <[u8; 32]>::from(Sha256::digest(bytes)) != source.digest { + return Err(MigrationError::Invalid); + } + let mut offset = 0; + while offset < bytes.len() { + let (_, consumed) = decode_one(&bytes[offset..]).ok_or(MigrationError::Invalid)?; + let end = offset.checked_add(consumed).ok_or(MigrationError::Invalid)?; + records.insert(bytes[offset..end].to_vec()); + if records.len() > limits.max_records { + return Err(SnapshotError::TooLarge.into()); + } + offset = end; + } + } + Ok(records.into_iter().collect()) + } + + fn validate_limits(&self, limits: MigrationLimits) -> Result<(), MigrationError> { + if self.version != 1 { + return Err(SnapshotError::Unsupported.into()); + } + if self.sources.is_empty() || self.sources.len() > MAX_DISKS * 2 { + return Err(MigrationError::Invalid); + } + if self + .sources + .len() + .checked_add(self.inherited.len()) + .is_none_or(|count| count > limits.max_sources) + { + return Err(SnapshotError::TooLarge.into()); + } + // Bound the raw input before allocating the JSON representation. + let total = self + .sources + .iter() + .chain(&self.inherited) + .try_fold(0usize, |total, source| { + total + .checked_add(source.bytes.as_ref().map_or(0, Vec::len)) + .and_then(|n| n.checked_add(128)) + }) + .ok_or(SnapshotError::TooLarge)?; + if total > limits.max_bytes { + return Err(SnapshotError::TooLarge.into()); + } + Ok(()) + } + + fn encode(&self, limits: MigrationLimits) -> Result, MigrationError> { + self.replay_records(limits)?; + let bytes = serde_json::to_vec(self).map_err(|_| MigrationError::Invalid)?; + if bytes.len() > limits.max_bytes { + return Err(SnapshotError::TooLarge.into()); + } + Ok(bytes) + } + + async fn revalidate(&self, disks: &[EcstoreDiskStore], limits: MigrationLimits) -> Result<(), MigrationError> { + let current = capture(disks, limits).await?; + if current.sources != self.sources { + return Err(MigrationError::SourceChanged); + } + Ok(()) + } +} + +async fn configured_disks(disks: &[Option]) -> Result, MigrationError> { + if disks.is_empty() || disks.len() > MAX_DISKS { + return Err(MigrationError::CoverageGap); + } + let mut ordered = std::collections::BTreeMap::new(); + for disk in disks { + let disk = disk.as_ref().ok_or(MigrationError::CoverageGap)?; + if !EcstoreDiskAPI::is_local(disk.as_ref()) { + return Err(MigrationError::CoverageGap); + } + let id = EcstoreDiskAPI::get_disk_id(disk.as_ref()) + .await + .map_err(SnapshotError::Disk)? + .filter(|id| !id.is_nil()) + .ok_or(MigrationError::CoverageGap)?; + if ordered.insert(id, disk.clone()).is_some() { + return Err(MigrationError::CoverageGap); + } + } + Ok(ordered.into_values().collect()) +} + +async fn capture(disks: &[EcstoreDiskStore], limits: MigrationLimits) -> Result { + let mut sources = Vec::with_capacity(disks.len() * 2); + let mut identities = BTreeSet::new(); + let mut remaining = limits.max_bytes; + for disk in disks { + let id = EcstoreDiskAPI::get_disk_id(disk.as_ref()) + .await + .map_err(SnapshotError::Disk)? + .filter(|id| !id.is_nil()) + .ok_or(MigrationError::CoverageGap)?; + if !identities.insert(id) { + return Err(MigrationError::CoverageGap); + } + for path in [LegacyPath::Scoped, LegacyPath::Mirror] { + // A missing metadata volume is a coverage gap, not an absent journal. + let bytes = match EcstoreDiskAPI::read_file(disk.as_ref(), RUSTFS_META_BUCKET, path.path()).await { + Ok(reader) => { + let maximum = u64::try_from(remaining.checked_add(1).ok_or(SnapshotError::TooLarge)?) + .map_err(|_| SnapshotError::TooLarge)?; + let mut bytes = Vec::new(); + reader + .take(maximum) + .read_to_end(&mut bytes) + .await + .map_err(SnapshotError::Read)?; + if bytes.len() > remaining { + return Err(SnapshotError::TooLarge.into()); + } + Some(bytes) + } + Err(EcstoreDiskError::FileNotFound) => None, + Err(error) => return Err(SnapshotError::Disk(error).into()), + }; + remaining = remaining + .checked_sub(bytes.as_ref().map_or(0, Vec::len)) + .ok_or(SnapshotError::TooLarge)?; + let digest = Sha256::digest(bytes.as_deref().unwrap_or_default()).into(); + sources.push(Source { + disk_id: id, + path, + digest, + bytes, + }); + } + } + sources.sort_by_key(|source| (source.disk_id, matches!(source.path, LegacyPath::Mirror))); + let candidate = PendingMigration { + version: 1, + sources, + inherited: Vec::new(), + }; + candidate.encode(limits)?; + Ok(candidate) +} + +/// Inspect every configured source without merging v1 mirrors into a claimed +/// latest snapshot. A complete subset/superset is retained as pending evidence. +pub async fn capture_legacy_migration( + disks: &[Option], + limits: MigrationLimits, +) -> Result { + capture(&configured_disks(disks).await?, limits).await +} + +struct Staged { + manifest: Manifest, + candidate: PendingMigration, + slot: usize, +} + +type PayloadIdentity = (usize, [u8; 32]); + +struct StagingLineage { + latest: Option, + // Each collection has at most two identities per configured disk. + committed_payloads: BTreeSet, + orphaned_payloads: Vec, +} + +fn payload_identity(payload: &[u8]) -> PayloadIdentity { + (payload.len(), Sha256::digest(payload).into()) +} + +impl StagingLineage { + fn validate_orphans(&self, retry_payload: Option<&[u8]>) -> Result<(), MigrationError> { + let retry_identity = retry_payload.map(payload_identity); + if self + .orphaned_payloads + .iter() + .any(|identity| Some(*identity) != retry_identity && !self.committed_payloads.contains(identity)) + { + return Err(MigrationError::Conflict); + } + Ok(()) + } +} + +async fn read_staging_lineage(disks: &[EcstoreDiskStore], limits: MigrationLimits) -> Result { + let mut selected: Option = None; + let mut identities = std::collections::BTreeMap::new(); + let mut orphaned_payloads = Vec::new(); + let mut committed_payloads = BTreeSet::new(); + let mut mismatched_manifests = Vec::new(); + for disk in disks { + for slot in 0..2 { + let result = async { + let payload = read_bounded(disk, PAYLOADS[slot], limits.max_bytes).await?; + let Some(bytes) = read_bounded(disk, COMMITS[slot], MANIFEST_LEN).await? else { + if let Some(payload) = payload.as_deref() { + orphaned_payloads.push(payload_identity(payload)); + } + return Ok(None); + }; + let manifest = Manifest::decode(&bytes, limits.max_bytes)?; + let identity = (manifest.owner, manifest.payload_digest); + if identities + .insert(manifest.sequence, identity) + .is_some_and(|old| old != identity) + { + return Err(MigrationError::Conflict); + } + let payload = payload.ok_or(MigrationError::Invalid)?; + if payload.len() != manifest.payload_len || payload_identity(&payload).1 != manifest.payload_digest { + // A reused slot can hold the next retry payload while the + // old manifest still names the previous generation. + orphaned_payloads.push(payload_identity(&payload)); + mismatched_manifests.push(manifest); + return Ok(None); + } + let candidate: PendingMigration = serde_json::from_slice(&payload).map_err(|_| MigrationError::Invalid)?; + if candidate.encode(limits)? != payload || candidate.replay_records(limits)?.is_empty() { + return Err(MigrationError::Invalid); + } + Ok(Some(Staged { + manifest, + candidate, + slot, + })) + } + .await; + match result { + Ok(Some(next)) => { + committed_payloads.insert((next.manifest.payload_len, next.manifest.payload_digest)); + if selected + .as_ref() + .is_none_or(|old| old.manifest.sequence < next.manifest.sequence) + { + selected = Some(next); + } + } + Ok(None) => {} + Err(error) => return Err(error), + } + } + } + // Only a validated successor can supersede an unmatched older manifest. + // A newer or unordered manifest may still name responsibilities absent from + // the remaining payload, even if that payload matches another valid slot. + if mismatched_manifests.iter().any(|manifest| { + selected + .as_ref() + .is_none_or(|latest| latest.manifest.owner != manifest.owner || latest.manifest.sequence <= manifest.sequence) + }) { + return Err(MigrationError::Invalid); + } + Ok(StagingLineage { + latest: selected, + committed_payloads, + orphaned_payloads, + }) +} + +async fn install(disk: &EcstoreDiskStore, path: &str, bytes: &[u8], limit: usize) -> Result<(), MigrationError> { + let expected = read_bounded(disk, path, limit).await?.map(EcstoreDiskBytes::from); + let result = EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + path, + expected, + Some(EcstoreDiskBytes::copy_from_slice(bytes)), + ) + .await + .map_err(SnapshotError::Disk)?; + if result != EcstoreConditionalFileUpdate::Updated { + return Err(MigrationError::Conflict); + } + Ok(()) +} + +/// Persist an explicitly requested pending import using the storage owner's CAS +/// (and its configured metadata durability). This does not freeze legacy ingress +/// or grant a durable-acceptance/GC receipt. Activation requires W14/W21 evidence. +pub async fn stage_legacy_migration( + disks: &[Option], + candidate: &PendingMigration, + owner: Uuid, + limits: MigrationLimits, +) -> Result { + let disks = configured_disks(disks).await?; + // Claim every configured disk in identity order. A crash/cancellation leaves + // claims intact; a new process cannot guess that the old writer is fenced. + let claim = EcstoreDiskBytes::copy_from_slice(Uuid::new_v4().as_bytes()); + let mut claimed = Vec::new(); + let result = async { + for disk in &disks { + match EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, CLAIM, None, Some(claim.clone())) + .await + .map_err(SnapshotError::Disk)? + { + EcstoreConditionalFileUpdate::Updated => claimed.push(disk.clone()), + _ => return Err(MigrationError::Claimed), + } + } + stage_claimed(&disks, candidate, owner, limits).await + } + .await; + let mut release_error = None; + for disk in claimed { + let release = async { + #[cfg(test)] + tests::interrupt_at(owner, tests::Boundary::Release).await?; + let released = + EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, CLAIM, Some(claim.clone()), None) + .await + .map_err(SnapshotError::Disk)?; + if released != EcstoreConditionalFileUpdate::Updated { + return Err(MigrationError::Claimed); + } + Ok(()) + } + .await; + if let Err(error) = release + && release_error.is_none() + { + release_error = Some(error); + } + } + if let Some(error) = release_error { + return Err(error); + } + result +} + +async fn stage_claimed( + disks: &[EcstoreDiskStore], + candidate: &PendingMigration, + owner: Uuid, + limits: MigrationLimits, +) -> Result { + candidate.revalidate(disks, limits).await?; + candidate.validate_limits(limits)?; + let lineage = read_staging_lineage(disks, limits).await?; + let previous = lineage.latest.as_ref(); + if previous.is_some_and(|old| old.manifest.owner != owner) { + return Err(MigrationError::Conflict); + } + let mut candidate = candidate.clone(); + let key = |source: &Source| (source.disk_id, source.path, source.digest, source.bytes.is_some()); + let mut source_index = std::collections::HashMap::new(); + for (index, source) in candidate.sources.iter().chain(&candidate.inherited).enumerate() { + if source_index.insert(key(source), index).is_some() { + return Err(MigrationError::Invalid); + } + } + if let Some(old) = previous { + for source in old.candidate.sources.iter().chain(&old.candidate.inherited) { + if let Some(index) = source_index.get(&key(source)).copied() { + let existing = if index < candidate.sources.len() { + &candidate.sources[index] + } else { + &candidate.inherited[index - candidate.sources.len()] + }; + if existing != source { + return Err(MigrationError::Conflict); + } + } else { + if source_index.len() >= limits.max_sources { + return Err(SnapshotError::TooLarge.into()); + } + source_index.insert(key(source), candidate.sources.len() + candidate.inherited.len()); + candidate.inherited.push(source.clone()); + } + } + } + if candidate.replay_records(limits)?.is_empty() { + return Err(MigrationError::Empty); + } + let payload = candidate.encode(limits)?; + // Exact retry comparison must include all inherited responsibilities. + // Validating only the freshly captured sources would reject our own + // interrupted successor payload before it can be completed. + lineage.validate_orphans(Some(&payload))?; + let digest: [u8; 32] = Sha256::digest(&payload).into(); + let (sequence, slot) = previous.map_or((1, 0), |old| { + if old.manifest.payload_digest == digest { + (old.manifest.sequence, old.slot) + } else { + (old.manifest.sequence.saturating_add(1), 1 - old.slot) + } + }); + let manifest = Manifest::encode(owner, sequence, &payload)?; + for disk in disks { + install(disk, PAYLOADS[slot], &payload, limits.max_bytes).await?; + } + #[cfg(test)] + tests::interrupt_at(owner, tests::Boundary::AfterPayload).await?; + candidate.revalidate(disks, limits).await?; + #[cfg(test)] + tests::interrupt_at(owner, tests::Boundary::BeforeManifest).await?; + for disk in disks { + install(disk, COMMITS[slot], &manifest, MANIFEST_LEN).await?; + } + #[cfg(test)] + tests::interrupt_at(owner, tests::Boundary::AfterManifest).await?; + let readback = read_staging_lineage(disks, limits).await?; + readback.validate_orphans(None)?; + let recovered = readback.latest.ok_or(MigrationError::Invalid)?; + if recovered.manifest.sequence != sequence || recovered.manifest.payload_digest != digest { + return Err(MigrationError::Conflict); + } + recovered.candidate.revalidate(disks, limits).await?; + #[cfg(test)] + tests::interrupt_at(owner, tests::Boundary::AfterReadback).await?; + Ok(sequence) +} + +/// Reload pending obligations after process restart. Manager admission never +/// removes them. Missing/changed sources block migration, preserving all files. +pub async fn recover_pending_migration( + disks: &[Option], + limits: MigrationLimits, +) -> Result, MigrationError> { + let disks = configured_disks(disks).await?; + let lineage = read_staging_lineage(&disks, limits).await?; + lineage.validate_orphans(None)?; + let Some(staged) = lineage.latest else { + return Ok(None); + }; + staged.candidate.revalidate(&disks, limits).await?; + Ok(Some(staged.candidate)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::heal::mrf_queue::encode_intent; + use crate::heal::{DiskOption, Endpoint, new_disk}; + use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope}; + use std::sync::Arc; + use tempfile::TempDir; + + const LIMITS: MigrationLimits = MigrationLimits { + max_bytes: 64 * 1024, + max_records: 100, + max_sources: 64, + }; + + #[derive(Clone, Copy, PartialEq, Eq)] + pub(super) enum Boundary { + AfterPayload, + BeforeManifest, + AfterManifest, + AfterReadback, + Release, + } + + static INTERRUPTIONS: std::sync::LazyLock>> = + std::sync::LazyLock::new(Default::default); + + type SourceChange = (EcstoreDiskStore, Vec); + type SourceChangeMap = std::collections::BTreeMap; + + static SOURCE_CHANGES: std::sync::LazyLock> = std::sync::LazyLock::new(Default::default); + + pub(super) async fn interrupt_at(owner: Uuid, boundary: Boundary) -> Result<(), MigrationError> { + if boundary == Boundary::AfterPayload { + let change = SOURCE_CHANGES.lock().expect("source fault map").remove(&owner); + if let Some((disk, bytes)) = change { + source(&disk, &bytes).await; + } + } + let mut interruptions = INTERRUPTIONS.lock().expect("fault map"); + if interruptions.get(&owner) == Some(&boundary) { + interruptions.remove(&owner); + return Err(SnapshotError::Read(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "injected migration boundary failure", + )) + .into()); + } + Ok(()) + } + + fn record(object: &str, kind: MrfKind, scope: Option) -> Vec { + let mut bytes = Vec::new(); + assert!(encode_intent( + &MrfIntent { + bucket: Arc::from("bucket"), + object: Arc::from(object), + version_id: None, + kind, + scope, + lease: None, + enqueued_at_ms: 1, + attempts: 255, + }, + &mut bytes + )); + bytes + } + + async fn disk(root: &TempDir, name: &str) -> EcstoreDiskStore { + let path = root.path().join(name); + std::fs::create_dir_all(&path).expect("create test disk"); + let mut endpoint = Endpoint::try_from(path.to_string_lossy().as_ref()).expect("disk endpoint"); + endpoint.set_idx = 0; + endpoint.disk_idx = 0; + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("open disk"); + let created = EcstoreDiskAPI::make_volume(disk.as_ref(), RUSTFS_META_BUCKET).await; + assert!( + matches!(created, Ok(()) | Err(EcstoreDiskError::VolumeExists)), + "metadata volume: {created:?}" + ); + let id = Uuid::new_v4(); + let format = serde_json::json!({ + "version": "1", "format": "xl-single", "id": Uuid::new_v4(), + "xl": { "version": "3", "this": id, "sets": [[id]], "distributionAlgo": "SIPMOD+PARITY" } + }); + EcstoreDiskAPI::write_all( + disk.as_ref(), + RUSTFS_META_BUCKET, + "format.json", + serde_json::to_vec(&format).expect("format").into(), + ) + .await + .expect("format disk"); + assert_eq!(EcstoreDiskAPI::get_disk_id(disk.as_ref()).await.expect("formatted identity"), Some(id)); + disk + } + + async fn source(disk: &EcstoreDiskStore, bytes: &[u8]) { + // Legacy source paths predate the root-level COW control files. + EcstoreDiskAPI::write_all( + disk.as_ref(), + RUSTFS_META_BUCKET, + MRF_SCOPED_JOURNAL_PATH, + EcstoreDiskBytes::copy_from_slice(bytes), + ) + .await + .expect("write legacy source"); + } + + #[tokio::test] + async fn migration_subset_union_preserves_sources_across_reopen() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let a = record("a", MrfKind::PartialWrite, None); + let b = record( + "b", + MrfKind::DecodeFailure, + Some(MrfScope { + pool_index: 0, + set_index: 1, + }), + ); + source(&first, &a).await; + source(&second, &[a.clone(), b.clone()].concat()).await; + let disks = [Some(first.clone()), Some(second.clone())]; + let candidate = capture_legacy_migration(&disks, LIMITS) + .await + .expect("capture both complete sources"); + let reverse = capture_legacy_migration(&[Some(second.clone()), Some(first.clone())], LIMITS) + .await + .expect("reverse disk order"); + assert_eq!( + candidate.encode(LIMITS).expect("candidate"), + reverse.encode(LIMITS).expect("reversed candidate") + ); + assert_eq!(candidate.replay_records(LIMITS).expect("raw records").len(), 2); + let owner = Uuid::new_v4(); + assert_eq!( + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("stage candidate"), + 1 + ); + drop(candidate); + let recovered = recover_pending_migration(&disks, LIMITS) + .await + .expect("restart read") + .expect("pending import"); + // Reading or discarding a replay batch must not consume its stored anchor. + let mut admitted = recovered.replay_records(LIMITS).expect("replay batch"); + admitted.pop(); + drop(admitted); + assert_eq!( + recover_pending_migration(&disks, LIMITS) + .await + .expect("second restart") + .expect("anchor") + .replay_records(LIMITS) + .expect("records") + .len(), + 2 + ); + assert_eq!( + stage_legacy_migration(&disks, &recovered, owner, LIMITS) + .await + .expect("idempotent retry"), + 1 + ); + assert_eq!( + EcstoreDiskAPI::read_all(first.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("first source"), + a + ); + assert_eq!( + EcstoreDiskAPI::read_all(second.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("second source"), + [a, b].concat() + ); + assert!( + super::super::read_committed(&[first, second], LIMITS.max_bytes) + .await + .expect("active reader") + .is_none(), + "pending import must not activate the production snapshot" + ); + } + + #[tokio::test] + async fn migration_source_change_and_capacity_failure_keep_old_commit() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let a = record("a", MrfKind::PartialWrite, None); + let b = record("b", MrfKind::PartialWrite, None); + source(&disk, &a).await; + let original = capture_legacy_migration(&disks, LIMITS).await.expect("initial capture"); + stage_legacy_migration(&disks, &original, owner, LIMITS) + .await + .expect("initial commit"); + let before = EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, COMMITS[0]) + .await + .expect("old commit"); + source(&disk, &b).await; + assert!(matches!( + stage_legacy_migration(&disks, &original, owner, LIMITS).await, + Err(MigrationError::SourceChanged) + )); + let next = capture_legacy_migration(&disks, LIMITS).await.expect("new source capture"); + assert!(matches!( + stage_legacy_migration( + &disks, + &next, + owner, + MigrationLimits { + max_records: 1, + ..LIMITS + } + ) + .await, + Err(MigrationError::Snapshot(SnapshotError::TooLarge)) + )); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, COMMITS[0]) + .await + .expect("retained commit"), + before + ); + assert_eq!( + stage_legacy_migration(&disks, &next, owner, LIMITS) + .await + .expect("COW successor"), + 2 + ); + let records = recover_pending_migration(&disks, LIMITS) + .await + .expect("successor restart") + .expect("successor") + .replay_records(LIMITS) + .expect("responsibilities"); + assert!( + records.contains(&a) && records.contains(&b), + "successor must inherit old source responsibilities" + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, COMMITS[0]) + .await + .expect("previous slot retained"), + before + ); + } + + #[tokio::test] + async fn migration_torn_inactive_payload_and_manifest_keep_previous_anchor() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + source(&disk, &record("a", MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + stage_legacy_migration(&disks, &candidate, Uuid::new_v4(), LIMITS) + .await + .expect("initial commit"); + let previous = read_bounded(&disk, PAYLOADS[0], LIMITS.max_bytes).await.expect("old payload"); + for (path, bytes) in [ + (PAYLOADS[1], b"torn payload".as_slice()), + (COMMITS[1], b"torn manifest".as_slice()), + ] { + install(&disk, path, bytes, LIMITS.max_bytes) + .await + .expect("interrupted inactive write"); + assert!( + recover_pending_migration(&disks, LIMITS).await.is_err(), + "unknown successor responsibility must block recovery" + ); + assert_eq!(read_bounded(&disk, PAYLOADS[0], LIMITS.max_bytes).await.expect("old anchor"), previous); + } + } + + #[tokio::test] + async fn migration_missing_corrupt_and_empty_sources_fail_closed() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + assert!(matches!( + capture_legacy_migration(&[Some(disk.clone()), None], LIMITS).await, + Err(MigrationError::CoverageGap) + )); + let empty = capture_legacy_migration(&[Some(disk.clone())], LIMITS) + .await + .expect("observed empty sources"); + assert!(matches!( + stage_legacy_migration(&[Some(disk.clone())], &empty, Uuid::new_v4(), LIMITS).await, + Err(MigrationError::Empty) + )); + source(&disk, b"corrupt").await; + assert!(matches!( + capture_legacy_migration(&[Some(disk)], LIMITS).await, + Err(MigrationError::Invalid) + )); + } + + #[tokio::test] + async fn migration_commit_boundaries_and_lost_response_recover_idempotently() { + for boundary in [ + Boundary::AfterPayload, + Boundary::BeforeManifest, + Boundary::AfterManifest, + Boundary::AfterReadback, + ] { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let bytes = record("a", MrfKind::PartialWrite, None); + source(&disk, &bytes).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + let owner = Uuid::new_v4(); + INTERRUPTIONS.lock().expect("fault map").insert(owner, boundary); + assert!(stage_legacy_migration(&disks, &candidate, owner, LIMITS).await.is_err()); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("source survives interruption"), + bytes + ); + let recovery = recover_pending_migration(&disks, LIMITS).await; + if matches!(boundary, Boundary::AfterManifest | Boundary::AfterReadback) { + assert!(recovery.expect("committed restart").is_some()); + } else { + assert!( + matches!(recovery, Err(MigrationError::Conflict)), + "uncommitted candidate is not a committed recovery result" + ); + } + assert_eq!( + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("retry interrupted stage"), + 1 + ); + assert_eq!( + recover_pending_migration(&disks, LIMITS) + .await + .expect("restart after retry") + .expect("anchor") + .replay_records(LIMITS) + .expect("records"), + vec![bytes] + ); + } + } + + #[tokio::test] + async fn migration_interrupted_claim_does_not_authorize_takeover() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + source(&disk, &record("a", MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + let owner = Uuid::new_v4(); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("committed anchor"); + install(&disk, CLAIM, b"interrupted writer", 64) + .await + .expect("interrupted claim"); + assert!(matches!( + stage_legacy_migration(&disks, &candidate, owner, LIMITS).await, + Err(MigrationError::Claimed) + )); + assert_eq!( + recover_pending_migration(&disks, LIMITS) + .await + .expect("recovery remains read-only") + .expect("anchor") + .replay_records(LIMITS) + .expect("record") + .len(), + 1 + ); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, CLAIM) + .await + .expect("claim retained"), + b"interrupted writer".as_slice() + ); + } + + #[tokio::test] + async fn migration_source_change_after_payload_prevents_manifest_publication() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + source(&disk, &record("a", MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + let owner = Uuid::new_v4(); + let changed = record("b", MrfKind::PartialWrite, None); + SOURCE_CHANGES + .lock() + .expect("source fault map") + .insert(owner, (disk.clone(), changed.clone())); + assert!(matches!( + stage_legacy_migration(&disks, &candidate, owner, LIMITS).await, + Err(MigrationError::SourceChanged) + )); + assert!(matches!(recover_pending_migration(&disks, LIMITS).await, Err(MigrationError::Conflict))); + assert_eq!( + EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH) + .await + .expect("changed source survives"), + changed + ); + assert!( + read_bounded(&disk, PAYLOADS[0], LIMITS.max_bytes) + .await + .expect("candidate retained") + .is_some() + ); + } + + #[test] + fn migration_raw_identity_preserves_kind_scope_and_nil_version() { + let id = Uuid::new_v4(); + let mut variants = Vec::new(); + for kind in [MrfKind::PartialWrite, MrfKind::DecodeFailure] { + for scope in [ + None, + Some(MrfScope { + pool_index: 0, + set_index: 0, + }), + Some(MrfScope { + pool_index: 0, + set_index: 1, + }), + ] { + variants.push(record("same", kind, scope)); + } + } + let mut nil = record("same", MrfKind::PartialWrite, None); + nil[12] = 1; + nil.splice(13..13, [0; 16]); + let end = nil.len() - 4; + let mut crc = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + crc.update(&nil[..end]); + nil[end..].copy_from_slice(&u32::try_from(crc.finalize()).expect("CRC").to_le_bytes()); + variants.push(nil); + let bytes = variants.concat(); + let candidate = PendingMigration { + version: 1, + sources: vec![Source { + disk_id: id, + path: LegacyPath::Scoped, + digest: Sha256::digest(&bytes).into(), + bytes: Some(bytes), + }], + inherited: Vec::new(), + }; + let records = candidate.replay_records(LIMITS).expect("raw identities"); + assert_eq!(records.len(), variants.len()); + for variant in variants { + assert!(records.contains(&variant)); + } + } + + async fn slot_bytes(disk: &EcstoreDiskStore) -> Vec>> { + let mut bytes = Vec::new(); + for path in PAYLOADS.into_iter().chain(COMMITS) { + bytes.push(read_bounded(disk, path, LIMITS.max_bytes).await.expect("slot bytes")); + } + bytes + } + + #[tokio::test] + async fn migration_damaged_newer_commit_never_overwrites_successor_responsibility() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + for name in ["a", "b"] { + source(&disk, &record(name, MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("committed generation"); + } + install(&disk, COMMITS[1], b"torn higher manifest", MANIFEST_LEN) + .await + .expect("manifest fault"); + source(&disk, &record("c", MrfKind::PartialWrite, None)).await; + let before = slot_bytes(&disk).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("latest legacy source"); + assert!(stage_legacy_migration(&disks, &candidate, owner, LIMITS).await.is_err()); + assert_eq!(slot_bytes(&disk).await, before, "the only payload containing b must not be overwritten"); + } + + #[tokio::test] + async fn migration_lower_limits_never_fall_back_to_a_smaller_old_generation() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + for name in ["a", "b"] { + source(&disk, &record(name, MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("committed generation"); + } + source(&disk, &record("a", MrfKind::PartialWrite, None)).await; + let before = slot_bytes(&disk).await; + let first_len = before[0].as_ref().expect("first payload").len(); + for limits in [ + MigrationLimits { + max_bytes: first_len, + ..LIMITS + }, + MigrationLimits { + max_records: 1, + ..LIMITS + }, + MigrationLimits { + max_sources: 2, + ..LIMITS + }, + ] { + assert!(matches!( + recover_pending_migration(&disks, limits).await, + Err(MigrationError::Snapshot(SnapshotError::TooLarge)) + )); + assert_eq!(slot_bytes(&disk).await, before); + } + } + + #[tokio::test] + async fn migration_source_history_count_is_bounded_before_successor_write() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let limits = MigrationLimits { + max_sources: 3, + ..LIMITS + }; + let mut records = (0..10) + .map(|n| record(&n.to_string(), MrfKind::PartialWrite, None)) + .collect::>(); + for round in 0..3 { + records.rotate_left(1); + source(&disk, &records.concat()).await; + let candidate = capture_legacy_migration(&disks, limits) + .await + .expect("bounded current sources"); + assert_eq!(candidate.replay_records(limits).expect("same responsibilities").len(), 10); + let before = slot_bytes(&disk).await; + let result = stage_legacy_migration(&disks, &candidate, owner, limits).await; + if round < 2 { + assert_eq!(result.expect("within source count"), round + 1); + } else { + assert!(matches!(result, Err(MigrationError::Snapshot(SnapshotError::TooLarge)))); + assert_eq!(slot_bytes(&disk).await, before); + } + } + } + + #[tokio::test] + async fn migration_empty_legacy_sources_still_inherit_prior_responsibilities() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let bytes = record("a", MrfKind::PartialWrite, None); + source(&disk, &bytes).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("initial source"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("initial stage"); + EcstoreDiskAPI::compare_and_update_file( + disk.as_ref(), + RUSTFS_META_BUCKET, + MRF_SCOPED_JOURNAL_PATH, + Some(bytes.clone().into()), + None, + ) + .await + .expect("legacy removes source"); + let empty = capture_legacy_migration(&disks, LIMITS) + .await + .expect("valid absent-source observation"); + assert!(empty.replay_records(LIMITS).expect("empty observation").is_empty()); + assert_eq!( + stage_legacy_migration(&disks, &empty, owner, LIMITS) + .await + .expect("inherit earlier obligation"), + 2 + ); + assert_eq!( + recover_pending_migration(&disks, LIMITS) + .await + .expect("restart") + .expect("pending") + .replay_records(LIMITS) + .expect("inherited responsibility"), + vec![bytes] + ); + } + + #[tokio::test] + async fn migration_release_failure_still_releases_other_owned_claims() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + source(&first, &record("a", MrfKind::PartialWrite, None)).await; + source(&second, &record("a", MrfKind::PartialWrite, None)).await; + let disks = [Some(first), Some(second)]; + let ordered = configured_disks(&disks).await.expect("disk order"); + let owner = Uuid::new_v4(); + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + INTERRUPTIONS.lock().expect("fault map").insert(owner, Boundary::Release); + assert!(stage_legacy_migration(&disks, &candidate, owner, LIMITS).await.is_err()); + assert!( + read_bounded(&ordered[0], CLAIM, 64) + .await + .expect("failed release remains claimed") + .is_some() + ); + assert!( + read_bounded(&ordered[1], CLAIM, 64) + .await + .expect("later release attempted") + .is_none() + ); + } + + #[tokio::test] + async fn migration_retry_repairs_missing_manifest_replica() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let disks = [Some(first.clone()), Some(second.clone())]; + let owner = Uuid::new_v4(); + for name in ["a", "b"] { + let bytes = record(name, MrfKind::PartialWrite, None); + source(&first, &bytes).await; + source(&second, &bytes).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture generation"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("stage generation"); + } + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + let committed = read_bounded(&second, COMMITS[1], MANIFEST_LEN) + .await + .expect("manifest") + .expect("committed"); + EcstoreDiskAPI::compare_and_update_file( + second.as_ref(), + RUSTFS_META_BUCKET, + COMMITS[1], + Some(committed.clone().into()), + None, + ) + .await + .expect("lost replica"); + assert_eq!( + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("retry missing replica"), + 2 + ); + assert_eq!( + read_bounded(&second, COMMITS[1], MANIFEST_LEN) + .await + .expect("repaired manifest"), + Some(committed) + ); + } + + #[tokio::test] + async fn migration_old_payload_orphan_matches_any_validated_replica() { + let root = TempDir::new().expect("test directory"); + let first = disk(&root, "first").await; + let second = disk(&root, "second").await; + let disks = [Some(first.clone()), Some(second)]; + let owner = Uuid::new_v4(); + for name in ["a", "b"] { + let bytes = record(name, MrfKind::PartialWrite, None); + for disk in disks.iter().flatten() { + source(disk, &bytes).await; + } + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture generation"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("stage generation"); + } + let old_manifest = read_bounded(&first, COMMITS[0], MANIFEST_LEN) + .await + .expect("old manifest") + .expect("gen1"); + assert_eq!( + EcstoreDiskAPI::compare_and_update_file( + first.as_ref(), + RUSTFS_META_BUCKET, + COMMITS[0], + Some(old_manifest.into()), + None + ) + .await + .expect("remove one old manifest"), + EcstoreConditionalFileUpdate::Updated + ); + let before = slot_bytes(&first).await; + for ordered in [disks.clone(), [disks[1].clone(), disks[0].clone()]] { + let recovered = recover_pending_migration(&ordered, LIMITS) + .await + .expect("older orphan has independent proof") + .expect("gen2"); + assert_eq!(recovered.replay_records(LIMITS).expect("A and B obligations").len(), 2); + let candidate = capture_legacy_migration(&ordered, LIMITS).await.expect("current B source"); + assert_eq!( + stage_legacy_migration(&ordered, &candidate, owner, LIMITS) + .await + .expect("gen2 retry"), + 2 + ); + assert_eq!( + slot_bytes(&first).await, + before, + "known older orphan must not cause fallback or overwrite" + ); + } + } + + #[tokio::test] + async fn migration_successor_retry_validates_orphan_after_inheriting_previous_records() { + for boundary in [Boundary::AfterPayload, Boundary::BeforeManifest] { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let a = record("a", MrfKind::PartialWrite, None); + let b = record("b", MrfKind::PartialWrite, None); + source(&disk, &a).await; + let original = capture_legacy_migration(&disks, LIMITS).await.expect("source A"); + stage_legacy_migration(&disks, &original, owner, LIMITS).await.expect("gen1"); + let old_slot = slot_bytes(&disk).await; + source(&disk, &b).await; + let next = capture_legacy_migration(&disks, LIMITS).await.expect("source B"); + INTERRUPTIONS.lock().expect("fault map").insert(owner, boundary); + assert!(stage_legacy_migration(&disks, &next, owner, LIMITS).await.is_err()); + assert!( + matches!(recover_pending_migration(&disks, LIMITS).await, Err(MigrationError::Conflict)), + "uncommitted AB is not silently accepted as A" + ); + let captured_again = capture_legacy_migration(&disks, LIMITS) + .await + .expect("restart source capture contains only B"); + assert_eq!(captured_again.replay_records(LIMITS).expect("current source"), vec![b.clone()]); + assert_eq!( + stage_legacy_migration(&disks, &captured_again, owner, LIMITS) + .await + .expect("retry must compare inherited AB"), + 2 + ); + let recovered = recover_pending_migration(&disks, LIMITS) + .await + .expect("committed restart") + .expect("gen2"); + let records = recovered.replay_records(LIMITS).expect("retained A and B"); + assert_eq!(records.len(), 2); + assert!(records.contains(&a) && records.contains(&b)); + let after = slot_bytes(&disk).await; + assert_eq!(after[0], old_slot[0]); + assert_eq!(after[2], old_slot[2]); + } + } + + #[tokio::test] + async fn migration_third_generation_retry_keeps_reused_slot_recoverable() { + for boundary in [Boundary::AfterPayload, Boundary::BeforeManifest] { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let records = ["a", "b", "c"] + .into_iter() + .map(|name| record(name, MrfKind::PartialWrite, None)) + .collect::>(); + + for bytes in &records[..2] { + source(&disk, bytes).await; + let candidate = capture_legacy_migration(&disks, LIMITS) + .await + .expect("capture committed generation"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("stage committed generation"); + } + + source(&disk, &records[2]).await; + let third = capture_legacy_migration(&disks, LIMITS) + .await + .expect("capture third generation"); + INTERRUPTIONS.lock().expect("fault map").insert(owner, boundary); + assert!(stage_legacy_migration(&disks, &third, owner, LIMITS).await.is_err()); + assert!(matches!(recover_pending_migration(&disks, LIMITS).await, Err(MigrationError::Conflict))); + + let retry = capture_legacy_migration(&disks, LIMITS) + .await + .expect("recapture third generation"); + assert_eq!( + stage_legacy_migration(&disks, &retry, owner, LIMITS) + .await + .expect("retry third generation"), + 3 + ); + let recovered = recover_pending_migration(&disks, LIMITS) + .await + .expect("recover after third retry") + .expect("third generation"); + let replayed = recovered.replay_records(LIMITS).expect("all staged responsibilities"); + assert_eq!(replayed.len(), 3); + for record in &records { + assert!(replayed.contains(record)); + } + } + } + + #[tokio::test] + async fn migration_third_generation_source_change_retry_keeps_reused_slot_recoverable() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + let records = ["a", "b", "c", "d"] + .into_iter() + .map(|name| record(name, MrfKind::PartialWrite, None)) + .collect::>(); + + for bytes in &records[..2] { + source(&disk, bytes).await; + let candidate = capture_legacy_migration(&disks, LIMITS) + .await + .expect("capture committed generation"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("stage committed generation"); + } + + source(&disk, &records[2]).await; + let third = capture_legacy_migration(&disks, LIMITS) + .await + .expect("capture third generation"); + SOURCE_CHANGES + .lock() + .expect("source fault map") + .insert(owner, (disk.clone(), records[3].clone())); + assert!(matches!( + stage_legacy_migration(&disks, &third, owner, LIMITS).await, + Err(MigrationError::SourceChanged) + )); + assert!(matches!(recover_pending_migration(&disks, LIMITS).await, Err(MigrationError::Conflict))); + + source(&disk, &records[2]).await; + let retry = capture_legacy_migration(&disks, LIMITS) + .await + .expect("recapture restored third generation"); + assert_eq!( + stage_legacy_migration(&disks, &retry, owner, LIMITS) + .await + .expect("retry restored third generation"), + 3 + ); + let recovered = recover_pending_migration(&disks, LIMITS) + .await + .expect("recover after restored third retry") + .expect("third generation"); + let replayed = recovered.replay_records(LIMITS).expect("all staged responsibilities"); + assert_eq!(replayed.len(), 3); + for record in &records[..3] { + assert!(replayed.contains(record)); + } + assert!(!replayed.contains(&records[3])); + } + + #[tokio::test] + async fn migration_newer_manifest_with_stale_payload_fails_closed() { + let root = TempDir::new().expect("test directory"); + let disk = disk(&root, "disk").await; + let disks = [Some(disk.clone())]; + let owner = Uuid::new_v4(); + for name in ["a", "b"] { + source(&disk, &record(name, MrfKind::PartialWrite, None)).await; + let candidate = capture_legacy_migration(&disks, LIMITS).await.expect("capture"); + stage_legacy_migration(&disks, &candidate, owner, LIMITS) + .await + .expect("commit"); + } + let old = read_bounded(&disk, PAYLOADS[0], LIMITS.max_bytes) + .await + .expect("old read") + .expect("old payload"); + install(&disk, PAYLOADS[1], &old, LIMITS.max_bytes) + .await + .expect("stale payload corruption"); + source(&disk, &record("a", MrfKind::PartialWrite, None)).await; + let recovery = recover_pending_migration(&disks, LIMITS).await; + assert!( + recovery.is_err(), + "newer manifest must prevent fallback to old responsibilities: {recovery:?}" + ); + } +} diff --git a/crates/heal/src/heal/outcome.rs b/crates/heal/src/heal/outcome.rs new file mode 100644 index 000000000..ada74376d --- /dev/null +++ b/crates/heal/src/heal/outcome.rs @@ -0,0 +1,305 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Execution results are separate from repair responsibility. A legacy +//! successful storage call supplies no authoritative repair receipt. + +use std::{collections::VecDeque, time::SystemTime}; +use uuid::Uuid; + +const MAX_OUTCOME_ITEMS: usize = 128; +const MAX_OUTCOME_BYTES: usize = 64 * 1024; +const MAX_OUTCOME_DETAIL_BYTES: usize = 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealObjectKind { + Object, + Metadata, + Decode, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HealObjectIdentity { + pub kind: HealObjectKind, + pub bucket: String, + pub object: String, + /// The requested version; None remains unresolved, never an absence proof. + pub version_id: Option, + pub bucket_incarnation_id: Option, + pub pool_index: Option, + pub set_index: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealDeferredReason { + DanglingDeleteGrace, + TransientUsageCache, + TransientExistenceCheck, + Deadline, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealFailureClass { + Recoverable, + RetryExhausted, + Permanent, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HealObjectDisposition { + /// The legacy storage response does not prove the requested check or commit. + Unknown, + Repaired, + VerifiedHealthy, + AuthoritativelyAbsent, + Deferred { + reason: HealDeferredReason, + retry_not_before: Option, + }, + Failed(HealFailureClass), + Cancelled, + DryRunObserved, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HealObjectOutcome { + pub identity: HealObjectIdentity, + pub disposition: HealObjectDisposition, + pub detail: Option, +} + +impl HealObjectOutcome { + fn retained_bytes(&self) -> usize { + size_of::() + .saturating_add(self.identity.bucket.capacity()) + .saturating_add(self.identity.object.capacity()) + .saturating_add(self.identity.version_id.as_ref().map_or(0, String::capacity)) + .saturating_add(self.detail.as_ref().map_or(0, String::capacity)) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum HealTraversalCoverage { + #[default] + Unknown, + Partial, + Complete, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealAbortReason { + Cancelled, + Deadline, + Untraversable, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum HealExecutionOutcome { + #[default] + Pending, + Running, + Completed, + CompletedWithErrors, + Aborted(HealAbortReason), +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HealOutcomeCounters { + pub processed: u64, + pub healed: u64, + pub unchanged: u64, + /// Deferred, cancelled, dry-run and unverified results remain unresolved. + pub skipped: u64, + pub failed: u64, + pub unknown: u64, + pub attempt_failures: u64, + pub overflowed: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HealTaskOutcome { + pub execution: HealExecutionOutcome, + pub coverage: HealTraversalCoverage, + pub counters: HealOutcomeCounters, + /// A bounded diagnostic window, not a complete responsibility ledger. + pub objects: VecDeque, + pub objects_truncated: bool, + retained_object_bytes: usize, + untraversable: bool, +} + +impl HealTaskOutcome { + pub(crate) fn start(&mut self) { + if self.execution != HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) { + self.execution = HealExecutionOutcome::Running; + } + self.coverage = HealTraversalCoverage::Partial; + } + + pub(crate) fn attempt_failed(&mut self) { + self.counters.overflowed |= !super::progress::increment_counter(&mut self.counters.attempt_failures); + } + + pub(crate) fn mark_untraversable(&mut self) { + self.untraversable = true; + self.coverage = HealTraversalCoverage::Partial; + } + + pub(crate) fn finish(&mut self, abort: Option) { + if self.execution == HealExecutionOutcome::Aborted(HealAbortReason::Cancelled) { + return; + } + let abort = abort.or(self.untraversable.then_some(HealAbortReason::Untraversable)); + self.execution = match abort { + Some(reason) => HealExecutionOutcome::Aborted(reason), + None if self.counters.failed > 0 => HealExecutionOutcome::CompletedWithErrors, + None => HealExecutionOutcome::Completed, + }; + self.coverage = if abort.is_none() && !self.counters.overflowed { + HealTraversalCoverage::Complete + } else { + HealTraversalCoverage::Partial + }; + } + + pub(crate) fn record(&mut self, mut item: HealObjectOutcome) { + use super::progress::increment_counter; + let counters = &mut self.counters; + counters.overflowed |= !increment_counter(&mut counters.processed); + let counter = match item.disposition { + HealObjectDisposition::Repaired => &mut counters.healed, + HealObjectDisposition::VerifiedHealthy | HealObjectDisposition::AuthoritativelyAbsent => &mut counters.unchanged, + HealObjectDisposition::Failed(_) => &mut counters.failed, + HealObjectDisposition::Unknown => { + counters.overflowed |= !increment_counter(&mut counters.unknown); + &mut counters.skipped + } + _ => &mut counters.skipped, + }; + counters.overflowed |= !increment_counter(counter); + if let Some(detail) = &mut item.detail { + let mut end = detail.len().min(MAX_OUTCOME_DETAIL_BYTES); + while !detail.is_char_boundary(end) { + end -= 1; + } + self.objects_truncated |= end < detail.len(); + detail.truncate(end); + detail.shrink_to_fit(); + } + let bytes = item.retained_bytes(); + if bytes > MAX_OUTCOME_BYTES { + self.objects_truncated = true; + return; + } + while self.objects.len() >= MAX_OUTCOME_ITEMS || self.retained_object_bytes.saturating_add(bytes) > MAX_OUTCOME_BYTES { + let Some(oldest) = self.objects.pop_front() else { break }; + self.retained_object_bytes = self.retained_object_bytes.saturating_sub(oldest.retained_bytes()); + self.objects_truncated = true; + } + self.retained_object_bytes = self.retained_object_bytes.saturating_add(bytes); + self.objects.push_back(item); + } + + pub(crate) fn retained_bytes(&self) -> usize { + size_of::() + .saturating_add(self.retained_object_bytes) + .saturating_add(self.objects.capacity().saturating_mul(size_of::())) + } +} + +#[cfg(test)] +mod canonical_outcome_tests { + use super::*; + + fn item(disposition: HealObjectDisposition) -> HealObjectOutcome { + HealObjectOutcome { + identity: HealObjectIdentity { + kind: HealObjectKind::Object, + bucket: "bucket".to_string(), + object: "object".to_string(), + version_id: None, + bucket_incarnation_id: None, + pool_index: None, + set_index: None, + }, + disposition, + detail: None, + } + } + + #[test] + fn canonical_outcome_categories_have_one_terminal_count() { + let mut outcome = HealTaskOutcome::default(); + for disposition in [ + HealObjectDisposition::Unknown, + HealObjectDisposition::Repaired, + HealObjectDisposition::VerifiedHealthy, + HealObjectDisposition::AuthoritativelyAbsent, + HealObjectDisposition::Deferred { + reason: HealDeferredReason::DanglingDeleteGrace, + retry_not_before: None, + }, + HealObjectDisposition::Failed(HealFailureClass::Permanent), + HealObjectDisposition::Cancelled, + HealObjectDisposition::DryRunObserved, + ] { + outcome.record(item(disposition)); + } + let c = &outcome.counters; + assert_eq!((c.processed, c.healed, c.unchanged, c.skipped, c.failed, c.unknown), (8, 1, 2, 4, 1, 1)); + assert_eq!(c.processed, c.healed + c.unchanged + c.skipped + c.failed); + } + + #[test] + fn canonical_outcome_window_count_bytes_and_oversize_keep_total_counts() { + let mut outcome = HealTaskOutcome::default(); + for _ in 0..MAX_OUTCOME_ITEMS { + outcome.record(item(HealObjectDisposition::Unknown)); + } + assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS); + assert!(!outcome.objects_truncated); + outcome.record(item(HealObjectDisposition::Unknown)); + assert_eq!(outcome.objects.len(), MAX_OUTCOME_ITEMS); + assert!(outcome.objects_truncated); + let mut oversized = item(HealObjectDisposition::Failed(HealFailureClass::Permanent)); + oversized.identity.object = "x".repeat(MAX_OUTCOME_BYTES); + outcome.record(oversized); + assert_eq!(outcome.counters.processed, u64::try_from(MAX_OUTCOME_ITEMS + 2).expect("bounded count")); + assert_eq!(outcome.counters.failed, 1); + assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES); + for _ in 0..MAX_OUTCOME_ITEMS { + let mut failed = item(HealObjectDisposition::Failed(HealFailureClass::Permanent)); + failed.detail = Some("\u{4fee}".repeat(MAX_OUTCOME_DETAIL_BYTES)); + outcome.record(failed); + } + assert!(outcome.retained_object_bytes <= MAX_OUTCOME_BYTES); + assert!(outcome.objects.iter().all(|item| { + item.detail + .as_ref() + .is_none_or(|detail| detail.len() <= MAX_OUTCOME_DETAIL_BYTES) + })); + assert!(outcome.objects.len() < MAX_OUTCOME_ITEMS); + } + + #[test] + fn canonical_outcome_counter_overflow_cannot_claim_complete_coverage() { + let mut outcome = HealTaskOutcome::default(); + outcome.counters.processed = u64::MAX; + outcome.record(item(HealObjectDisposition::Unknown)); + outcome.finish(None); + assert!(outcome.counters.overflowed); + assert_eq!(outcome.counters.processed, u64::MAX); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + } +} diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index c4c1d902b..a1b09c3b7 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -15,6 +15,10 @@ use crate::heal::{ DiskError, EcstoreError, ErasureSetHealer, HealDiskExt as _, erasure_healer::target_outcomes_complete, + outcome::{ + HealAbortReason, HealDeferredReason, HealFailureClass, HealObjectDisposition, HealObjectIdentity, HealObjectKind, + HealObjectOutcome, HealTaskOutcome, + }, progress::HealProgress, resume::{ CheckpointManager, ReplacementPhase, ReplacementTargetIdentity, ResumeManager, replacement_target_identities_match, @@ -43,6 +47,26 @@ use uuid::Uuid; use super::{BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME, RUSTFS_META_BUCKET}; +#[cfg(test)] +pub(crate) struct OutcomeFinishTestHook { + pub(crate) task_id: String, + pub(crate) reached: tokio::sync::Notify, + pub(crate) release: tokio::sync::Notify, +} + +#[cfg(test)] +pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock>>> = + std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None)); + +#[cfg(test)] +async fn pause_outcome_finish(task_id: &str) { + let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone(); + if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) { + hook.reached.notify_one(); + hook.release.notified().await; + } +} + const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_TASK: &str = "task"; const LOG_SUBSYSTEM_OBJECT: &str = "object"; @@ -394,6 +418,7 @@ pub struct HealTask { pub status: Arc>, /// Progress tracking pub progress: Arc>, + outcome: Arc>, /// Result items collected from storage heal calls, each stamped with a /// monotonically increasing sequence number for incremental consumption /// (the client passes the last seen seq back and receives only newer @@ -460,6 +485,7 @@ impl HealTask { result_items_truncated: Arc::new(AtomicBool::new(false)), batch_failure: Arc::new(RwLock::new(None)), batch_failure_recorded: Arc::new(AtomicBool::new(false)), + outcome: Arc::new(RwLock::new(HealTaskOutcome::default())), created_at: request.created_at, enqueued_at: request.enqueued_at, started_at: Arc::new(RwLock::new(None)), @@ -507,6 +533,66 @@ impl HealTask { self.heal_type.kind_label() } + pub async fn get_outcome(&self) -> HealTaskOutcome { + self.outcome.read().await.clone() + } + + fn outcome_identity( + &self, + bucket: &str, + object: &str, + version_id: Option<&str>, + pool_index: Option, + set_index: Option, + ) -> HealObjectIdentity { + HealObjectIdentity { + kind: match self.heal_type { + HealType::Metadata { .. } => HealObjectKind::Metadata, + HealType::ECDecode { .. } => HealObjectKind::Decode, + _ => HealObjectKind::Object, + }, + bucket: bucket.to_owned(), + object: object.to_owned(), + version_id: version_id.map(ToOwned::to_owned), + bucket_incarnation_id: None, + pool_index, + set_index, + } + } + + fn single_object_identity(&self) -> Option { + let (bucket, object, version) = match &self.heal_type { + HealType::Object { + bucket, + object, + version_id, + } + | HealType::ECDecode { + bucket, + object, + version_id, + } => (bucket, object, version_id.as_deref()), + HealType::Metadata { bucket, object } => (bucket, object, None), + _ => return None, + }; + Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index)) + } + + async fn record_deferred_object(&self, reason: HealDeferredReason) { + if let Some(identity) = self.single_object_identity() { + let mut outcome = self.outcome.write().await; + outcome.attempt_failed(); + outcome.record(HealObjectOutcome { + identity, + disposition: HealObjectDisposition::Deferred { + reason, + retry_not_before: None, + }, + detail: None, + }); + } + } + pub(crate) fn has_batch_failure(&self) -> bool { self.batch_failure_recorded.load(Ordering::Acquire) } @@ -634,6 +720,7 @@ impl HealTask { } async fn skip_due_to_transient_object_exists(&self, bucket: &str, object: &str, err: &Error) -> Result<()> { + self.record_deferred_object(HealDeferredReason::TransientExistenceCheck).await; warn!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_RESULT, @@ -733,6 +820,8 @@ impl HealTask { return false; } + self.record_deferred_object(HealDeferredReason::TransientUsageCache).await; + warn!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_RESULT, @@ -755,6 +844,8 @@ impl HealTask { return false; } + self.record_deferred_object(HealDeferredReason::DanglingDeleteGrace).await; + warn!( target: "rustfs::heal::task", event = EVENT_HEAL_OBJECT_RESULT, @@ -801,6 +892,7 @@ impl HealTask { #[tracing::instrument(skip(self), fields(task_id = %self.id, heal_type = ?self.heal_type))] #[hotpath::measure] pub async fn execute(&self) -> Result<()> { + self.outcome.write().await.start(); // update status and timestamps atomically to avoid race conditions let now = SystemTime::now(); let start_instant = Instant::now(); @@ -860,6 +952,45 @@ impl HealTask { HealType::ErasureSet { buckets, set_disk_id } => self.heal_erasure_set(buckets.clone(), set_disk_id.clone()).await, }; + #[cfg(test)] + pause_outcome_finish(&self.id).await; + { + let mut outcome = self.outcome.write().await; + if outcome.counters.processed == 0 + && let Some(identity) = self.single_object_identity() + { + let disposition = match &result { + Ok(()) if self.options.dry_run => HealObjectDisposition::DryRunObserved, + Ok(()) => HealObjectDisposition::Unknown, + Err(Error::TaskCancelled) => HealObjectDisposition::Cancelled, + Err(Error::TaskTimeout) => HealObjectDisposition::Deferred { + reason: HealDeferredReason::Deadline, + retry_not_before: None, + }, + Err(error) => { + outcome.attempt_failed(); + HealObjectDisposition::Failed(if error.is_recoverable_heal() { + HealFailureClass::Recoverable + } else { + HealFailureClass::Permanent + }) + } + }; + outcome.record(HealObjectOutcome { + identity, + disposition, + detail: result.as_ref().err().map(ToString::to_string), + }); + } + let abort = match &result { + Err(Error::TaskCancelled) => Some(HealAbortReason::Cancelled), + Err(Error::TaskTimeout) => Some(HealAbortReason::Deadline), + Err(_) if !self.has_batch_failure() && !self.heal_type.is_per_object() => Some(HealAbortReason::Untraversable), + _ => None, + }; + outcome.finish(abort); + } + // update completed time and status { let mut completed_at = self.completed_at.write().await; @@ -944,6 +1075,7 @@ impl HealTask { pub async fn cancel(&self) -> Result<()> { self.cancel_token.cancel(); + self.outcome.write().await.finish(Some(HealAbortReason::Cancelled)); let mut status = self.status.write().await; *status = HealTaskStatus::Cancelled; debug!( diff --git a/crates/heal/src/heal/task/heal_bucket.rs b/crates/heal/src/heal/task/heal_bucket.rs index d44df549e..c6225c844 100644 --- a/crates/heal/src/heal/task/heal_bucket.rs +++ b/crates/heal/src/heal/task/heal_bucket.rs @@ -214,6 +214,7 @@ impl HealTask { continue; } failed = failed.saturating_add(1); + self.outcome.write().await.mark_untraversable(); if err.is_recoverable_heal() { retryable = retryable.saturating_add(1); } else { @@ -260,6 +261,7 @@ impl HealTask { #[hotpath::measure] async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> { + let previous_progress = self.get_progress().await; let mut scanned = 0u64; let mut healed = 0u64; let mut failed = 0u64; @@ -304,23 +306,47 @@ impl HealTask { let mut continuation_token: Option = None; loop { self.check_control_flags().await?; - let (objects, next_token, is_truncated) = if let Some(set_disk_id) = set_disk_id.as_deref() { - self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk( - set_disk_id, - bucket, - prefix, - continuation_token.as_deref(), - false, - )) - .await? - } else { - self.await_with_control(self.storage.list_objects_for_heal_page( - bucket, - prefix, - continuation_token.as_deref(), - false, - )) - .await? + let mut listing_attempt = 0; + let (objects, next_token, is_truncated) = loop { + let page = if let Some(set_disk_id) = set_disk_id.as_deref() { + self.await_with_control(self.storage.list_versions_for_heal_page_disk_walk( + set_disk_id, + bucket, + prefix, + continuation_token.as_deref(), + false, + )) + .await + } else { + self.await_with_control(self.storage.list_objects_for_heal_page( + bucket, + prefix, + continuation_token.as_deref(), + false, + )) + .await + }; + match page { + Ok(page) => break page, + Err(error @ (Error::TaskCancelled | Error::TaskTimeout)) => return Err(error), + Err(error) => { + self.outcome.write().await.attempt_failed(); + if error.is_recoverable_heal() && listing_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES { + listing_attempt += 1; + self.await_with_control(async { + tokio::time::sleep(self.bucket_object_retry_delay(listing_attempt)).await; + Ok(()) + }) + .await?; + continue; + } + self.outcome.write().await.mark_untraversable(); + return Err(Error::HealListingFailed { + bucket: bucket.to_string(), + source: Box::new(error), + }); + } + } }; let mut pending = objects; @@ -338,6 +364,14 @@ impl HealTask { self.check_control_flags().await?; let mut telemetry_unknown = false; let object = item.name.as_str(); + let identity = + self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set); + let mut disposition = if heal_opts.dry_run { + HealObjectDisposition::DryRunObserved + } else { + HealObjectDisposition::Unknown + }; + let mut detail = None; { let mut progress = self.progress.write().await; progress.set_current_object(Some(format!("{bucket}/{object}"))); @@ -380,7 +414,31 @@ impl HealTask { }; if let Some(err) = error { + match err { + Error::TaskCancelled | Error::TaskTimeout => { + let disposition = if matches!(err, Error::TaskCancelled) { + HealObjectDisposition::Cancelled + } else { + HealObjectDisposition::Deferred { + reason: HealDeferredReason::Deadline, + retry_not_before: None, + } + }; + self.outcome.write().await.record(HealObjectOutcome { + identity, + disposition, + detail: None, + }); + return Err(err); + } + _ => self.outcome.write().await.attempt_failed(), + } + detail = Some(err.to_string()); if Self::is_dangling_delete_grace_error(&err) { + disposition = HealObjectDisposition::Deferred { + reason: HealDeferredReason::DanglingDeleteGrace, + retry_not_before: None, + }; telemetry_unknown |= !increment_counter(&mut skipped); warn!( target: "rustfs::heal::task", @@ -395,6 +453,10 @@ impl HealTask { "Heal bucket object dangling cleanup deferred by grace window" ); } else if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) { + disposition = HealObjectDisposition::Deferred { + reason: HealDeferredReason::TransientUsageCache, + retry_not_before: None, + }; telemetry_unknown |= !increment_counter(&mut skipped); warn!( target: "rustfs::heal::task", @@ -425,6 +487,11 @@ impl HealTask { ); retry.push(item); } else { + disposition = HealObjectDisposition::Failed(if err.is_recoverable_heal() { + HealFailureClass::RetryExhausted + } else { + HealFailureClass::Permanent + }); telemetry_unknown |= !increment_counter(&mut failed); if err.is_recoverable_heal() { retryable_failed = retryable_failed.saturating_add(1); @@ -459,8 +526,20 @@ impl HealTask { continue; } + self.outcome.write().await.record(HealObjectOutcome { + identity, + disposition, + detail, + }); + let mut progress = self.progress.write().await; - progress.update_object_progress(scanned, healed, failed, skipped, bytes); + progress.update_object_progress( + previous_progress.objects_scanned.saturating_add(scanned), + previous_progress.objects_healed.saturating_add(healed), + previous_progress.objects_failed.saturating_add(failed), + previous_progress.skipped_objects.saturating_add(skipped), + previous_progress.bytes_processed.saturating_add(bytes), + ); if telemetry_unknown { progress.mark_unknown(); } @@ -475,7 +554,7 @@ impl HealTask { continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?; if continuation_token.is_none() { - // Truncated but no continuation token: end of listing. + // Truncated without a continuation token is a compatibility EOF. break; } } diff --git a/crates/heal/src/heal/task/heal_metadata.rs b/crates/heal/src/heal/task/heal_metadata.rs index fe2dc98ac..fad5a403e 100644 --- a/crates/heal/src/heal/task/heal_metadata.rs +++ b/crates/heal/src/heal/task/heal_metadata.rs @@ -261,8 +261,8 @@ impl HealTask { update_parity: true, no_lock: self.options.no_lock, read_repair: false, - pool: None, - set: None, + pool: self.options.pool_index, + set: self.options.set_index, }; let heal_result = self diff --git a/crates/heal/src/heal/task/tests.rs b/crates/heal/src/heal/task/tests.rs index 1734d0c7a..8cabf1214 100644 --- a/crates/heal/src/heal/task/tests.rs +++ b/crates/heal/src/heal/task/tests.rs @@ -14,6 +14,364 @@ use super::super::{DiskOption, DiskStore, Endpoint, new_disk}; use super::*; + +mod canonical_outcome { + use super::*; + use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage}; + + fn bucket_task(storage: Arc) -> HealTask { + HealTask::from_request( + HealRequest::new( + HealType::Bucket { + bucket: "bucket-a".to_string(), + }, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage, + ) + } + + #[tokio::test(start_paused = true)] + async fn cluster_retries_only_the_failed_listing_page() { + let storage = Arc::new(MockStorage { + recoverable_second_page_failures: Mutex::new(Some(1)), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage.clone(), + ); + task.execute().await.expect("second-page retry succeeds"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!(outcome.counters.processed, 2); + assert_eq!(outcome.counters.attempt_failures, 1); + assert_eq!(task.get_progress().await.objects_scanned, 2); + assert_eq!( + storage.heal_object_calls.lock().expect("object calls").as_slice(), + ["object-a", "object-b"] + ); + assert_eq!( + storage.listing_tokens.lock().expect("listing tokens").as_slice(), + [None, Some("second".to_string()), Some("second".to_string())] + ); + } + + #[tokio::test(start_paused = true)] + async fn exhausted_listing_page_cannot_restart_the_bucket() { + let storage = Arc::new(MockStorage { + recoverable_second_page_failures: Mutex::new(Some(4)), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage.clone(), + ); + task.execute().await.expect_err("listing page budget exhausted"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable)); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!(outcome.counters.processed, 1); + assert_eq!(outcome.counters.attempt_failures, 4); + assert_eq!(task.get_progress().await.objects_scanned, 1); + assert_eq!(storage.heal_object_calls.lock().expect("object calls").as_slice(), ["object-a"]); + assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]); + } + + #[tokio::test] + async fn listing_failure_preserves_processed_objects_and_partial_coverage() { + let storage = Arc::new(MockStorage { + fail_second_listing_page: true, + ..Default::default() + }); + let task = bucket_task(storage); + task.execute().await.expect_err("second page cannot be traversed"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable)); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!(outcome.counters.processed, 1); + assert_eq!(outcome.objects[0].identity.object, "object-a"); + assert_eq!(task.get_progress().await.objects_scanned, 1); + } + + #[tokio::test] + async fn cluster_preserves_cumulative_progress_across_buckets() { + let storage = Arc::new(MockStorage { + list_each_bucket: true, + listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage, + ); + task.execute().await.expect("both buckets complete"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.counters.processed, 4); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + let progress = task.get_progress().await; + assert_eq!((progress.objects_scanned, progress.objects_healed), (4, 4)); + assert_eq!( + outcome + .objects + .iter() + .filter(|item| item.identity.bucket == "bucket-b") + .count(), + 2 + ); + } + + #[tokio::test(start_paused = true)] + async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() { + let storage = Arc::new(MockStorage::default()); + storage.heal_object_outcomes.lock().expect("outcomes").insert( + "object-a".to_string(), + (0..4).map(|_| MockHealObjectOutcome::RetryableReadQuorum).collect(), + ); + let task = bucket_task(storage.clone()); + task.execute().await.expect_err("legacy adapter retains batch failure"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!((outcome.counters.processed, outcome.counters.failed, outcome.counters.unknown), (2, 1, 1)); + assert_eq!(outcome.counters.attempt_failures, 4); + let failed = outcome + .objects + .iter() + .find(|item| item.identity.object == "object-a") + .expect("failed object"); + assert_eq!(failed.disposition, HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)); + let object_b_calls = { + let calls = storage.heal_object_calls.lock().expect("calls"); + calls.iter().filter(|object| object.as_str() == "object-b").count() + }; + assert_eq!(object_b_calls, 1); + let progress = task.get_progress().await; + assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1)); + } + + #[tokio::test(start_paused = true)] + async fn retry_success_counts_one_terminal_outcome() { + let storage = Arc::new(MockStorage::default()); + storage + .heal_object_outcomes + .lock() + .expect("outcomes") + .insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::RetryableReadQuorum])); + let task = bucket_task(storage); + task.execute().await.expect("retry should recover"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.counters.processed, 2); + assert_eq!(outcome.counters.failed, 0); + assert_eq!(outcome.counters.attempt_failures, 1); + assert_eq!( + outcome + .objects + .iter() + .filter(|item| item.identity.object == "object-a") + .count(), + 1 + ); + assert_eq!( + outcome.counters.processed, + outcome.counters.healed + outcome.counters.unchanged + outcome.counters.skipped + outcome.counters.failed + ); + } + + #[tokio::test] + async fn mixed_grace_and_legacy_success_keep_distinct_dispositions() { + let storage = Arc::new(MockStorage::default()); + storage + .heal_object_outcomes + .lock() + .expect("outcomes") + .insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::DanglingGraceDeferred])); + let task = bucket_task(storage); + task.execute().await.expect("grace permits traversal completion"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!(outcome.counters.processed, 2); + assert_eq!(outcome.counters.healed, 0, "legacy result is not a repair receipt"); + assert!(matches!( + outcome.objects[0].disposition, + HealObjectDisposition::Deferred { + reason: HealDeferredReason::DanglingDeleteGrace, + .. + } + )); + assert_eq!(outcome.objects[1].disposition, HealObjectDisposition::Unknown); + assert!( + outcome + .objects + .iter() + .all(|item| item.identity.bucket_incarnation_id.is_none()) + ); + assert_eq!( + task.get_progress().await.objects_healed, + 1, + "legacy display count remains distinct from proof" + ); + } + + #[tokio::test] + async fn grace_single_object_is_completed_but_deferred() { + let storage = Arc::new(MockStorage { + heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::DanglingGraceDeferred)), + ..Default::default() + }); + let task = HealTask::from_request(HealRequest::object("bucket-a".to_string(), "recent.txt".to_string(), None), storage); + task.execute().await.expect("grace is deferred"); + let outcome = task.get_outcome().await; + assert_eq!(task.get_status().await, HealTaskStatus::Completed); + assert_eq!(outcome.counters.processed, 1); + assert!(matches!( + outcome.objects[0].disposition, + HealObjectDisposition::Deferred { + reason: HealDeferredReason::DanglingDeleteGrace, + .. + } + )); + assert_eq!(outcome.counters.attempt_failures, 1); + } + + #[tokio::test] + async fn dry_run_and_transient_existence_do_not_prove_repair() { + for transient in [false, true] { + let storage = Arc::new(MockStorage::default()); + if transient { + storage + .object_exists_by_name + .lock() + .expect("existence fixture") + .insert("object".to_string(), MockObjectExists::TransientSkip("retry later")); + } + let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None); + request.options.dry_run = !transient; + let task = HealTask::from_request(request, storage); + task.execute().await.expect("observation may complete"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.counters.healed, 0); + if transient { + assert!(matches!( + outcome.objects[0].disposition, + HealObjectDisposition::Deferred { + reason: HealDeferredReason::TransientExistenceCheck, + .. + } + )); + } else { + assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved); + } + } + } + + #[tokio::test] + async fn untraversable_bucket_does_not_claim_complete_cluster_coverage() { + let storage = Arc::new(MockStorage { + listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])), + bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from(["metadata unavailable"]))])), + ..Default::default() + }); + let task = HealTask::from_request( + HealRequest::new( + HealType::Cluster, + HealOptions { + recursive: true, + timeout: None, + ..Default::default() + }, + HealPriority::Normal, + ), + storage.clone(), + ); + task.execute().await.expect_err("structural bucket error"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable)); + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!( + storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), + ["bucket-a", "bucket-b"] + ); + } + + #[tokio::test(start_paused = true)] + async fn cancellation_and_deadline_leave_partial_coverage() { + for cancel in [false, true] { + let storage = Arc::new(MockStorage { + block_heal_object: Mutex::new(true), + ..Default::default() + }); + let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None); + request.options.timeout = Some(Duration::from_secs(1)); + let task = HealTask::from_request(request, storage); + if cancel { + task.cancel().await.expect("cancel request"); + } + task.execute().await.expect_err("control interruption"); + let outcome = task.get_outcome().await; + assert_eq!(outcome.coverage, HealTraversalCoverage::Partial); + assert_eq!( + outcome.execution, + HealExecutionOutcome::Aborted(if cancel { + HealAbortReason::Cancelled + } else { + HealAbortReason::Deadline + }) + ); + } + } + + #[tokio::test] + async fn decode_keeps_the_requested_pool_and_set() { + let storage = Arc::new(MockStorage::default()); + let mut request = HealRequest::ec_decode("bucket-a".to_string(), "object".to_string(), Some("version-a".to_string())); + request.options.pool_index = Some(2); + request.options.set_index = Some(3); + let task = HealTask::from_request(request, storage.clone()); + task.execute().await.expect("decode fixture"); + let pool_and_set = { + let options = storage.object_heal_opts.lock().expect("storage options"); + (options[0].pool, options[0].set) + }; + assert_eq!(pool_and_set, (Some(2), Some(3))); + let outcome = task.get_outcome().await; + let identity = &outcome.objects[0].identity; + assert_eq!((identity.pool_index, identity.set_index), (Some(2), Some(3))); + assert_eq!(identity.version_id.as_deref(), Some("version-a")); + assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown); + } +} use crate::heal::storage::{HealListItem, HealObjectInfo}; use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; @@ -582,6 +940,10 @@ async fn verified_recovery_keeps_state_when_marker_clear_fails() { #[derive(Default)] struct MockStorage { listed: Mutex, + list_each_bucket: bool, + fail_second_listing_page: bool, + recoverable_second_page_failures: Mutex>, + listing_tokens: Mutex>>, healed_objects: Mutex>, heal_object_calls: Mutex>, heal_object_version_ids: Mutex>>, @@ -995,12 +1357,41 @@ impl HealStorageAPI for MockStorage { _include_lifecycle_object_info: bool, ) -> Result<(Vec, Option, bool)> { self.listed_prefixes.lock().unwrap().push(prefix.to_string()); + self.listing_tokens + .lock() + .expect("listing tokens") + .push(continuation_token.map(ToOwned::to_owned)); + if let Some(remaining) = self + .recoverable_second_page_failures + .lock() + .expect("listing failures") + .as_mut() + { + if continuation_token.is_none() { + return Ok((vec![heal_item("object-a")], Some("second".to_string()), true)); + } + if *remaining > 0 { + *remaining -= 1; + return Err(Error::Storage(EcstoreError::InsufficientReadQuorum( + bucket.to_string(), + "page".to_string(), + ))); + } + return Ok((vec![heal_item("object-b")], None, false)); + } + if self.fail_second_listing_page { + return if continuation_token.is_none() { + Ok((vec![heal_item("object-a")], Some("next-page".to_string()), true)) + } else { + Err(Error::other("listing unavailable")) + }; + } if *self.truncate_without_token.lock().unwrap() { return Ok((vec![heal_item("object-a")], None, true)); } let mut listed = self.listed.lock().unwrap(); - if continuation_token.is_none() && !*listed { + if continuation_token.is_none() && (!*listed || self.list_each_bucket) { *listed = true; let objects = if bucket == RUSTFS_META_BUCKET { vec![ @@ -1393,6 +1784,8 @@ async fn test_recursive_bucket_heal_skips_object_dir_candidates() { #[tokio::test] async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() { + use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage}; + // A version listing can report the final page as truncated with no // continuation token. That is treated as end-of-listing (not an error), // so the returned page is healed and the pass terminates cleanly instead @@ -1414,10 +1807,16 @@ async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() { ); let task = HealTask::from_request(request, storage.clone()); - task.heal_bucket("bucket-a") + task.execute() .await .expect("truncated-without-token must terminate cleanly, not loop or error"); + assert_eq!(task.get_status().await, HealTaskStatus::Completed); + let outcome = task.get_outcome().await; + assert_eq!(outcome.execution, HealExecutionOutcome::Completed); + assert_eq!(outcome.coverage, HealTraversalCoverage::Complete); + assert_eq!(outcome.counters.processed, 1); + assert_eq!( storage.healed_objects.lock().unwrap().as_slice(), ["object-a".to_string()], diff --git a/crates/lifecycle/src/core.rs b/crates/lifecycle/src/core.rs index bfa2b93c1..c157156e3 100644 --- a/crates/lifecycle/src/core.rs +++ b/crates/lifecycle/src/core.rs @@ -18,7 +18,7 @@ use s3s::dto::{ BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, NoncurrentVersionTransition, ObjectLockConfiguration, ObjectLockEnabled, RestoreRequest, Transition, }; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use time::macros::offset; use time::{self, Duration, OffsetDateTime}; @@ -65,6 +65,66 @@ const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str = "Rule with ExpiredObjectDeleteMarker cannot have tags based filtering"; const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration"; const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element."; +const ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS: &str = "'NewerNoncurrentVersions' must be a non-negative integer"; +const ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES: &str = + "Filter must have at most one of Prefix, Tag, ObjectSizeGreaterThan, ObjectSizeLessThan or And; combine predicates with And"; +const ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES: &str = "Filter And must contain at least two predicates"; +const ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY: &str = "Filter must not repeat a tag key"; +const ERR_LIFECYCLE_FILTER_INVALID_TAG: &str = "Tag key must be 1-128 characters and tag value must be at most 256 characters"; +const ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE: &str = "ObjectSizeGreaterThan and ObjectSizeLessThan must not be negative"; +const ERR_LIFECYCLE_FILTER_SIZE_RANGE: &str = "ObjectSizeGreaterThan must be smaller than ObjectSizeLessThan"; +/// Longest tag key S3 accepts. +const MAX_TAG_KEY_LEN: usize = 128; +/// Longest tag value S3 accepts. +const MAX_TAG_VALUE_LEN: usize = 256; + +/// A validation failure that the S3 boundary must answer with `MalformedXML` +/// rather than `InvalidArgument`: the document does not match the published +/// schema shape (wrong number of `Filter` predicates, a one-member `And`). +/// +/// Everything else stays [`std::io::ErrorKind::Other`], which the boundary +/// already maps to `InvalidArgument`. +pub const LIFECYCLE_MALFORMED_XML_ERROR_KIND: std::io::ErrorKind = std::io::ErrorKind::InvalidData; + +/// A persisted rule that could never have passed validation. Callers that can +/// report an error surface it; evaluation itself stays fail-closed and takes +/// no action for the rule. +pub const LIFECYCLE_CORRUPT_RULE_ERROR_KIND: std::io::ErrorKind = std::io::ErrorKind::InvalidData; + +fn malformed_xml_error(message: &'static str) -> std::io::Error { + std::io::Error::new(LIFECYCLE_MALFORMED_XML_ERROR_KIND, message) +} + +/// The retention count a rule keeps, or `None` when the persisted value is +/// negative — a shape PUT validation rejects, so reaching it means the rule +/// came from older persistence or an import. +/// +/// A negative count must never be read as "retain everything": that is how an +/// invalid configuration silently stopped deleting versions (backlog#2201). +pub fn retained_noncurrent_versions(count: i32) -> Option { + usize::try_from(count).ok() +} + +/// Does any rule carry a retention count that validation would have rejected? +pub fn lifecycle_has_corrupt_retention_count(lc: &BucketLifecycleConfiguration) -> bool { + lc.rules.iter().any(rule_has_corrupt_retention_count) +} + +fn rule_has_corrupt_retention_count(rule: &LifecycleRule) -> bool { + let expiration_count = rule + .noncurrent_version_expiration + .as_ref() + .and_then(|expiration| expiration.newer_noncurrent_versions); + let transition_counts = rule + .noncurrent_version_transitions + .iter() + .flatten() + .filter_map(|transition| transition.newer_noncurrent_versions); + expiration_count + .into_iter() + .chain(transition_counts) + .any(|count| retained_noncurrent_versions(count).is_none()) +} pub use rustfs_scanner_metrics::metrics::IlmAction; @@ -141,6 +201,17 @@ impl RuleValidate for LifecycleRule { return Err(std::io::Error::other(ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT)); } + if let Some(filter) = self.filter.as_ref() { + validate_lifecycle_filter(filter)?; + } + + // A negative retention count was accepted and then read as "retain + // (almost) everything" during evaluation, so an HTTP-accepted rule + // silently stopped deleting versions (backlog#2201). + if rule_has_corrupt_retention_count(self) { + return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS)); + } + // Rule with DelMarkerExpiration cannot have tags based filtering let has_tag_filter = self .filter @@ -173,11 +244,14 @@ impl RuleValidate for LifecycleRule { // Rule must have at least one action let has_expiration = self.expiration.is_some(); let has_transition = self.transitions.as_ref().is_some_and(|t| !t.is_empty()); - let has_noncurrent_expiration = self - .noncurrent_version_expiration - .as_ref() - .and_then(|e| e.noncurrent_days) - .is_some(); + // `NewerNoncurrentVersions` on its own is a MinIO extension, not an AWS + // form: it keeps the newest N noncurrent versions and expires the rest + // with no age condition. RustFS accepts it for MinIO compatibility, so + // it has to count as an action here — otherwise a count-only rule was + // rejected as actionless (backlog#2201). + let has_noncurrent_expiration = self.noncurrent_version_expiration.as_ref().is_some_and(|expiration| { + expiration.noncurrent_days.is_some() || expiration.newer_noncurrent_versions.is_some_and(|count| count > 0) + }); let has_noncurrent_transition = self .noncurrent_version_transitions .as_ref() @@ -203,6 +277,81 @@ impl RuleValidate for LifecycleRule { } } +/// Structural validation for `LifecycleRuleFilter`. +/// +/// The generated DTO is all-`Option`, so the S3 schema constraints have to be +/// checked here: at most one top-level predicate, an `And` that actually +/// combines at least two, no repeated tag key, tag key/value limits, and a +/// coherent non-negative size range (backlog#2201). +/// +/// A filter with no predicate at all stays valid: AWS documents an empty +/// `Filter` as "applies to every object in the bucket", and rejecting it would +/// break the most common way to write an unconditional rule. +fn validate_lifecycle_filter(filter: &LifecycleRuleFilter) -> Result<(), std::io::Error> { + let top_level_predicates = usize::from(filter.prefix.is_some()) + + usize::from(filter.tag.is_some()) + + usize::from(filter.object_size_greater_than.is_some()) + + usize::from(filter.object_size_less_than.is_some()) + + usize::from(filter.and.is_some()); + if top_level_predicates > 1 { + return Err(malformed_xml_error(ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES)); + } + + if let Some(tag) = filter.tag.as_ref() { + validate_lifecycle_tag(tag)?; + } + + if let Some(and) = filter.and.as_ref() { + let tags = and.tags.as_deref().unwrap_or(&[]); + let and_predicates = usize::from(and.prefix.is_some()) + + tags.len() + + usize::from(and.object_size_greater_than.is_some()) + + usize::from(and.object_size_less_than.is_some()); + if and_predicates < 2 { + return Err(malformed_xml_error(ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES)); + } + let mut seen_keys = HashSet::with_capacity(tags.len()); + for tag in tags { + validate_lifecycle_tag(tag)?; + let key = tag.key.as_deref().unwrap_or_default(); + if !seen_keys.insert(key) { + return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY)); + } + } + validate_lifecycle_size_bounds(and.object_size_greater_than, and.object_size_less_than)?; + } + + validate_lifecycle_size_bounds(filter.object_size_greater_than, filter.object_size_less_than)?; + + Ok(()) +} + +/// S3 requires a tag to carry a key and value; both are length-bounded. +/// The DTO makes both optional, so incomplete tags have to be rejected here +/// rather than silently matching nothing. +fn validate_lifecycle_tag(tag: &s3s::dto::Tag) -> Result<(), std::io::Error> { + let key = tag.key.as_deref().unwrap_or_default(); + let Some(value) = tag.value.as_deref() else { + return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_INVALID_TAG)); + }; + if key.is_empty() || key.chars().count() > MAX_TAG_KEY_LEN || value.chars().count() > MAX_TAG_VALUE_LEN { + return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_INVALID_TAG)); + } + Ok(()) +} + +fn validate_lifecycle_size_bounds(greater_than: Option, less_than: Option) -> Result<(), std::io::Error> { + if greater_than.is_some_and(|size| size < 0) || less_than.is_some_and(|size| size < 0) { + return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE)); + } + if let (Some(greater_than), Some(less_than)) = (greater_than, less_than) + && greater_than >= less_than + { + return Err(std::io::Error::other(ERR_LIFECYCLE_FILTER_SIZE_RANGE)); + } + Ok(()) +} + fn lifecycle_rule_prefix(rule: &LifecycleRule) -> Option<&str> { // Prefer a non-empty legacy prefix; treat an empty legacy prefix as if it were not set if let Some(p) = rule.prefix.as_deref() @@ -293,6 +442,10 @@ impl Lifecycle for BucketLifecycleConfiguration { { return true; } + // A positive count is an action on its own (the MinIO count-only + // form). Zero means "no count constraint" here, exactly as the + // batch limit path reads it, and a negative count is corrupt — + // neither makes the rule active (backlog#2201). if let Some(newer_noncurrent_versions) = rule_noncurrent_version_expiration.newer_noncurrent_versions && newer_noncurrent_versions > 0 { @@ -563,6 +716,23 @@ impl Lifecycle for BucketLifecycleConfiguration { if let Some(ref lc_rules) = self.filter_rules(obj).await { for rule in lc_rules.iter() { + // A retention count that PUT validation would have rejected can + // only come from older persistence or an import. Take no action + // for the rule instead of allowing another action on the same + // corrupt rule to delete or transition an object (backlog#2201). + if rule_has_corrupt_retention_count(rule) { + debug!( + event = EVENT_LIFECYCLE_NONCURRENT_EXPIRY_SKIPPED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + object = %obj.name, + rule_id = %rule.id.clone().unwrap_or_default(), + reason = "corrupt_newer_noncurrent_versions", + "Skipped lifecycle evaluation for a rule with an invalid retention count" + ); + continue; + } + if obj.is_latest && obj.expired_object_deletemarker() { if let Some(expiration) = rule.expiration.as_ref() && expiration.expired_object_delete_marker.is_some_and(|v| v) @@ -619,11 +789,18 @@ impl Lifecycle for BucketLifecycleConfiguration { if !obj.is_latest && let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration - && let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days + && (noncurrent_version_expiration.noncurrent_days.is_some() + || noncurrent_version_expiration + .newer_noncurrent_versions + .is_some_and(|count| count > 0)) && noncurrent_version_expiration .newer_noncurrent_versions .is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain)) { + // A count-only rule (MinIO extension) has no age condition: + // every version past the retained count is due as soon as it + // became noncurrent, i.e. zero days after the successor. + let noncurrent_days = noncurrent_version_expiration.noncurrent_days.unwrap_or(0); if let Some(successor_mod_time) = obj.successor_mod_time { let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days); if now.unix_timestamp() >= expected_expiry.unix_timestamp() { @@ -791,15 +968,18 @@ impl Lifecycle for BucketLifecycleConfiguration { for rule in filter_rules.iter() { if let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration { return if let Some(newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions { - if newer_noncurrent_versions == 0 { + // Zero means "no count constraint"; a negative count is + // corrupt and must not be read as "retain everything" + // (backlog#2201). Neither yields a limit event. + let Some(retained) = retained_noncurrent_versions(newer_noncurrent_versions).filter(|c| *c > 0) else { continue; - } + }; Event { action: IlmAction::DeleteVersionAction, rule_id: rule.id.clone().unwrap_or_default(), noncurrent_days: u32::try_from(noncurrent_version_expiration.noncurrent_days.unwrap_or(0)) .unwrap_or(u32::MAX), - newer_noncurrent_versions: usize::try_from(newer_noncurrent_versions).unwrap_or(usize::MAX), + newer_noncurrent_versions: retained, due: Some(OffsetDateTime::UNIX_EPOCH), storage_class: "".into(), } @@ -1162,7 +1342,11 @@ mod tests { use super::*; use metrics_util::MetricKind; use metrics_util::debugging::{DebugValue, DebuggingRecorder}; - use s3s::dto::{LifecycleRuleFilter, TransitionStorageClass}; + use s3s::dto::{ + LifecycleRuleAndOperator, LifecycleRuleFilter, NoncurrentVersionExpiration, NoncurrentVersionTransition, + TransitionStorageClass, + }; + use s3s::xml::{Deserialize as XmlDeserialize, SerializeContent as XmlSerializeContent}; use serial_test::serial; use std::sync::Arc; use time::macros::datetime; @@ -4183,6 +4367,583 @@ mod tests { assert_eq!(event.action, IlmAction::NoneAction); } + // ---- backlog#2201: retention-count and Filter invariants ----------------- + + fn rule_with_noncurrent_expiration(expiration: NoncurrentVersionExpiration) -> LifecycleRule { + LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("noncurrent".to_string()), + noncurrent_version_expiration: Some(expiration), + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + } + } + + fn rule_with_filter(filter: LifecycleRuleFilter) -> LifecycleRule { + LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(1), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: Some(filter), + id: Some("filtered".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + } + } + + fn config_with_rules(rules: Vec) -> BucketLifecycleConfiguration { + BucketLifecycleConfiguration { + expiry_updated_at: None, + rules, + } + } + + fn tag(key: &str, value: &str) -> s3s::dto::Tag { + s3s::dto::Tag { + key: Some(key.to_string()), + value: Some(value.to_string()), + } + } + + #[tokio::test] + async fn validate_rejects_negative_newer_noncurrent_versions() { + // A negative retention count used to be accepted and then read as + // usize::MAX during evaluation, so the rule silently stopped deleting + // versions (backlog#2201). + let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(30), + newer_noncurrent_versions: Some(-1), + })]); + + let err = lc + .validate(&ObjectLockConfiguration::default()) + .await + .expect_err("a negative retention count must be rejected"); + + assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_NEWER_NONCURRENT_VERSIONS); + assert_ne!(err.kind(), LIFECYCLE_MALFORMED_XML_ERROR_KIND, "value errors stay InvalidArgument"); + } + + #[tokio::test] + async fn validate_rejects_negative_newer_noncurrent_versions_on_transition() { + let mut rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(30), + newer_noncurrent_versions: None, + }); + rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition { + newer_noncurrent_versions: Some(-3), + noncurrent_days: Some(1), + storage_class: Some(TransitionStorageClass::from_static(TransitionStorageClass::GLACIER)), + }]); + + // The transition validator already refuses a negative count, and it runs + // first, so this pins the rejection rather than the message. The gap + // this PR closes is the expiration side, which had no such check. + config_with_rules(vec![rule]) + .validate(&ObjectLockConfiguration::default()) + .await + .expect_err("a negative retention count on a transition must be rejected"); + } + + #[tokio::test] + async fn zero_newer_noncurrent_versions_means_no_count_constraint() { + // Zero carries no constraint, matching how the batch limit path has + // always read it. Alongside an age condition the rule is valid; on its + // own it says nothing, so the rule has no action. + config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(30), + newer_noncurrent_versions: Some(0), + })]) + .validate(&ObjectLockConfiguration::default()) + .await + .expect("zero count alongside NoncurrentDays is valid"); + + let err = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: None, + newer_noncurrent_versions: Some(0), + })]) + .validate(&ObjectLockConfiguration::default()) + .await + .expect_err("a zero count on its own is not an action"); + assert_eq!(err.to_string(), ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION); + } + + #[tokio::test] + async fn validate_accepts_count_only_noncurrent_expiration() { + // MinIO extension: NewerNoncurrentVersions with no NoncurrentDays. It + // used to be rejected as an actionless rule (backlog#2201). + let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: None, + newer_noncurrent_versions: Some(2), + })]); + + lc.validate(&ObjectLockConfiguration::default()) + .await + .expect("a count-only noncurrent expiration rule is accepted"); + } + + #[tokio::test] + async fn eval_inner_expires_versions_beyond_count_only_retention() { + // Count-only rules have no age condition: everything past the retained + // count is due as soon as it became noncurrent. + let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: None, + newer_noncurrent_versions: Some(2), + })]); + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(datetime!(2025-01-15 10:30:45 UTC)), + successor_mod_time: Some(datetime!(2025-01-15 10:30:45 UTC)), + is_latest: false, + num_versions: 5, + ..Default::default() + }; + + // Rank 2 is the third-newest noncurrent version: past a retention of 2. + let expired = lc.eval_inner(&opts, datetime!(2025-01-15 10:30:46 UTC), 2).await; + assert_eq!(expired.action, IlmAction::DeleteVersionAction); + assert_eq!(expired.rule_id, "noncurrent"); + + // Rank 1 is still within the retained count. + let retained = lc.eval_inner(&opts, datetime!(2025-01-15 10:30:46 UTC), 1).await; + assert_eq!(retained.action, IlmAction::NoneAction); + } + + #[tokio::test] + #[serial] + async fn eval_inner_keeps_age_condition_when_count_and_days_are_set() { + // With both set, the count gates which versions are candidates and the + // age condition still decides when they are due. + with_default_ilm_process_time(|| {}); + let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(10), + newer_noncurrent_versions: Some(1), + })]); + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)), + successor_mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)), + is_latest: false, + num_versions: 3, + ..Default::default() + }; + + let too_young = lc.eval_inner(&opts, datetime!(2025-01-05 00:00:00 UTC), 2).await; + assert_eq!(too_young.action, IlmAction::NoneAction, "the age condition still applies"); + + let due = lc.eval_inner(&opts, datetime!(2025-01-20 00:00:00 UTC), 2).await; + assert_eq!(due.action, IlmAction::DeleteVersionAction); + } + + #[tokio::test] + async fn eval_inner_takes_no_action_for_a_corrupt_retention_count() { + // Reachable only from older persistence or an import; it must not be + // read as "retain everything", and it must not delete either. + let lc = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(1), + newer_noncurrent_versions: Some(-1), + })]); + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)), + successor_mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)), + is_latest: false, + num_versions: 3, + ..Default::default() + }; + + let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 2).await; + + assert_eq!(event.action, IlmAction::NoneAction); + } + + #[tokio::test] + async fn eval_inner_does_not_expire_latest_object_for_a_corrupt_retention_rule() { + let mut rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(1), + newer_noncurrent_versions: Some(-1), + }); + rule.expiration = Some(LifecycleExpiration { + days: Some(1), + ..Default::default() + }); + let lc = config_with_rules(vec![rule]); + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)), + is_latest: true, + ..Default::default() + }; + + let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 0).await; + + assert_eq!(event.action, IlmAction::NoneAction); + } + + #[tokio::test] + async fn eval_inner_does_not_delete_latest_marker_for_a_corrupt_retention_rule() { + let mut expired_marker_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(1), + newer_noncurrent_versions: Some(-1), + }); + expired_marker_rule.expiration = Some(LifecycleExpiration { + expired_object_delete_marker: Some(true), + ..Default::default() + }); + + let mut aged_marker_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(1), + newer_noncurrent_versions: Some(-1), + }); + aged_marker_rule.del_marker_expiration = Some(s3s::dto::DelMarkerExpiration { days: Some(1) }); + + for rule in [expired_marker_rule, aged_marker_rule] { + let lc = config_with_rules(vec![rule]); + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)), + version_id: Some(Uuid::new_v4()), + is_latest: true, + delete_marker: true, + num_versions: 1, + ..Default::default() + }; + + let event = lc.eval_inner(&opts, datetime!(2025-06-01 00:00:00 UTC), 0).await; + + assert_eq!(event.action, IlmAction::NoneAction); + } + } + + #[test] + fn corrupt_retention_count_is_detected_on_either_action() { + let mut transition_rule = rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(1), + newer_noncurrent_versions: Some(0), + }); + transition_rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition { + newer_noncurrent_versions: Some(-1), + noncurrent_days: Some(1), + storage_class: Some(TransitionStorageClass::from_static(TransitionStorageClass::GLACIER)), + }]); + + assert!(lifecycle_has_corrupt_retention_count(&config_with_rules(vec![ + rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(1), + newer_noncurrent_versions: Some(-1), + }) + ]))); + assert!(lifecycle_has_corrupt_retention_count(&config_with_rules(vec![transition_rule]))); + assert!(!lifecycle_has_corrupt_retention_count(&config_with_rules(vec![ + rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(1), + newer_noncurrent_versions: Some(3), + }) + ]))); + } + + #[test] + fn count_only_rules_are_active_only_for_a_positive_count() { + let positive = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: None, + newer_noncurrent_versions: Some(2), + })]); + assert!(positive.has_active_rules("")); + + let corrupt = config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: None, + newer_noncurrent_versions: Some(-1), + })]); + assert!(!corrupt.has_active_rules(""), "a corrupt retention count must not make a rule active"); + } + + #[tokio::test] + async fn noncurrent_versions_expiration_limit_ignores_a_corrupt_count() { + // The batch path must not read a negative count as "retain everything". + let lc = Arc::new(config_with_rules(vec![rule_with_noncurrent_expiration(NoncurrentVersionExpiration { + noncurrent_days: Some(1), + newer_noncurrent_versions: Some(-1), + })])); + let opts = ObjectOpts { + name: "obj".to_string(), + mod_time: Some(datetime!(2025-01-01 00:00:00 UTC)), + is_latest: false, + ..Default::default() + }; + + let event = lc.noncurrent_versions_expiration_limit(&opts).await; + + assert_eq!(event.action, IlmAction::NoneAction); + assert_eq!(event.newer_noncurrent_versions, 0); + } + + #[tokio::test] + async fn validate_covers_filter_invariants() { + struct Case { + name: &'static str, + filter: LifecycleRuleFilter, + expected: Option<(&'static str, std::io::ErrorKind)>, + } + + let cases = vec![ + Case { + // AWS documents an empty Filter as "every object in the bucket". + name: "empty filter applies to all objects", + filter: LifecycleRuleFilter::default(), + expected: None, + }, + Case { + name: "single prefix predicate", + filter: LifecycleRuleFilter { + prefix: Some("logs/".to_string()), + ..Default::default() + }, + expected: None, + }, + Case { + name: "two top-level predicates", + filter: LifecycleRuleFilter { + prefix: Some("logs/".to_string()), + tag: Some(tag("env", "prod")), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)), + }, + Case { + name: "prefix alongside And", + filter: LifecycleRuleFilter { + prefix: Some("logs/".to_string()), + and: Some(LifecycleRuleAndOperator { + prefix: Some("logs/".to_string()), + tags: Some(vec![tag("env", "prod")]), + ..Default::default() + }), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_TOO_MANY_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)), + }, + Case { + name: "And with a single member", + filter: LifecycleRuleFilter { + and: Some(LifecycleRuleAndOperator { + prefix: Some("logs/".to_string()), + ..Default::default() + }), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_AND_TOO_FEW_PREDICATES, LIFECYCLE_MALFORMED_XML_ERROR_KIND)), + }, + Case { + name: "And with two members", + filter: LifecycleRuleFilter { + and: Some(LifecycleRuleAndOperator { + prefix: Some("logs/".to_string()), + tags: Some(vec![tag("env", "prod")]), + ..Default::default() + }), + ..Default::default() + }, + expected: None, + }, + Case { + name: "And with two tags", + filter: LifecycleRuleFilter { + and: Some(LifecycleRuleAndOperator { + tags: Some(vec![tag("env", "prod"), tag("team", "storage")]), + ..Default::default() + }), + ..Default::default() + }, + expected: None, + }, + Case { + name: "And repeating a tag key", + filter: LifecycleRuleFilter { + and: Some(LifecycleRuleAndOperator { + tags: Some(vec![tag("env", "prod"), tag("env", "dev")]), + ..Default::default() + }), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_DUPLICATE_TAG_KEY, std::io::ErrorKind::Other)), + }, + Case { + name: "empty tag key", + filter: LifecycleRuleFilter { + tag: Some(tag("", "prod")), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)), + }, + Case { + name: "missing tag key", + filter: LifecycleRuleFilter { + tag: Some(s3s::dto::Tag { + key: None, + value: Some("prod".to_string()), + }), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)), + }, + Case { + name: "missing tag value", + filter: LifecycleRuleFilter { + tag: Some(s3s::dto::Tag { + key: Some("env".to_string()), + value: None, + }), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)), + }, + Case { + name: "empty tag value", + filter: LifecycleRuleFilter { + tag: Some(tag("env", "")), + ..Default::default() + }, + expected: None, + }, + Case { + name: "tag key at the limit", + filter: LifecycleRuleFilter { + tag: Some(tag(&"k".repeat(MAX_TAG_KEY_LEN), "prod")), + ..Default::default() + }, + expected: None, + }, + Case { + name: "tag key past the limit", + filter: LifecycleRuleFilter { + tag: Some(tag(&"k".repeat(MAX_TAG_KEY_LEN + 1), "prod")), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)), + }, + Case { + name: "tag value past the limit", + filter: LifecycleRuleFilter { + tag: Some(tag("env", &"v".repeat(MAX_TAG_VALUE_LEN + 1))), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_INVALID_TAG, std::io::ErrorKind::Other)), + }, + Case { + name: "negative ObjectSizeGreaterThan", + filter: LifecycleRuleFilter { + object_size_greater_than: Some(-1), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE, std::io::ErrorKind::Other)), + }, + Case { + name: "negative ObjectSizeLessThan", + filter: LifecycleRuleFilter { + object_size_less_than: Some(-5), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_NEGATIVE_SIZE, std::io::ErrorKind::Other)), + }, + Case { + name: "inverted size range inside And", + filter: LifecycleRuleFilter { + and: Some(LifecycleRuleAndOperator { + object_size_greater_than: Some(100), + object_size_less_than: Some(100), + ..Default::default() + }), + ..Default::default() + }, + expected: Some((ERR_LIFECYCLE_FILTER_SIZE_RANGE, std::io::ErrorKind::Other)), + }, + Case { + name: "valid size range inside And", + filter: LifecycleRuleFilter { + and: Some(LifecycleRuleAndOperator { + object_size_greater_than: Some(1), + object_size_less_than: Some(2), + ..Default::default() + }), + ..Default::default() + }, + expected: None, + }, + ]; + + for case in cases { + let result = config_with_rules(vec![rule_with_filter(case.filter)]) + .validate(&ObjectLockConfiguration::default()) + .await; + match (case.expected, result) { + (None, Ok(())) => {} + (None, Err(err)) => panic!("{}: expected acceptance, got {err}", case.name), + (Some((message, _)), Ok(())) => panic!("{}: expected rejection with {message}", case.name), + (Some((message, kind)), Err(err)) => { + assert_eq!(err.to_string(), message, "{}", case.name); + assert_eq!(err.kind(), kind, "{}: wrong S3 error category", case.name); + } + } + } + } + + #[tokio::test] + async fn validate_keeps_legacy_prefix_and_filter_mutually_exclusive() { + let mut rule = rule_with_filter(LifecycleRuleFilter { + prefix: Some("logs/".to_string()), + ..Default::default() + }); + rule.prefix = Some("legacy/".to_string()); + + let err = config_with_rules(vec![rule]) + .validate(&ObjectLockConfiguration::default()) + .await + .expect_err("legacy Prefix and Filter cannot both be present"); + + assert_eq!(err.to_string(), ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT); + } + + #[test] + fn count_only_rule_round_trips_through_xml() { + // The MinIO count-only form has to survive the wire codec, or the rule + // this PR now accepts could not be persisted and read back. + let xml = br#"count-onlyEnabled2"#; + let mut deserializer = s3s::xml::Deserializer::new(xml); + let parsed = + ::deserialize(&mut deserializer).expect("count-only XML parses"); + + let expiration = parsed.rules[0] + .noncurrent_version_expiration + .as_ref() + .expect("noncurrent expiration is present"); + assert_eq!(expiration.newer_noncurrent_versions, Some(2)); + assert_eq!(expiration.noncurrent_days, None); + + let mut buf = Vec::new(); + let mut serializer = s3s::xml::Serializer::new(&mut buf); + XmlSerializeContent::serialize_content(&parsed, &mut serializer).expect("count-only config serializes"); + let serialized = String::from_utf8(buf).expect("serialized XML is UTF-8"); + assert!( + serialized.contains("2"), + "retention count survives the round trip: {serialized}" + ); + assert!( + !serialized.contains(""), + "a count-only rule must not gain an age condition: {serialized}" + ); + } + mod adversarial_regressions { use super::*; use s3s::dto::NoncurrentVersionExpiration; diff --git a/crates/lifecycle/src/evaluator.rs b/crates/lifecycle/src/evaluator.rs index 2da4ae76c..d492b4645 100644 --- a/crates/lifecycle/src/evaluator.rs +++ b/crates/lifecycle/src/evaluator.rs @@ -22,7 +22,10 @@ use rustfs_replication::ReplicationStatusType; use rustfs_scanner_metrics::metrics::IlmAction; use crate::object_lock; -use crate::{Event, Lifecycle, ObjectOpts, expiration_action_has_valid_target}; +use crate::{ + Event, LIFECYCLE_CORRUPT_RULE_ERROR_KIND, Lifecycle, ObjectOpts, expiration_action_has_valid_target, + lifecycle_has_corrupt_retention_count, +}; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; @@ -152,6 +155,17 @@ impl Evaluator { format!("number of versions mismatch, expected {}, got {}", objs[0].num_versions, objs.len()), )); } + // PUT validation rejects a negative retention count, so a rule that + // carries one came from older persistence or an import. Report it + // instead of evaluating a configuration that cannot be honoured; + // `eval_inner` independently takes no action for such a rule + // (backlog#2201). + if lifecycle_has_corrupt_retention_count(&self.policy) { + return Err(std::io::Error::new( + LIFECYCLE_CORRUPT_RULE_ERROR_KIND, + "lifecycle configuration carries a negative 'NewerNoncurrentVersions'", + )); + } Ok(self.eval_inner(objs, OffsetDateTime::now_utc()).await) } } diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index e57a6cc1a..b9de1c073 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1888,9 +1888,9 @@ where // A remote restart or movement flip invalidates // the token proof; usage_store interprets this // as a publication barrier and performs no PUT. - return true; + return Some(ScannerCycleDeferReason::DataMovement); } - storeapi.scanner_data_usage_publication_blocked().await + scanner_local_publication_defer_reason(storeapi.as_ref()).await } }, ) @@ -3239,8 +3239,8 @@ where { match status { ScannerCycleStatus::Complete | ScannerCycleStatus::Superseded => { - if storeapi.scanner_data_usage_publication_blocked().await { - return Some(ScannerCycleDeferReason::DataMovement); + if let Some(reason) = scanner_local_publication_defer_reason(storeapi).await { + return Some(reason); } if status == ScannerCycleStatus::Complete { let distributed = storeapi.setup_is_dist_erasure().await; @@ -3263,6 +3263,22 @@ where } } +async fn scanner_local_publication_defer_reason(storeapi: &S) -> Option +where + S: ScannerStorage, +{ + if !storeapi.scanner_data_usage_publication_blocked().await { + return None; + } + // Pending namespace commits invalidate this publication attempt, but only + // storage movement creates durable, rate-limited catch-up debt. + if storeapi.scanner_data_movement_pause_status().await.paused { + Some(ScannerCycleDeferReason::DataMovement) + } else { + Some(ScannerCycleDeferReason::ActivityBaselineUnavailable) + } +} + fn scanner_post_lease_activity_defer_reason( expected_digest: Option<[u8; 32]>, activity: Result, diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index dbbbdca62..37d45aa89 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -266,6 +266,11 @@ async fn running_main_loop_catches_up_pause_cleared_after_startup_observe() { } let pause_status = store.scanner_data_movement_pause_status().await; assert!(pause_status.paused); + assert_eq!( + scanner_local_publication_defer_reason(store.as_ref()).await, + Some(ScannerCycleDeferReason::DataMovement), + "an actual data-movement pause must retain durable catch-up tracking" + ); paused_probe.wait().await; drop(paused_probe); @@ -1171,6 +1176,9 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() { async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledging_usage() { crate::scanner_io::clear_dirty_usage_buckets_for_tests(); let (_temp_dir, store) = setup_scanner_cycle_store().await; + let mut pause_backlog = ScannerPauseBacklogController::claim(store.clone(), scanner_pause_backlog_now()) + .await + .expect("scanner pause backlog should be available"); let bucket = format!("scanner-coordinator-pending-{}", Uuid::new_v4().simple()); store .make_bucket(&bucket, &crate::storage_api::scan::MakeBucketOptions::default()) @@ -1195,6 +1203,13 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin .await .expect("fixture usage baseline should be readable"); let pending = ecstore_hold_namespace_commit(store.as_ref()); + assert_eq!( + scanner_local_publication_defer_reason(store.as_ref()).await, + Some(ScannerCycleDeferReason::ActivityBaselineUnavailable), + "an ordinary namespace commit must not be classified as data movement" + ); + let pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await; + assert_eq!(pause_backlog_attempt, ScannerPauseBacklogAttemptDecision::Untracked); let ctx = CancellationToken::new(); let budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, ScannerCycleBudgetConfig::default()); let mut cycle_info = CurrentCycle { @@ -1209,7 +1224,15 @@ async fn coordinator_walks_during_pending_put_without_persisting_or_acknowledgin .await .expect("the coordinator must finish its namespace walk while a PUT is pending"); assert_eq!(budget.progress().0, 1, "the coordinator must reach actual object traversal"); - assert_eq!(outcome, ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::DataMovement)); + assert_eq!( + outcome, + ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable) + ); + finish_scanner_pause_backlog_cycle(&mut pause_backlog, &store, pause_backlog_attempt, outcome).await; + let pause_backlog_status = scanner_pause_backlog_status(store.clone()).await; + assert_eq!(pause_backlog_status.phase, ScannerPauseBacklogPhase::Idle); + assert!(!pause_backlog_status.pending_full_scan); + assert_eq!(pause_backlog_status.catch_up_attempts, 0); assert_eq!(cycle_info.next, 1, "a rejected publication must not advance the cycle"); assert_eq!(revision, DataUsageCacheRevision::Missing); assert_eq!(crate::scanner_io::dirty_usage_buckets_for_tests(), dirty_before); @@ -5826,7 +5849,7 @@ async fn test_usage_save_object_not_found_defers_only_with_a_fresh_route_barrier let probe_calls = route_probe_calls.clone(); async move { let call = probe_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - route_blocked && call > 1 + (route_blocked && call > 1).then_some(ScannerCycleDeferReason::DataMovement) } }, ) @@ -5872,16 +5895,19 @@ async fn test_usage_save_route_barrier_prevents_missing_snapshot_creation() { data: None, revision: DataUsageCacheRevision::Missing, }), - || async { true }, + || async { Some(ScannerCycleDeferReason::ActivityBaselineUnavailable) }, ) .await; - assert_eq!(outcome, DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement)); + assert_eq!( + outcome, + DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable) + ); assert!(!store.objects.lock().await.contains_key(&target_key)); assert_eq!( store.put_counts.lock().await.get(&target_key), None, - "the final pool-state fence must run before the first PUT" + "the final publication fence must run before the first PUT" ); } } @@ -5902,7 +5928,7 @@ async fn test_observational_usage_defers_when_authoritative_baseline_is_missing( receiver, None, None, - || async { false }, + || async { None }, ) .await; @@ -5950,7 +5976,7 @@ async fn test_observational_usage_uses_fenced_backup_when_v2_primary_has_no_iden receiver, None, None, - || async { false }, + || async { None }, ) .await; @@ -5991,7 +6017,7 @@ async fn test_observational_usage_uses_bootstrap_pending_primary_as_baseline() { receiver, None, None, - || async { false }, + || async { None }, ) .await; @@ -6051,7 +6077,7 @@ async fn test_usage_route_barrier_precedes_durable_reconciliation() { data: Some(Bytes::from(snapshot_data)), revision: DataUsageCacheRevision::Etag("memory-1".to_string()), }), - || async { true }, + || async { Some(ScannerCycleDeferReason::DataMovement) }, ) .await; @@ -6091,7 +6117,7 @@ async fn coordinator_does_not_put_after_remote_generation_flip() { // Model the remote lease holder flipping its movement generation // after the activity probe but before the coordinator's PUT. route_store.publication_admission_blocked.store(true, Ordering::Release); - false + None } }, ) @@ -6129,7 +6155,7 @@ async fn coordinator_classifies_an_expired_publication_lease() { revision: DataUsageCacheRevision::Missing, }), ScannerPublicationFence::new(None, Some(expired), None), - || async { false }, + || async { None }, ) .await; @@ -6208,7 +6234,7 @@ async fn test_deferred_usage_save_keeps_last_real_save_metric() { data: None, revision: DataUsageCacheRevision::Missing, }), - || async { true }, + || async { Some(ScannerCycleDeferReason::DataMovement) }, ) .await; diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs index af7c616ee..08a3c9996 100644 --- a/crates/scanner/src/scanner/usage_store.rs +++ b/crates/scanner/src/scanner/usage_store.rs @@ -265,7 +265,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel receiver, leader_epoch, initial_baseline, - || async { false }, + || async { None }, ) .await } @@ -280,7 +280,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel ) -> DataUsagePersistOutcome where F: Fn() -> Fut + Send + Sync, - Fut: Future + Send, + Fut: Future> + Send, { store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch( ctx, @@ -308,7 +308,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel ) -> DataUsagePersistOutcome where F: Fn() -> Fut + Send + Sync, - Fut: Future + Send, + Fut: Future> + Send, { store_data_usage_in_backend_with_outcome_for_epoch_and_baseline_and_route_probe_for_publication_epoch_and_lease_fence( ctx, @@ -336,7 +336,7 @@ pub(super) async fn store_data_usage_in_backend_with_outcome_for_epoch_and_basel ) -> DataUsagePersistOutcome where F: Fn() -> Fut + Send + Sync, - Fut: Future + Send, + Fut: Future> + Send, { let ScannerPublicationFence { expected_publication_epoch, @@ -374,18 +374,19 @@ where } else { DATA_USAGE_OBJ_NAME_PATH.as_str() }; - if route_probe().await { + if let Some(reason) = route_probe().await { debug!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, state = "publication_blocked_before_reconcile", - "Scanner data usage publication deferred by the pool-state fence" + reason = reason.as_str(), + path = %target_path, + "Scanner data usage publication deferred by the publication fence" ); - global_metrics().record_scanner_usage_deferred(ScannerCycleDeferReason::DataMovement.as_str()); - outcome = DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + global_metrics().record_scanner_usage_deferred(reason.as_str()); + outcome = DataUsagePersistOutcome::Deferred(reason); break; } @@ -626,17 +627,18 @@ where if ctx.is_cancelled() { break 'updates; } - if route_probe().await { + if let Some(reason) = route_probe().await { debug!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, state = "publication_blocked_before_save", - "Scanner data usage publication deferred by the final pool-state fence" + reason = reason.as_str(), + path = %target_path, + "Scanner data usage publication deferred by the final publication fence" ); - break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break DataUsagePersistOutcome::Deferred(reason); } if remote_lease_expired(remote_lease_deadline) { break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::PublicationLeaseDeadlineExceeded); @@ -722,19 +724,19 @@ where ); } Err(e @ EcstoreError::ObjectNotFound(_, _)) => { - let route_blocked = route_probe().await; - if route_blocked { + if let Some(reason) = route_probe().await { warn!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, component = LOG_COMPONENT_SCANNER, subsystem = LOG_SUBSYSTEM_RUNTIME, - path = %target_path, state = "publication_deferred", + reason = reason.as_str(), + path = %target_path, error = %e, - "Scanner data usage route is blocked by data movement; retrying later" + "Scanner data usage route remains blocked; retrying later" ); - break DataUsagePersistOutcome::Deferred(ScannerCycleDeferReason::DataMovement); + break DataUsagePersistOutcome::Deferred(reason); } error!( target: "rustfs::scanner", diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index 410cf6b48..0c2a8c455 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -1294,6 +1294,8 @@ impl FolderScanner { } Err(e) => return Err(ScannerError::Io(e)), }; + #[cfg(test)] + tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget); pending_entry_progress = pending_entry_progress.saturating_add(1); if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH || last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL diff --git a/crates/scanner/src/scanner_folder/tests.rs b/crates/scanner/src/scanner_folder/tests.rs index 8e26429cc..d47b17acc 100644 --- a/crates/scanner/src/scanner_folder/tests.rs +++ b/crates/scanner/src/scanner_folder/tests.rs @@ -25,6 +25,7 @@ use std::os::unix::fs::{PermissionsExt, symlink}; use std::sync::Mutex; mod checkpoint_fixture; +pub(super) mod enumeration_restart; /// Reset the process-global alert cooldown map; test-only. fn reset_alert_cooldowns() { diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs index eb10f02e6..7052e9de4 100644 --- a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs @@ -20,6 +20,8 @@ use crate::{DataUsageCacheSource, DataUsageScanPlanDigest}; use std::io::Cursor; use tokio::io::AsyncReadExt; +mod segment_observation; + const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin"; const STATIC_OBJECTS: u64 = 24; const MAX_CACHE_BYTES: u64 = 1024 * 1024; diff --git a/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/segment_observation.rs b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/segment_observation.rs new file mode 100644 index 000000000..b54f36665 --- /dev/null +++ b/crates/scanner/src/scanner_folder/tests/checkpoint_fixture/segment_observation.rs @@ -0,0 +1,202 @@ +//! Fixture-only range diagnostics. No result is supplied to a scan selector. + +use super::*; +use std::collections::BTreeSet; + +const MAX_SEGMENTS: usize = 4; +const MAX_SEGMENT_BYTES: usize = 128; +const MAX_WALK_SAMPLES: usize = 32; +const MAX_WALK_BYTES: usize = 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ProposalError { + EntryLimit, + ByteLimit, + InvalidKey, +} + +// Keys come from successful fixture writes, not a production mutation stream. +fn fixture_proposal(keys: &[&str]) -> Result, ProposalError> { + let mut segments = BTreeSet::new(); + let mut bytes = 0; + for key in keys { + if key.is_empty() || key.contains(['\\', '\0']) || key.split('/').any(|part| matches!(part, "" | "." | "..")) { + return Err(ProposalError::InvalidKey); + } + let segment = key.split('/').next().expect("validated nonempty key"); + if segments.contains(segment) { + continue; + } + if segments.len() == MAX_SEGMENTS { + return Err(ProposalError::EntryLimit); + } + if segment.len() > MAX_SEGMENT_BYTES - bytes { + return Err(ProposalError::ByteLimit); + } + bytes += segment.len(); + segments.insert(segment.to_string()); + } + Ok(segments) +} + +#[test] +fn segment_observation_fixture_proposal_bounds() { + assert_eq!(fixture_proposal(&["hot/one", "hot/two"]), Ok(BTreeSet::from(["hot".to_string()]))); + assert_eq!(fixture_proposal(&["a", "b", "c", "d"]).expect("entry boundary").len(), MAX_SEGMENTS); + assert_eq!(fixture_proposal(&["a", "b", "c", "d", "e"]), Err(ProposalError::EntryLimit)); + let exact = "x".repeat(MAX_SEGMENT_BYTES); + assert!(fixture_proposal(&[&exact]).is_ok()); + assert_eq!(fixture_proposal(&[&exact, "y"]), Err(ProposalError::ByteLimit)); + let oversized = "x".repeat(MAX_SEGMENT_BYTES + 1); + assert_eq!(fixture_proposal(&[&oversized]), Err(ProposalError::ByteLimit)); + for key in ["", "/hot", "hot/../cold", "hot//one", "hot\\one", "hot/\0"] { + assert_eq!(fixture_proposal(&[key]), Err(ProposalError::InvalidKey)); + } +} + +fn cache_value(cache: &DataUsageCache) -> serde_json::Value { + let mut value = serde_json::to_value(cache).expect("serialize the entire cache"); + // Children are a HashSet: canonicalize only that unordered field, without + // discarding any cache fields or changing ordered histogram arrays. + for (path, entry) in &cache.cache { + value["cache"][path]["children"] = + serde_json::to_value(entry.children.iter().collect::>()).expect("canonical child set"); + } + value +} + +async fn walk_and_save(observe: bool) -> (Vec, serde_json::Value) { + let (mut scanner, root) = build_test_scanner().await; + let _guard = TestGuard { + temp_dir: Some(root.clone()), + }; + for prefix in ["hot", "cold", "other"] { + for leaf in ["one", "two"] { + let object = format!("{prefix}/{leaf}"); + let mut metadata = FileMeta::new(); + let mut info = FileInfo::new(&object, 4, 2); + info.volume = "bucket".to_string(); + info.name = object.clone(); + info.size = 1; + info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture timestamp")); + info.metadata.insert("etag".to_string(), "before".to_string()); + metadata.add_version(info).expect("construct segment fixture metadata"); + write_test_object_metadata_bytes(&root, "bucket", &object, &metadata.marshal_msg().expect("encode metadata")).await; + } + } + let changed_key = "hot/one"; + let changed_path = root.join("bucket").join(changed_key).join("xl.meta"); + let before = tokio::fs::read(&changed_path).await.expect("read initial hot metadata"); + let mut metadata = FileMeta::new(); + let mut info = FileInfo::new(changed_key, 4, 2); + info.volume = "bucket".to_string(); + info.name = changed_key.to_string(); + info.size = 1; + info.mod_time = Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture timestamp")); + info.metadata.insert("etag".to_string(), "after!".to_string()); + metadata.add_version(info).expect("construct same-size hot mutation"); + write_test_object_metadata_bytes(&root, "bucket", changed_key, &metadata.marshal_msg().expect("encode hot mutation")).await; + let after = tokio::fs::read(&changed_path) + .await + .expect("read back committed fixture mutation"); + assert_eq!(before.len(), after.len(), "fixture rewrite must keep metadata byte length unchanged"); + assert_ne!(before, after, "a changed key requires an observable successful fixture write"); + scanner.old_cache.info.name = "bucket".to_string(); + scanner.new_cache.info.name = "bucket".to_string(); + scanner.update_cache.info.name = "bucket".to_string(); + let paths = Arc::new(Mutex::new(Vec::::new())); + let proposed_walked = Arc::new(Mutex::new(BTreeSet::::new())); + scanner.update_current_path = Arc::new({ + let paths = paths.clone(); + let proposed_walked = proposed_walked.clone(); + move |path: &str| { + let mut paths = paths.lock().expect("lock bounded actual-walk samples"); + assert!(paths.len() < MAX_WALK_SAMPLES, "fixture walk exceeded its entry budget"); + let bytes: usize = paths.iter().map(String::len).sum(); + assert!(path.len() <= MAX_WALK_BYTES - bytes, "fixture walk exceeded its byte budget"); + paths.push(path.to_string()); + if observe { + let proposed = fixture_proposal(&[changed_key]).expect("bounded successful fixture mutation"); + if let Some(segment) = path.strip_prefix("bucket/").and_then(|path| path.split('/').next()) + && proposed.contains(segment) + { + proposed_walked + .lock() + .expect("lock bounded observed segments") + .insert(segment.to_string()); + } + } + Box::pin(async {}) + } + }); + scanner + .scan_folder( + CancellationToken::new(), + CachedFolder { + name: "bucket".to_string(), + parent: None, + object_heal_prob_div: 1, + }, + &mut DataUsageEntry::default(), + ) + .await + .expect("actual folder walker must finish independently of diagnostics"); + let paths = paths.lock().expect("read walk samples").clone(); + assert!(!paths.is_empty()); + for prefix in ["hot", "cold", "other"] { + assert!( + paths.iter().any(|path| path == &format!("bucket/{prefix}")), + "all fixture segments must actually be walked" + ); + } + let store = FixtureStore::new(); + let revisions = DataUsageCache::default() + .load_with_revisions(store.clone(), CACHE_NAME) + .await + .expect("read empty fixture revisions"); + scanner + .new_cache + .save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0) + .await + .expect("save actual walker output through the cache codec and revision gate"); + let loaded = store.strict_load().await; + assert_eq!(loaded.checked_flatten("bucket").expect("complete fixture tree").objects, 6); + assert_eq!( + cache_value(&loaded), + cache_value(&scanner.new_cache), + "codec round-trip must retain the entire cache, not just aggregate size" + ); + if observe { + let proposed = proposed_walked.lock().expect("read callback observations").clone(); + assert_eq!(proposed, BTreeSet::from(["hot".to_string()])); + let walked_segments: BTreeSet<_> = paths + .iter() + .filter_map(|path| path.strip_prefix("bucket/")) + .filter_map(|path| path.split('/').next()) + .collect(); + assert_eq!(walked_segments, BTreeSet::from(["cold", "hot", "other"])); + assert!(proposed.iter().all(|segment| walked_segments.contains(segment.as_str()))); + assert_eq!( + walked_segments.len() - proposed.len(), + 2, + "the two non-proposed segments must still be walked" + ); + eprintln!( + "segment fixture: proposed={proposed:?}, actual_segments={walked_segments:?}, actual_walk_callbacks={}, production_producer_coverage=unverified", + paths.len() + ); + } else { + assert!(proposed_walked.lock().expect("read disabled observations").is_empty()); + } + // Compare semantic values because map encoding order is not content identity. + (paths, cache_value(&loaded)) +} + +#[tokio::test] +#[serial] +async fn segment_observation_on_off_preserves_actual_walk_and_saved_cache() { + let off = walk_and_save(false).await; + let on = walk_and_save(true).await; + assert_eq!(off.0, on.0, "diagnostics must not change actual traversal order or coverage"); + assert_eq!(off.1, on.1, "diagnostics must not change the saved cache result"); +} diff --git a/crates/scanner/src/scanner_folder/tests/enumeration_restart.rs b/crates/scanner/src/scanner_folder/tests/enumeration_restart.rs new file mode 100644 index 000000000..90a1b30eb --- /dev/null +++ b/crates/scanner/src/scanner_folder/tests/enumeration_restart.rs @@ -0,0 +1,181 @@ +// Copyright 2026 RustFS Team +// Licensed under the Apache License, Version 2.0. + +use super::*; +use std::path::{Path, PathBuf}; +use tokio::io::AsyncReadExt; + +const MAX_CACHE_BYTES: u64 = 1024 * 1024; +const REQUEST_ENV: &str = "RUSTFS_ENUMERATION_REQUEST"; + +struct Observation { + root: PathBuf, + limit: u64, + entries: u64, + name_bytes: u64, +} + +static OBSERVATION: Mutex> = Mutex::new(None); + +// Only the selected synthetic disk is observed; concurrent unrelated scanners +// do not consume its budget. This hook is absent from non-test builds. +pub(in crate::scanner_folder) fn observe_raw_entry(dir: &str, name: &std::ffi::OsStr, budget: &ScannerCycleBudget) { + let mut guard = OBSERVATION.lock().expect("enumeration observation lock"); + if let Some(observation) = guard.as_mut() + && Path::new(dir).starts_with(&observation.root) + { + observation.entries += 1; + observation.name_bytes += u64::try_from(name.as_encoded_bytes().len()).expect("bounded entry name"); + if observation.entries >= observation.limit { + budget.cancel_for_runtime(); + } + } +} + +struct ObservationGuard; + +impl Drop for ObservationGuard { + fn drop(&mut self) { + *OBSERVATION.lock().expect("enumeration observation cleanup") = None; + } +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct Request { + workspace: PathBuf, + objects: usize, + raw_entry_budget: u64, + round: u32, +} + +async fn read_bounded(path: &Path) -> Vec { + let file = tokio::fs::File::open(path).await.expect("open fixture artifact"); + let mut bytes = Vec::new(); + file.take(MAX_CACHE_BYTES + 1) + .read_to_end(&mut bytes) + .await + .expect("read fixture artifact"); + assert!(u64::try_from(bytes.len()).expect("artifact size") <= MAX_CACHE_BYTES); + bytes +} + +async fn round(request: &Request) -> serde_json::Value { + assert!((1..=1024).contains(&request.objects)); + assert!((1..=4096).contains(&request.raw_entry_budget)); + assert!(request.round < 64); + let disk_root = request.workspace.join("disk"); + let cache_path = request.workspace.join("cache.bin"); + if request.round == 0 { + tokio::fs::create_dir(&disk_root).await.expect("create fresh synthetic disk"); + for index in 0..request.objects { + let object = format!("object-{index:04}"); + let version = Uuid::from_u128(u128::try_from(index).expect("fixture index") + 1); + let bytes = metadata_for_object_version("bucket", &object, Some(version)); + write_test_object_metadata_bytes(&disk_root, "bucket", &object, &bytes).await; + } + let mut initial = DataUsageCache::default(); + initial.info.name = "bucket".to_string(); + initial.info.skip_healing = true; + initial.info.snapshot_complete = false; + initial.replace("bucket", "", DataUsageEntry::default()); + tokio::fs::write(&cache_path, initial.marshal_msg().expect("initial cache codec")) + .await + .expect("persist initial cache"); + } + let cache = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload cache codec before scan"); + assert_eq!(cache.info.name, "bucket"); + let before = cache.checked_flatten("bucket").expect("persisted bucket root").objects; + let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("fixture endpoint"); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("open synthetic disk in this process"); + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default()); + *OBSERVATION.lock().expect("install observation") = Some(Observation { + root: disk.path(), + limit: request.raw_entry_budget, + entries: 0, + name_bytes: 0, + }); + let _observation_guard = ObservationGuard; + let result = scan_data_folder( + budget.token(), + budget.clone(), + vec![disk.clone()], + disk, + cache.clone(), + None, + HealScanMode::Normal, + SCANNER_SLEEPER.clone(), + ) + .await; + let (returned, outcome) = match result { + Ok(cache) => (cache, "complete"), + Err(ScannerError::PartialCache(cache)) => (*cache, "partial"), + Err(ScannerError::Other(message)) if budget.token().is_cancelled() && message == "Operation cancelled" => { + (cache, "cancelled_without_cache") + } + Err(error) => panic!("unexpected real scanner failure: {error}"), + }; + let encoded = returned.marshal_msg().expect("returned cache codec"); + assert!(u64::try_from(encoded.len()).expect("encoded length") <= MAX_CACHE_BYTES); + tokio::fs::write(&cache_path, encoded).await.expect("persist returned cache"); + let reloaded = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload returned cache codec"); + let retained = reloaded.checked_flatten("bucket").expect("reloaded bucket root"); + let scanned = returned.checked_flatten("bucket").expect("returned bucket root"); + assert_eq!( + (retained.objects, retained.versions, retained.size), + (scanned.objects, scanned.versions, scanned.size) + ); + assert_eq!(reloaded.info.snapshot_complete, returned.info.snapshot_complete); + let guard = OBSERVATION.lock().expect("read observation"); + let observation = guard.as_ref().expect("installed observation"); + serde_json::json!({ + "schema": 1, "pid": std::process::id(), "round": request.round, + "objects_expected": request.objects, "raw_entry_budget": request.raw_entry_budget, + "raw_entries": observation.entries, "raw_name_bytes": observation.name_bytes, + "objects_processed": budget.progress().0, + "objects_before": before, "objects_retained": retained.objects, + "versions_retained": retained.versions, "bytes_retained": retained.size, + "snapshot_complete": reloaded.info.snapshot_complete, "outcome": outcome, + }) +} + +/// Default CI is a positive healthy control. The external driver selects the +/// same worker in a fresh OS process per round and applies its strict oracle. +#[tokio::test] +#[serial] +async fn enumeration_restart_worker() { + if let Some(path) = std::env::var_os(REQUEST_ENV) { + let request: Request = serde_json::from_slice(&read_bounded(Path::new(&path)).await).expect("bounded worker request"); + let report = round(&request).await; + tokio::fs::write( + request.workspace.join(format!("round-{}.json", request.round)), + serde_json::to_vec(&report).expect("report JSON"), + ) + .await + .expect("write worker report"); + } else { + let temp = tempfile::tempdir().expect("healthy fixture directory"); + let report = round(&Request { + workspace: temp.path().to_path_buf(), + objects: 4, + raw_entry_budget: 16, + round: 0, + }) + .await; + assert_eq!(report["outcome"], "complete"); + assert_eq!(report["snapshot_complete"], true); + assert_eq!(report["objects_retained"], 4); + assert_eq!(report["versions_retained"], 4); + assert_eq!(report["bytes_retained"], 4); + assert!(report["raw_entries"].as_u64().expect("observed entries") >= 8, "{report}"); + } +} diff --git a/crates/scanner/src/scanner_io/tests.rs b/crates/scanner/src/scanner_io/tests.rs index a008dc81e..bd4add6df 100644 --- a/crates/scanner/src/scanner_io/tests.rs +++ b/crates/scanner/src/scanner_io/tests.rs @@ -126,6 +126,16 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc) { (temp_dir, store) } +async fn wait_for_namespace_commit_tails(store: &ECStore) { + tokio::time::timeout(Duration::from_secs(30), async { + while store.scanner_data_usage_publication_blocked().await { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("namespace commit tails should drain before the scanner fixture runs"); +} + #[tokio::test] #[serial] async fn checkpoint_fixture_bucket_identity_uses_its_set_instance_owner() { @@ -336,6 +346,7 @@ async fn scoped_scan_production_entry_preserves_deep_and_full_maintenance_work() .await .expect("initial object should persist"); } + wait_for_namespace_commit_tails(store.as_ref()).await; let mut baseline = None; for (index, (scan_mode, requires_full_scan, explicit_scope)) in [ (HealScanMode::Normal, true, false), @@ -354,6 +365,7 @@ async fn scoped_scan_production_entry_preserves_deep_and_full_maintenance_work() .put_object("cold-bucket", &format!("added-{index}"), &mut reader, &ScannerObjectOptions::default()) .await .expect("cold bucket mutation should persist"); + wait_for_namespace_commit_tails(store.as_ref()).await; // Only the hot bucket is in the usage hint. The cold result must // come from this cycle's storage walk, not its previous baseline. record_dirty_usage_bucket("hot-bucket"); diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 2911d9c49..784bffd81 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -1020,6 +1020,7 @@ mod serial_tests { } let (_disk_paths, ecstore) = setup_isolated_test_env(false).await; + let expired_recovery_time = i128::from(i64::MAX / 2); for case in [ CleanupCase::Persisted, @@ -1117,7 +1118,7 @@ mod serial_tests { .await .expect("active unknown ownership must remain fenced after the transaction store was offline"); assert_eq!((retained.scanned, retained.recovered, retained.retained, retained.failed), (1, 0, 1, 0)); - let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, i128::MAX) + let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, expired_recovery_time) .await .expect("expired unknown ownership may use the provider's missing proof"); assert_eq!( @@ -1166,7 +1167,7 @@ mod serial_tests { assert_eq!(retained.recovered, 0); assert_eq!(retained.retained + retained.failed, 1); backend.set_remove_failure(false); - let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, i128::MAX) + let recovered = recover_transition_transaction_records_at(ecstore.clone(), 100, None, expired_recovery_time) .await .expect("expired recovery should delete the candidate after the backend becomes available"); assert_eq!( diff --git a/docs/architecture/ilm-tiering-persistence-contracts.md b/docs/architecture/ilm-tiering-persistence-contracts.md index 8bff02a9f..c20204d03 100644 --- a/docs/architecture/ilm-tiering-persistence-contracts.md +++ b/docs/architecture/ilm-tiering-persistence-contracts.md @@ -91,7 +91,7 @@ Lock ordering is part of the recovery contract. Callers acquire only the locks n | Path | Current acquisition order | Operations allowed while held | Operations forbidden while held | |---|---|---|---| -| Tier add/edit/remove/clear | A short tier-config namespace WRITE lock captures the persisted config ETag, then releases before backend validation. After validation, namespace WRITE then `admin_updates` protect the ETag check and durable coordinator Prepared write. Both guards are released for lease drain, peer Prepare, and reference proof, then reacquired in the same order for final identity checks and config CAS. Both are released again after the coordinator becomes durably Committed | Backend validation, peer fanout, and reference proof run without either exclusive guard. Immediately before config CAS the coordinator revalidates the ETag, candidate digest, exact Prepared intent identity, and intent expiry. The durable Committed intent is recovery authority while peer Commit and local publication finish without the exclusive guards | Ordinary manager `RwLock` and runtime-state `Mutex` guards must not cross awaited network I/O. Recovery must retain an unexpired Prepared coordinator whose old ETag is still current; the old ETag alone is not abandonment proof. Remote object DELETE is never part of mutation | +| Tier add/edit/remove/clear | A short tier-config namespace WRITE lock captures the persisted config ETag, then releases before backend validation. After validation, namespace WRITE then `admin_updates` protect the ETag check and durable coordinator Prepared write. The Prepared runtime block rejects new tier-operation leases while already-issued leases remain current and drain through their complete local cleanup. Both guards are released for that drain, peer Prepare, and reference proof. Only after the proof succeeds does the coordinator revoke the blocked generation, then it reacquires both guards in the same order for final identity checks and config CAS. Both are released again after the coordinator becomes durably Committed | Backend validation, existing leased cleanup, peer fanout, and reference proof run without either exclusive guard. The exact Prepared mutation ID is the only allowance for the late publish transition; its generation drain is expected to be empty because the Prepared block admitted no new lease. Immediately before config CAS the coordinator revalidates the ETag, candidate digest, exact Prepared intent identity, and intent expiry. The durable Committed intent is recovery authority while peer Commit and local publication finish without the exclusive guards | Revoking a generation before already-leased cleanup has removed its exact local ownership marker; admitting a new lease after Prepared; ordinary manager `RwLock` or runtime-state `Mutex` across awaited network I/O. Recovery must retain an unexpired Prepared coordinator whose old ETag is still current; the old ETag alone is not abandonment proof. Remote object DELETE is never part of mutation | | v6 manifest prepare | Caller already holds the bucket-lifecycle WRITE fence; caller acquires a bucket-metadata transaction READ guard covering the Object Lock and bucket-incarnation snapshot and keeps it through local mutation; exact tier-generation leases; fleet/topology proof; for a single dispatch, synthetic manifest-operation WRITE; for a child, parent-operation WRITE then child-operation WRITE | Build and write one immutable bounded journal set and manifest, validate exact set/digest, then authorize local dispatch while both caller-held bucket guards and all leases remain current. A parent binding is durable before child authorization | Remote tier DELETE; per-object worker cleanup; releasing the metadata guard or a required lease before the authorized local mutation completes; child-to-parent nested lock acquisition | | v6 manifest/parent recovery | Fleet/topology proof; bucket-lifecycle WRITE lock; then exactly one synthetic manifest- or parent-operation WRITE lock | Read/write manifest, parent, and journal metadata; verify exact set/digest/binding; converge or roll back child records; advance a parent only after child completion | Remote tier DELETE; per-object worker cleanup; rollback after authorization; taking a child lock while holding a parent lock in background recovery | | v5 journal destructive recovery | Synthetic per-journal recovery lock; bucket-lifecycle READ lock; exact tier-generation lease; all physical object READ locks in stable pool/set order | Authoritative source/free-version scan; fenced state CAS; for an eligible terminal state, one bounded remote DELETE; conditional record cleanup | Any delete when a lock or lease is lost; publishing local metadata; selecting an arbitrary backend/version | diff --git a/docs/operations/bucket-metadata-recovery.md b/docs/operations/bucket-metadata-recovery.md index 92227c359..f467ceeb1 100644 --- a/docs/operations/bucket-metadata-recovery.md +++ b/docs/operations/bucket-metadata-recovery.md @@ -1,12 +1,12 @@ # Bucket metadata diagnostics and recovery -`GET /rustfs/admin/v3/export-bucket-metadata` keeps its strict behavior: an unreadable configuration fails the export. The optional `bucket` query selects one bucket; omitting it selects all buckets. +`GET /rustfs/admin/v3/export-bucket-metadata` exports every configuration it can read. A configuration that is stored but unreadable is never exported and never replaced by a fabricated default; instead the bucket gains an entry `/rustfs-unreadable-configs.json` of the shape `{"bucket": …, "unreadable": [{"config": …, "error": …}]}` naming each configuration that could not be read and why, and the export continues, so one bucket's undecodable payload cannot cost an operator the whole-cluster backup (rustfs/backlog#2309). The marker name is outside the configuration namespace importers dispatch on, so importing the archive back leaves the affected bucket's stored bytes untouched. A failure of the server's own serialization or archive writing still fails the export. The optional `bucket` query selects one bucket; omitting it selects all buckets. -To inspect readable configurations while identifying failures, use the same authenticated endpoint with `?diagnostic=true`. This requires the existing `ExportBucketMetadataAction` permission. A successful response has: +To collect a shareable support artifact that identifies the failures without carrying parser detail, use the same authenticated endpoint with `?diagnostic=true`. This requires the existing `ExportBucketMetadataAction` permission. A successful response has: - Filename `bucket-meta-diagnostic.zip` and header `x-rustfs-bucket-metadata-export: diagnostic`. - Readable entries under `_diagnostic//`; target credentials remain redacted. -- `_diagnostic-manifest.json`, containing `version: 1`, `mode: "diagnostic"`, `complete`, and an `errors` array. Each error identifies `bucket`, `config`, and the fixed code `configuration_unavailable`. The archive excludes unreadable payloads and parser error details. +- `_diagnostic-manifest.json`, containing `version: 1`, `mode: "diagnostic"`, `complete`, and an `errors` array. Each error identifies `bucket`, `config`, and the fixed code `configuration_unavailable`. The archive excludes unreadable payloads and parser error details, and therefore carries no `rustfs-unreadable-configs.json` marker: the manifest reports the same failures with less detail, which is what makes a diagnostic archive safe to hand out. `complete` reports whether all supported configuration reads succeeded. A diagnostic archive is never a restorable backup, including when `complete` is true. Import rejects the manifest or reserved directory before any bucket creation or configuration write. The reserved directory is not a valid bucket name, so older importers cannot restore diagnostic entries as ordinary bucket configurations. @@ -14,9 +14,11 @@ To inspect readable configurations while identifying failures, use the same auth RustFS currently accepts the documented `{"targets": [...]}` object format. It cannot decrypt MinIO KMS-encrypted target metadata. Unreadable target payloads remain failures instead of being interpreted as an empty target set; diagnostic export and replacement import do not add MinIO KMS decryption support. -1. Inspect the diagnostic manifest to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery. +1. Inspect the diagnostic manifest, or the `rustfs-unreadable-configs.json` marker in an ordinary export, to identify affected buckets. Preserve a separate backup of the original source configuration and any credentials needed for recovery. 2. Prepare a ZIP containing `/bucket-targets.json` with a valid RustFS replacement, whose top-level shape is `{"targets": [...]}`. Supply the intended target settings and credentials; exported credentials are redacted. Use `{"targets": []}` only when intentionally clearing all targets, and reconcile any replication rules that reference removed targets. 3. Submit the ZIP to the existing authenticated `PUT /rustfs/admin/v3/import-bucket-metadata` endpoint with `ImportBucketMetadataAction` permission. Import validates the replacement and persists it against the bucket incarnation; it does not need to parse the old target payload successfully. -4. Verify target listing and the intended replication configuration. Retry the ordinary strict metadata export to confirm the unreadable configuration no longer blocks it. +4. Verify target listing and the intended replication configuration. Retry the ordinary metadata export to confirm the bucket no longer carries an unreadable marker. + +Alternatively, `PUT /rustfs/admin/v3/set-remote-target?replace-unreadable=true` discards an undecodable target set as part of setting a replacement target. The flag is the operator's explicit acknowledgement that the stored set is being thrown away; without it the request is refused rather than rewriting an unreadable set from a partial view. Do not submit the diagnostic archive itself to the import endpoint. Copy only reviewed replacement entries into an ordinary import archive. diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index 83e9315a2..04f06aee4 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -34,6 +34,149 @@ The `scanner` and `heal` subsystems are served by `GetConfigKVHandler` (`rustfs/ ## Test Matrix +### Formal Scanner/Heal ABBA + +The `--abba` mode runs five independent scenario cells: `cold-hot`, `fresh-hot`, +`multi-hot-new`, `running-heal`, and `mrf-replay`. Each scenario runs at least +three A1/B1/B2/A2 groups for both baseline/candidate with background work on, +and candidate-only background off/on. A measured leg lasts at least 900 +seconds; the minimum matrix contains 120 legs (30 hours before setup/oracles). +The existing `performance-ab.yml` supplies the pattern for immutable build +provenance and failure propagation, but its short Warp workload is not this +scanner gate. No scheduled workflow starts this matrix automatically. + +```bash +scripts/run_scanner_validation_harness.sh --abba \ + --manifest scanner-abba.json --adapter /path/to/isolated-deployment-adapter \ + --out-dir /path/to/new-artifacts --data-root /path/to/new-test-data +``` + +Both roots must be new and non-overlapping. Every leg receives a unique data +directory. The runner checks disk capacity before each leg, never removes data, +and stops the adapter after success or failure. Retain raw artifacts and inspect +task ownership before removing any test data. The operator must reserve the +target machines and map the assigned directory to separate data paths on every +node; the runner cannot prove remote isolation from local path names. + +The manifest has the following JSON contract (all fields are required): + +| Field | Value | +|---|---| +| `schema`, `evidence` | `1`, and `measured` or `synthetic`. | +| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 900..86400 seconds for measured runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. | +| `baseline`, `candidate` | Each contains executable `binary`, full 40-character `revision`, and verified `sha256`. The runner rehashes binaries before every leg. | +| `fixed` | `config_sha256`, `dataset_sha256`, `release_flags`, `durability`, `disk_type`, `cache_state`, `load_command`, `resource_isolation`, `topology` (`EC8+4`), and positive `offered_load_ops`. Hashes use 64 lowercase hexadecimal characters. | +| `oracles` | A map with all five scenario names. Each value contains positive integer `objects`, `versions`, `bytes`, and `sha256` of the independently prepared canonical object/version/content manifest. | +| `expected_healed_objects` | A map with all five scenario names and independently seeded repair counts. Running-heal and MRF-replay require a positive count. | + +Record exact build flags and effective durability settings, not just defaults. +Use deterministic workload seeds so every isolated leg has the same expected +object/version/content result. Fix the foreground arrival rate (offered load), +cache preparation procedure, configuration, and hardware across every leg. +Do not include credentials in the manifest, adapter output, or saved commands; +the collector reads `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` from its environment. + +#### Deployment Adapter Contract + +The runner invokes an executable as `adapter ACTION request.json response.json` +with no shell evaluation. Actions are separate processes: `prepare`, `measure`, +`oracle`, and `stop`. Every action must return zero and write a JSON object of +at most 1 MiB. Logs are kept separately and require an operator-managed disk +quota. Missing output, timeout, nonzero exit, unknown/missing metrics, zero +samples, and request errors fail the run. Adapters must terminate their own +children on failure and `stop` must be idempotent even after partial preparation. +The runner keeps its session leader unreaped while stopping a failed command +or collector: it sends TERM, allows the existing ten-second grace period, then +kills the remaining process group before reaping. This prevents a parent exit +from hiding live descendants or allowing the group ID to be reused before its +last signal. A successful `prepare` preserves adapter-owned services until +`stop`; services that leave the command's process group remain the adapter's +cleanup responsibility. + +The request contains the fixed manifest fields, selected build, scenario, round, +leg, comparison (`build` or `background`), background mode (`on` or `off`), +duration, unique `data_dir`, expected object oracle, and expected repair count. +Adapter responsibilities: + +1. `prepare` deploys the selected binary into an authorized isolated topology, + checks actual binary/config/durability, initializes deterministic scenario + data and the requested cache state, and returns `{"ready": true}`. For measured + runs it also returns `collector` with exactly `alias`, `endpoint`, and + comma-separated `metrics_endpoints`; the runner starts the existing scanner + collector at 60-second cadence while `measure` runs. +2. `measure` maintains the fixed offered load for the entire requested duration. + `cold-hot` retains cold buckets while mutating a hot bucket; `fresh-hot` + creates a bucket after scanner startup; `multi-hot-new` combines several hot + buckets with a newly created bucket; `running-heal` applies foreground load + during active repair; `mrf-replay` replays independently seeded durable repair + work. Capture same-window status for bucket-freshness issue #7108. Actual + fault injection and dataset generation belong to the reviewed adapter. +3. `oracle` independently enumerates all objects and versions, reads and checks + their complete bytes, and verifies repairs. Return `complete: true`, integer + `errors: 0`, and `actual` matching the manifest's expected oracle. Never copy + expected values into a measured oracle or infer completion from empty queues. +4. `stop` stops task-owned workload/server processes and returns `stopped: true`. + Preserve data and artifacts for diagnosis. An adapter may restore previous + settings but must not delete arbitrary paths or stop unrelated deployments. + +The `measure` response echoes the observed `evidence`, `fixed`, `build`, +`data_dir`, and `background`, plus `sample_count` (1..3600), `elapsed_seconds`, +and `metrics`. All metrics must be finite nonnegative numbers: `p99_ms`, +`throughput_ops`, `rss_bytes`, `cpu_seconds`, `iops`, `rpc_count`, +`cache_clone_bytes`, `encode_bytes`, `save_bytes`, `oldest_age_seconds`, +`walk_objects`, `cold_walk_objects`, `healed_objects`, `errors`, and `requests`. +Requests, throughput, and p99 must be positive; errors must be zero. Repair +counts must match the manifest when background work is on. Keep underlying +request samples, counter reset checks, profiler captures, and per-node telemetry +in the cell artifact directory; aggregate values alone do not establish their +measurement provenance. Missing production instrumentation is a pending gate, +not permission to report a fabricated zero. + +For P2, `measure.convergence` contains booleans `writes_stopped`, +`last_mutation_observed`, `first_complete_publication`; numeric +`last_mutation_time`, `last_mutation_observed_time`, `writes_stopped_time`, `window_start`, `window_end`, +`budget_available_seconds`, `walk_objects`, and `full_walk_objects`. Times use +one monotonic clock. The window starts after writes stop and the final mutation +is observed, and ends at the first complete publication. The reference is an +independent full walk of the same static namespace. Record available budget +seconds to interpret elapsed time. During continuing writes, omit this proof +and report useful-work ratio and justified invalidation/re-scan work separately; +the runner reports P2 pending and does not impose a fixed cumulative walk bound. + +The nightly heal workflow clones **`rustfs/auto-testing`** separately and invokes +`auto-testing/rustfs_heal_test.sh`; that script is not a local `scripts/test` +entry point. If an adapter uses it, record and verify the external checkout's +owner and full commit before use. The current workflow clones the default branch, +so its contents must not be attributed to a RustFS source SHA. + +#### Evidence Gates + +`report.json` records each group's verdict and the raw responses remain in their +cell directories. Candidate/build p99 regression must be at most 5% and +throughput loss at most 3%; candidate background on/off limits are 10% and 5%. +P1 requires cold-hot walk reduction of at least the baseline cold-walk share +times 80%, rather than a fixed 80% reduction for every workload. P2 requires +candidate post-stop work at most 1.2 times the independent full-walk reference. +Missing candidate convergence proof yields `inconclusive`. A2/A1 or B2/B1 p99 +or throughput drift above 5% also yields `inconclusive`, with exit code 3. +Correctness errors and non-noisy performance regressions exit 1. Every group +must pass; a favorable median cannot hide a failing group. + +Synthetic success is explicitly `synthetic_validated`, with `performance: +pending`. It validates orchestration and gate logic only. It proves no runtime, +distributed, crash, mixed-version, or performance behavior and cannot close the +performance acceptance gate. Run the fake-adapter self-tests with: + +```bash +scripts/test_scanner_validation_harness.sh +``` + +They cover the complete 120-cell schedule, data isolation, missing builds and +oracles, zero samples/requests, swallowed request errors, offered-load drift, +incomplete repairs, missing metrics, noise, and P1/P2/p99 regressions. A real +deployment adapter and actual ABBA artifacts remain required before any measured +performance or release claim. + Collect at least two runs on the same RustFS commit and the same workload. Keep hardware, commit, object count, object size, bucket count, scanner-enabled state, and foreground workload constant between runs. | Run | Purpose | Example scanner settings | diff --git a/docs/testing/README.md b/docs/testing/README.md index 60343b9e2..55dc6a4e0 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -12,10 +12,10 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh |---|---|---|---| | Unit & crate integration | Per-crate logic and in-process integration tests | `cargo nextest run --all --exclude e2e_test` (or `-p `); `make test` wraps it | Every PR, required (`Test and Lint`, `ci` profile) | | ecstore black-box | Erasure-coded read/write/recovery validation; profiles `quick` / `full` / `destructive` / `fuzz` | `scripts/run_ecstore_validation_suite.sh --profile quick` | Local and release validation only; not wired into any workflow. Contract: [ecstore-validation-suite-design.md](ecstore-validation-suite-design.md) | -| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md) | +| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`, `e2e-distributed`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md); 4-node 4-disk map: [distributed-e2e.md](distributed-e2e.md) | | s3s-e2e conformance | External S3 conformance tool against a live server | `./scripts/e2e-run.sh ./target/debug/rustfs ` | PR, report-only (second half of the `End-to-End Tests` job) | | S3 compatibility | `ceph/s3-tests` (boto3; allow-list `scripts/s3-tests/implemented_tests.txt`) and MinIO `mint` | `scripts/s3-tests/run.sh`; mint via `.github/workflows/mint.yml` | s3-tests: PR report-only plus a weekly full sweep; mint: weekly, report-only | -| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) used by the reliability and heal e2e modules | Part of the e2e crate (`e2e-reliability` test-group) | With the `e2e-full` and nightly e2e lanes. A multi-node power-loss harness is not in tree | +| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) plus the 4-node kill/fresh-drive/blackhole cases in `crates/e2e_test/src/distributed/chaos_test.rs` | Part of the e2e crate (`e2e-reliability` and `e2e-distributed`) | Reliability cases with `e2e-full`; 4-node chaos on storage-sensitive PRs and nightly via `e2e-distributed` | | Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus | | Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p ` | On demand; never a gate | @@ -63,6 +63,7 @@ All profiles are defined in `.config/nextest.toml`; its block comments hold the | `e2e-full` | Merge-queue / main-push single-node e2e lane | | `e2e-repl-nightly` | Nightly slow / cross-process replication lane | | `e2e-nightly` | Nightly serial multi-process cluster fault lane | +| `e2e-distributed` | Storage-sensitive PR and nightly 4-node 4-disk S3 / lock / versioning / replication / quota / expand / decommission / rebalance / site-replication / chaos / upgrade (history + IAM AK/SK) lane | | `e2e-protocols` | Nightly fixed-port FTPS/SFTP/WebDAV lane, run with `-j 1` | Membership of each e2e profile is pinned by a digest in `.config/e2e--selection.txt` and checked by `scripts/check_test_wiring.py --check-profile ` before the lane runs. To list what a profile selects on your platform (the result is platform-dependent because some modules are linux-only): diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 81418c911..b40f1ee5e 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -74,6 +74,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched | `ci.yml` (weekly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | per-job | yes | dispatch `ci.yml` | | `build.yml` (weekly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set | | `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` | +| `e2e-distributed.yml` (storage-sensitive PRs + nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, then `cargo nextest run --profile e2e-distributed -p e2e_test` | | `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target | | `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME= ./scripts/fuzz/run.sh` | | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow | @@ -120,8 +121,113 @@ Update this file in the same PR when a job or check name changes, a workflow gai The existing `ci.yml` test-and-lint job runs the ordinary ECStore and filemeta tests. After that run, `scripts/check_test_wiring.py --check-core` checks the same nextest profile and package selection against `.config/ecstore-required-tests.json`. Every named test must exist, match the filter, and be non-ignored; the job also requires a nonempty JUnit report. This checks membership without running the tests twice. `core-test-listing.json`, JUnit, and the run log are retained in the existing test-and-lint artifact. -The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, and corrupt part arrays. Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered. +The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, corrupt part arrays, and the shared on-demand-migration source-backend contract for each provider dialect (S3, Azure, native GCS). The three contract entries live in the `rustfs` suite and reach the lane through that package's default features, so dropping `gcs` from `rustfs`'s defaults fails this check instead of silently deselecting the GCS contract (rustfs/backlog#2323). Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered. The checked-in MinIO corpus is pinned by file SHA256 and its documented source release. The static wiring guard and the CI selection check both reject missing or changed fixtures. These are metadata fixtures, not a legacy shard-body corpus or proof of crash durability. Optional `legacy_bitrot_read_test` runs may still skip when their external corpus is absent; they do not satisfy a required compatibility lane. Real encrypted fixture reads remain in `minio-interop.yml`, and multi-node fault schedules remain in the existing nightly cluster lane. In-process reopen tests do not establish power-loss durability. Run `python3 scripts/check_test_wiring.py --self-test` to exercise the negative cases: removed/ignored/filtered tests, malformed listing, absent fixtures, and wrong fixture hashes. Do not update hashes merely to silence the guard; a fixture change needs source/provenance and compatibility review. +## Scanner/Heal Evidence Receipts + +The existing `scripts/check_test_wiring.py` also validates Scanner/Heal case +evidence registered in `.config/scanner-heal-required-tests.json`. It records +already-built binaries and checks existing nextest output; it does not build, +run tests, deploy servers, inject faults, or start another CI lane. + +The initial case is `background-target-restart`, emitted by +`heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart`. +That test already runs in `e2e-nightly`. When `RUSTFS_SCANNER_HEAL_RUN_DIR` is set, +it checks the actual server and test-executable hashes against `run.json`, pins +the same server binary for all node starts, and writes its oracle only after +the real assertions pass. The artifact contains the actual pre/post target +PIDs, per-node S3 listings, expected and downloaded complete-body hashes/lengths, +and target-disk `VersionShardCensus` fingerprints. Existing baseline objects +must match their pre-fault physical manifests; the object created during the +outage has no pre-fault target shard and is checked for complete physical parts +and exact S3 content. + +This case is a **four-node, one-drive-per-node process-restart test**. It is not +power-loss validation, a 3x4 EC8+4 experiment, an all-version inventory, or proof +of scanner enumeration, exact MRF disposition, legacy migration, or rollback. +The registry keeps all G01-G14/P1-P4 and R-E/R-D/R-L release requirements pending +until their actual feature-specific oracles and required topologies exist. +Missing cases cannot be supplied by synthetic W20 results. W20's bounded JSON +and file-hash helpers are reused; its ABBA performance contracts remain in +`docs/operations/scanner-benchmark-runbook.md`. + +### Recording One Case + +Use a committed source tree, independently built current binaries, sufficient +free disk space, and a task-owned artifact directory that does not yet exist. +Set `SERVER_BINARY` and `TEST_BINARY` to those exact executable paths. The begin +command requires the server's embedded `--version` commit to match the clean +checkout and its embedded Git status to be clean. The E2E crate's build script +embeds its build-time Git revision/dirty state, lockfile Git blob, enabled crate +features, target, profile and encoded Rust flags. It tracks the crate/dependency +trees, Cargo inputs and Git HEAD/ref/index, including `common.rs` restart logic. +The producer checks this compiled identity against the receipt; it does not +copy a current source revision into an older test binary's identity. The E2E +uses its existing temporary cluster directories and cleanup. `CARGO_TARGET_DIR` +controls compilation output; nextest's default report store remains the +workspace's `target/nextest`. Execute the existing selected case as follows: + +```bash +CASE=background-target-restart +FILTER='test(test_cluster_root_heal_recovers_remote_shards_after_background_target_restart)' +RUN_DIR="$PWD/artifacts/scanner-heal-run" +export RUSTFS_E2E_EXPECTED_FEATURES=default +scripts/python_bin.sh scripts/check_test_wiring.py \ + --begin-scanner-heal "$RUN_DIR" "$SERVER_BINARY" "$TEST_BINARY" +export RUSTFS_SCANNER_HEAL_RUN_DIR="$RUN_DIR" +export CARGO_BIN_EXE_rustfs="$SERVER_BINARY" +cargo nextest list --profile e2e-nightly -p e2e_test -E "$FILTER" \ + --message-format json > "$RUN_DIR/listing.json" +rm -f target/nextest/e2e-nightly/junit.xml +set +e +cargo nextest run --profile e2e-nightly -p e2e_test -E "$FILTER" +test_exit=$? +set -e +cp target/nextest/e2e-nightly/junit.xml "$RUN_DIR/junit.xml" +scripts/python_bin.sh scripts/check_test_wiring.py --finish-scanner-heal "$RUN_DIR" "$test_exit" +scripts/python_bin.sh scripts/check_test_wiring.py --check-scanner-heal "$RUN_DIR" "$CASE" +``` + +Set `RUSTFS_E2E_EXPECTED_FEATURES` to the actual intended e2e crate feature set, +including `default` for a default-feature build, comma-separated for extra +features, or empty for `--no-default-features`. It is mandatory when beginning +a run. Crate features are distinct from the spawned server's build features. + +Do not replace a nonzero command exit with zero. Missing JUnit or an oracle +emission failure also fails acceptance. Each retry needs a new run directory; +the producer refuses to overwrite an existing oracle. Keep failed-run logs and +artifacts. The receipt pins source revision, actual binary hashes, run identity, +start/finish times, and the artifact hashes. `listing.json`, `junit.xml`, and +each oracle are limited to 1 MiB; object evidence has the fixture's 9..65 object +bound. Credentials are not included in the receipt. + +The checker binds nextest's flattened suite `binary-id`/`binary-path` to the +actual test executable and requires the JUnit testcase's embedded execution +timestamp to fall inside the receipt window (with millisecond precision). +Copying an old JUnit file and refreshing its mtime does not make it new evidence. +Schema versions, topology counts, PIDs, EC geometry and shard indices require +actual integers: booleans and fractional values are rejected, and an index must +fit the physical data-plus-parity geometry. + +The checker rejects unselected/ignored tests, zero/duplicate JUnit cases, +failures, skipped tests, retry/flaky records, stale or changed artifacts, +different builds or run IDs, unchanged process IDs, wrong topology, missing +shard parts, and mismatched S3 content/listings. The raw oracle JSON is emitted +by the real E2E producer, not accepted from an adapter copying expectations. + +`--check-scanner-heal "$RUN_DIR" release` checks available case evidence and +returns nonzero for every pending release requirement. A focused case pass +does not approve release. In particular, R-E requires fixed-budget real +restarts without an unbudgeted final sweep, R-D requires the full +manager/event/ledger disposition chain, and R-L requires source-conflict and +crash/retirement evidence. Reader-only or unit fixtures cannot substitute for +these. The external `rustfs/auto-testing` functional workflows propagate suite +failures. Their workflow status does not establish this registry's required +case coverage, build provenance, or object-level oracles. + +Run parser/receipt regressions with +`scripts/python_bin.sh scripts/check_test_wiring.py --self-test`. Those fixtures +validate the checker only and produce no runtime or performance evidence. diff --git a/docs/testing/distributed-e2e.md b/docs/testing/distributed-e2e.md new file mode 100644 index 000000000..10d882f9b --- /dev/null +++ b/docs/testing/distributed-e2e.md @@ -0,0 +1,80 @@ +# Distributed 4-node 4-disk e2e + +**Use this when:** adding or diagnosing GitHub Actions coverage for a 4-node cluster, or deciding whether a behaviour belongs in `e2e-distributed` versus the single-node `e2e-full` lane, the nightly cluster-fault lane, or the hardware functional chain. +**Source of truth:** `crates/e2e_test/src/distributed/`, `[profile.e2e-distributed]` in `.config/nextest.toml`, `.github/workflows/e2e-distributed.yml`. + +## Topology + +The in-tree harness runs every node on `127.0.0.1` with a distinct port. That matches `RustFSTestClusterEnvironment` in `crates/e2e_test/src/common.rs`: + +| Layout | Constructor | Use | +|---|---|---| +| 4 nodes × 4 drives, one pool | `ClusterTopology::single_pool_multidrive(4, 4)` | S3, object lock, versioning, quota, observability, concurrency, chaos | +| 4 nodes × 1 drive, one pool | `ClusterTopology::single_pool(4)` | Two-site replication (8 processes total); direct/rolling upgrade from the pinned previous release | +| 1 single-node pool × 4 drives, then `append_single_node_pool` three times | expansion seed | Pool expand, then decommission / rebalance / integrity | + +A multi-pool layout in which any pool spans several localhost ports is not expressible (`RUSTFS_VOLUMES` host ellipses would collide on disk paths). Multi-host striped expansion pools remain the hardware functional-chain / backlog #1313 / #1314 lane. + +Data-movement cases fail closed. A decommission or rebalance test must observe a successful start response, an active state, a clean terminal state, non-zero movement counters, and post-operation object integrity. An unsupported response, HTTP 5xx, missing status fields, cleanup warning, or zero-progress terminal response fails the case; pre/post S3 availability alone is not evidence that movement ran. + +The four expansion pools must report independent capacity. Four directories on one runner filesystem all return the same `statfs` totals, so RustFS correctly concludes that no pool is less free than the cluster average and performs no rebalance. The Actions job mounts four isolated 1 GiB tmpfs filesystems and exports their absolute paths through `RUSTFS_E2E_POOL_ROOTS`. It does not use ext4 loop devices: the `sm-standard-4` ARC pods have no `/dev/loop-control`, so `mount -o loop` fails with `No such file or directory`. Sized tmpfs still reports a distinct `st_dev` and independent 1 GiB `statfs` capacity. The harness rejects missing, duplicate, relative, nonexistent, or same-device roots instead of allowing a vacuous movement pass. Planned pool additions stop every process with SIGTERM; hard process termination remains a chaos-only fault. After the fourth pool joins, the harness performs one full graceful persistent restart: this proves the expanded pool map survives restart and ensures movement begins only after every replica can load the converged metadata. + +The expansion fixture is an all-current-binary fleet, so it initializes pool metadata with the documented V3 write and fleet-confirmation gates. Decommission cases write their baseline objects, version history, and multipart data into pool 0 before adding pools 1–3, then retire pool 0. This makes a passing result evidence of user-data movement rather than merely an internal-metadata counter changing. + +## What this lane covers + +`cargo nextest run --profile e2e-distributed -p e2e_test` selects `distributed::*`: + +- S3 put / get / head / list / copy / rename / delete / presign, range and conditional reads, special keys, metadata, tags, pagination, empty objects, multipart complete and abort +- Object Lock COMPLIANCE, GOVERNANCE and bypass, legal hold, bucket default retention, and non-lock bucket rejection +- Versioning, exact historical reads, delete-marker removal, and suspended null-version overwrite semantics +- Bucket replication between two 4-node clusters, including metadata/tags and target-outage retry; hard quota admission and absence of rejected keys +- Ready/live probes on every node, exact 4-server/16-disk inventory, realtime metrics on every node, and correlated audit-webhook delivery +- Pool expand, decommission, rebalance, checksum integrity, versioned and multipart data, and S3 during active movement +- Bidirectional site-replication convergence plus enabled/synchronized peer state on both sites +- A 24-worker mixed PUT/HEAD/GET/COPY/DELETE workload; concurrent PUT during active decommission +- Node kill/restart, full process restart, node-facing TCP blackhole/recovery, in-flight streaming GETs across a peer kill, and fresh-drive replacement verified by physical `xl.meta`/part-shard census +- Multipart, cross-node listing, list-buckets agreement +- Direct and rolling upgrade from the pinned previous release: historical objects, versioned history, and IAM user AK/SK still work afterwards + +## Existing Actions gaps this lane does not replace + +Those suites stay in place; this lane fills the in-tree 4×4 hole they leave. + +| Existing lane | Gap | +|---|---| +| `rustfs-*-test.yml` functional chain | Clones private `rustfs/auto-testing`, runs on three shared VMs (`vm000`–`vm002`), `continue-on-error: true`, not a merge signal, not 4 nodes. Hardware `rustfs-upgrade-test.yml` stays there | +| `e2e-upgrade.yml` | Single-node SSE/multipart/delete-marker contracts plus mixed-version listing; does not pin IAM user AK/SK on a 4-node cluster | +| `e2e-smoke` / `e2e-full` | Most selected cases are single-node; distributed modules are intentionally owned by this serialized lane | +| `e2e-nightly` | 4-node cluster faults and heal, not S3/lock/versioning/quota/decommission matrix | +| `e2e-repl-nightly` | Site and bucket replication on 1–3 *single-node* processes | +| `e2e-s3tests.yml` `multi` | Weekly ceph/s3-tests against Docker 4-node; not lock/WORM, decommission, chaos, or checksum integrity | +| `crates/e2e_test/src/chaos.rs` | Single-node disk faults only | + +Hardware power-loss, physical NIC pull, authenticated inter-node partition, firmware/media errors, and replacement-server provisioning still belong on the hardware validation VMs. This lane provides deterministic process kill, fresh local-volume replacement, and node-facing TCP blackhole analogues; it does not claim physical fault certification. + +## Run + +```bash +cargo build -p rustfs --bins +# Expansion/decommission/rebalance cases require four paths on distinct filesystems. +# If you do not already have four disks, sized tmpfs is enough: +# for p in 0 1 2 3; do +# sudo mkdir -p /mnt/rustfs-pool-$p +# sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs /mnt/rustfs-pool-$p +# done +export RUSTFS_E2E_POOL_ROOTS=/mnt/rustfs-pool-0:/mnt/rustfs-pool-1:/mnt/rustfs-pool-2:/mnt/rustfs-pool-3 +# Upgrade cases require the pinned previous binary (CI downloads it). +export RUSTFS_UPGRADE_SOURCE_BINARY=/path/to/rustfs-1.0.0-rc.2 +cargo nextest run --profile e2e-distributed -p e2e_test +``` + +Without `RUSTFS_UPGRADE_SOURCE_BINARY` the two `distributed::upgrade_test::*` cases fail closed. Without four distinct `RUSTFS_E2E_POOL_ROOTS`, the expansion and data-movement cases fail closed. Filter upgrades out for a local run that is not checking upgrade: + +```bash +cargo nextest run --profile e2e-distributed -p e2e_test -E 'not test(/^distributed::upgrade_test::/)' +``` + +The upgrade topology is `ClusterTopology::single_pool(4)` (4 nodes × 1 drive). That matches the proven mixed-version fixture in `upgrade_compatibility_test`; 4×4 localhost drives are rejected by the previous release's same-device disk check. + +Membership is pinned by `.config/e2e-distributed-selection.txt`. Update the Linux and Darwin entries with `python3 ./scripts/check_test_wiring.py --update-profile e2e-distributed ` after adding or renaming a case. diff --git a/docs/testing/mrf-pending-migration.md b/docs/testing/mrf-pending-migration.md new file mode 100644 index 000000000..f07638c1c --- /dev/null +++ b/docs/testing/mrf-pending-migration.md @@ -0,0 +1,21 @@ +# Pending MRF Migration + +`heal::mrf_queue::snapshot::migration` exposes explicit capture, staging, and readback of pending legacy responsibility evidence. Nothing invokes it from the production MRF consumer. It does not enable the committed-snapshot writer, freeze legacy ingress, acknowledge durable admission, or authorize source garbage collection. + +The caller supplies every configured local disk slot, including missing slots. Missing/unformatted/duplicate disks, unavailable metadata volumes, invalid records, and aggregate byte/record/source-history overflow fail closed. A valid empty source observation can inherit earlier pending responsibilities, but staging without any current or inherited responsibility is rejected. Both legacy paths retain their original bytes, disk identity, absent-versus-empty state, and SHA-256 digest. Complete subset/superset replicas become conservative pending evidence, never a claimed newest legacy snapshot. Raw record replay preserves kind, scope and nil/absent version encodings; unknown incarnation stays unknown. + +Staging writes only `.heal-mrf-import-pending.{0,1}.bin`, `.heal-mrf-import-commit.{0,1}.bin`, and `.heal-mrf-import-claim.bin` under the metadata volume. It reuses the committed reader's manifest codec and the storage owner's conditional-file operation, including the configured metadata durability policy. A candidate is written before sources are revalidated, its manifest is then committed, and committed bytes plus source coverage are read back before success. Success is pending staging evidence, not a power-loss or cluster-quorum durability receipt. + +A successor inherits prior source bytes even if a replay consumer has already read or admitted their records. Independent byte, record, and source-history limits include inherited evidence, and source identities use a hash index with full-byte conflict checks. There is no completion-based pruning. A changed source blocks recovery of that pending generation; an explicit new capture can stage a successor that retains both the previous and current responsibilities. The inactive slot is replaced while the preceding committed slot remains intact. Any corrupt, unsupported or over-budget slot blocks selection rather than falling back to an older generation. The reader first collects bounded lineage evidence from at most two slots per configured disk. A payload without a manifest is repairable only when its length and digest match the complete retry candidate after inheritance, or any independently validated committed payload, including an older generation on another replica. Unknown payloads still block writing. Retries also repair missing replica manifests. + +All participating disks are claimed in disk-identity order through CAS. Normal completion conditionally releases only the current invocation's claim, attempts every acquired claim even after a release error, and reports the first release error. Cancellation, process death, or an ambiguous claim/release I/O failure may leave a claim behind. Read-only recovery remains available when the snapshot/source proof is valid, but further staging is blocked until a separate storage-fenced recovery procedure is implemented. Process liveness and the legacy ingress lease do not authorize taking over or deleting a claim. + +Run the focused fixtures with a nonzero test count: + +```sh +cargo test -p rustfs-heal --lib heal::mrf_queue::snapshot::migration::tests +``` + +Fixtures use real local disks and the production CAS/readback path. They cover disk-order independence, retained raw identities, source change after candidate write, capacity rejection, interrupted commit boundaries, lost responses, torn inactive slots, and refusal to take over an interrupted claim. Boundary injection and same-process reopen are not process-kill, directory-fsync failure, disk-full, mixed-version, or power-loss tests. The actual manager Full/Accepted-to-crash pipeline remains outside this staged API. + +Rollback leaves all pending and legacy artifacts untouched. Activation still requires legacy-writer coordination, bounded recoverable ingress, exact object-disposition/successor receipts, and the W14/W21 process-crash and compatibility gates. The production legacy replay deletion window remains unresolved by this staging-only phase. diff --git a/docs/testing/scanner-checkpoint-fixture.md b/docs/testing/scanner-checkpoint-fixture.md index 1c413a4b3..f9d437905 100644 --- a/docs/testing/scanner-checkpoint-fixture.md +++ b/docs/testing/scanner-checkpoint-fixture.md @@ -1,5 +1,38 @@ # Scanner Checkpoint Fixture +## Raw Enumeration Restart Diagnostic + +`enumeration_restart_worker` exercises the real `scan_data_folder` with a local disk and valid `xl.meta` objects. Without configuration it is a positive CI control: four one-byte objects must complete and survive a cache codec round trip. It is not an ignored test or an assertion that a known defect must persist. + +```sh +RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib enumeration_restart_worker -- --nocapture +cargo test -p rustfs-scanner --lib --no-run --message-format=json +python3 -m unittest discover -s scripts -p 'test_diagnose_scanner_enumeration_restart.py' +``` + +Use the `executable` from the scanner library test `compiler-artifact` JSON record as `--test-binary` below. The driver verifies that it contains the exact worker test before doing any work; a zero-test filter cannot pass. + +```sh +python3 scripts/diagnose_scanner_enumeration_restart.py \ + --test-binary /path/to/compiled/scanner-libtest \ + --output /tmp/scanner-enumeration-new-run \ + --objects 128 --raw-entry-budget 8 --rounds 8 +``` + +The output directory must not exist. Each round starts a new OS test-worker process, opens the same synthetic disk, decodes the preceding cache, invokes the real scanner, encodes the returned cache, and decodes it again. When cancellation returns no useful partial cache, it preserves the previous cache. Reports identify the actual child PID, round, raw entries and name bytes observed, processed objects, retained object/version/byte counts, and completeness. No observed-name set, `readdir` offset, or assumed stable ordering is used as durable progress. Namespace creation happens only during fixture setup, before scan accounting. + +The `cfg(test)` hook observes actual entries delivered by `read_dir` and cancels the existing cycle token at the fixed entry limit. This is a deterministic injected **raw-entry work budget**, not a wall-clock performance measurement or a claim that kernel prefetch, probes, allocations, name bytes, or cache I/O are independently budgeted. The watchdog timeout only bounds worker lifetime. The hook does not replace enumeration, classification, or recursion, and does not exist in production builds. In particular, `xl.meta` object-boundary classification is unchanged. + +Exit 0 requires exact complete object/version/byte coverage within the same fixed budget on every executed round. Exit 1 means the strict convergence oracle remains unmet, including the current flat-directory enumeration starvation case. Exit 2 means invalid input, worker failure, or invalid evidence; it is not a successful reproduction. There is no final unbudgeted sweep. Small fixtures can pass; that does not establish the general R-E gate from [the scanner review comment](https://github.com/rustfs/backlog/issues/2240#issuecomment-5549222480). Raw entries observed are not a retained enumeration watermark. This is scanner-worker process restart plus codec evidence, **not** whole-daemon restart, EC quorum persistence, crash/fsync durability, remote RPC, or a throughput benchmark. The caller owns the bounded evidence directory and may remove it after inspection. + +### Missing Storage Capability + +The current `scanner_folder::FolderScanner::scan_folder` collects child folders before recursing. `LocalDisk::scan_dir` also reads the whole parent before sorting and applying `forward_to`. The persistent key-only listing index's `collect_persistent_key_only_index_objects` / `rebuild_persistent_key_only_index` collects all objects in memory before publication and excludes deleted entries. It cannot supply a restartable first-build cursor over per-disk raw entries, orphan directories, and metadata boundaries. Repeated listing from the beginning is real work, not free pagination. + +A future storage-owner capability must expose an explicit unsupported/building/ready state and a durable snapshot/index identity bound to disk mount, bucket incarnation, and directory identity. It must budget the first build and every page, including entry count, name bytes, metadata probes, I/O and time; survive a process restart during first build; seal page data before advancing the manifest; and distinguish enumerated, classified, and fully processed frontiers. An uncommitted page may be replayed only within a bounded cost. `xl.meta` classification must finish before descendants become traversable namespace. Missing capability or invalid identities must not become fabricated progress or completeness. No such capability is implemented by this diagnostic, and ordinary local storage remains without this R-E guarantee. + +## Completed Subtree Checkpoint Fixture + The `checkpoint_fixture` tests exercise a bounded namespace of 24 static objects and one repeatedly updated hot object. Each of three rounds runs the production local disk scanner with an object budget, saves the returned partial cache through the production persistence codec and revision checks to a two-file test backend, and reloads it before preparing the next round. The fixture prints static-subtree coverage at each boundary and cumulative visited entries. This is a diagnostic of retained coverage, not a throughput benchmark. Run the fixture and confirm the test filter selects a nonzero number of tests: @@ -24,3 +57,19 @@ This fixture bounds object processing after directory enumeration. It does not p For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store. The synthetic namespace contains no customer data. Temporary files are removed with their owning fixture. Rolling back to a reader without the optional checkpoint metadata rebuilds partial coverage; it must not clear quota floors or complete authoritative snapshots. A passing fixture alone does not establish that the field report in [issue #7108](https://github.com/rustfs/rustfs/issues/7108) has been independently reproduced or fixed. A field diagnosis must separately identify the source capture, cycle and leader identity, and decoded bucket/set caches. + +## Segment Observation Diagnostics + +The nested `segment_observation` fixture compares diagnostic on/off runs of the real folder walker over six objects in `hot/`, `cold/`, and `other/`. Each run first rewrites `hot/one` with a different, equal-length ETag in real fixture metadata and reads it back to verify changed bytes at unchanged length. The successful fixture write supplies its known key to a diagnostic executed inside the real walker's path callback. Both runs save and reload the actual cache through the existing codec and revision-aware file backend. Assertions compare traversal order and the entire decoded cache, not encoded map order or aggregate size alone. Proposed top-level segments never reach a scanner selector or publication decision, and non-proposed segments must still be walked. The diagnostic retains at most four segments and 128 segment-name bytes; actual-walk samples are limited to 32 entries and 1,024 bytes. Exceeding sample limits fails the fixture rather than silently truncating its oracle. Saving a cache here is not an authoritative root publication. + +Entry/byte overflow and malformed keys reject the fixture proposal. Missing producers, process restarts, event gaps, and compacted child coverage remain **unverified production capabilities**, not simulated success cases in this fixture. Mainline bucket dirty generations and hashed metadata-cache invalidation stripes are not an exact, replayable object-key stream. The open [prefix reuse proposal #7208](https://github.com/rustfs/rustfs/pull/7208) is a separate candidate implementation; these tests neither import its hint map nor activate its skip path. + +The ECStore `segment_observation_equal_size_mutations_retire_metadata_generation` test uses the existing exact-key, test-only invalidation probe and actual owner operations. A same-length PUT must change the returned body and ETag while retiring the old generation; metadata-only PUT must change returned metadata and retire the old generation while size and ETag remain equal. Setup uses the existing full-fanout cache-priming helper; the observed mutations use normal owner locking. This is focused producer evidence, not an end-to-end connection between the owner probe and scanner range selection. The existing semantic mutation matrix covers additional owner entry points separately. + +```sh +cargo test -p rustfs-scanner --lib segment_observation -- --list +RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib segment_observation -- --nocapture +RUST_MIN_STACK=4194304 cargo test -p rustfs-ecstore --lib segment_observation_equal_size_mutations_retire_metadata_generation -- --nocapture +``` + +[W19](https://github.com/rustfs/backlog/issues/2272) remains open for trustworthy producer coverage, source/incarnation binding, and production shadow observations. No production stream, durable journal, runtime feature switch, scan skipping, or performance claim is introduced here. No restart/gap detection or restart-safe production coverage is established, and the revision-aware file backend does not prove EC publication durability. diff --git a/protocol/agent/v1/fixtures/offline-enrollment/MANIFEST.sha256 b/protocol/agent/v1/fixtures/offline-enrollment/MANIFEST.sha256 index 45c1b0e23..065f17825 100644 --- a/protocol/agent/v1/fixtures/offline-enrollment/MANIFEST.sha256 +++ b/protocol/agent/v1/fixtures/offline-enrollment/MANIFEST.sha256 @@ -1,5 +1,6 @@ 5133761d19d6a64c18b6b5f871d646f6a2da4ceccc998d3cf7e22f692ca2d925 accept-vectors.json -c7da10d173e7fafa112743d9a41e2bc94df58bf88a0542d80350b74da8f382a5 error-codes.json +ce9f5b66c629b31e14c3986937565d3cb253d37bb7d546102cc73614b1de85ee boundary-vectors.json +dbb1ad902d24a4c4508dba6d63244dd567bfc58e778b76a61941605ac43a29dc error-codes.json e98cfbedfb385defdaa9d001c85fdebcf9df2b4d054930951ff59dfa1385e52f reject-vectors.json 69d43c8266d7bb29b4df7105c49250293943583f2202b93d922d9a924fca0c09 trust-chain.json -e60cfca04bf0ce2f69495c49a95e4cc42e92e8114f6ad43449084527b06a0939 trust-model.json +7116f55de42a438bf5de8f6cc1e3c636b216a4db5a37f4caf49fe07d226498b1 trust-model.json diff --git a/protocol/agent/v1/fixtures/offline-enrollment/boundary-vectors.json b/protocol/agent/v1/fixtures/offline-enrollment/boundary-vectors.json new file mode 100644 index 000000000..0d4824344 --- /dev/null +++ b/protocol/agent/v1/fixtures/offline-enrollment/boundary-vectors.json @@ -0,0 +1,181 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "offline-enrollment", + "fixture": "boundary-vectors", + "description": "Frozen challenge boundary and decision vectors for failures that cannot be added as newly signed golden documents because no fixture private key exists. Mutations start from the accepted pinned-root challenge and are applied without re-signing; every targeted rule runs before the signature invalidated by that mutation.", + "sourceVector": "challenge signed by a chained signing key under the pinned root", + "preparseMutations": [ + { + "name": "challenge envelope is not JSON", + "scope": "serializedEnvelope", + "value": "{", + "expectedReason": "DOCUMENT_MALFORMED" + }, + { + "name": "challenge bytes are non-canonical padded base64", + "scope": "envelopeBytes", + "value": "QR==", + "expectedReason": "DOCUMENT_MALFORMED" + }, + { + "name": "challenge bytes decode to a JSON scalar", + "scope": "envelopeBytes", + "value": "bnVsbA==", + "expectedReason": "DOCUMENT_MALFORMED" + }, + { + "name": "challenge is missing trustChain", + "scope": "challenge", + "operation": "remove", + "field": "trustChain", + "expectedReason": "DOCUMENT_MALFORMED" + }, + { + "name": "challenge trustChain is a JSON object", + "scope": "challengeChain", + "operation": "objectWithFirst", + "expectedReason": "DOCUMENT_MALFORMED" + }, + { + "name": "challenge issuedAt is not a real calendar instant", + "scope": "challenge", + "operation": "replace", + "field": "issuedAt", + "value": "2026-02-31T00:00:00Z", + "expectedReason": "DOCUMENT_MALFORMED" + }, + { + "name": "first trust link issuerKeyId is malformed", + "scope": "trustLink", + "index": 0, + "operation": "replace", + "field": "issuerKeyId", + "value": "not-a-key-id", + "expectedReason": "DOCUMENT_MALFORMED" + } + ], + "verificationMutations": [ + { + "name": "challenge signature algorithm is not ES256", + "scope": "envelopeSignature", + "operation": "replace", + "field": "algorithm", + "value": "ES384", + "expectedReason": "SIGNATURE_MALFORMED" + }, + { + "name": "challenge signature keyId is malformed", + "scope": "envelopeSignature", + "operation": "replace", + "field": "keyId", + "value": "not-a-key-id", + "expectedReason": "SIGNATURE_MALFORMED" + }, + { + "name": "challenge signature keyId names another well-formed key", + "scope": "envelopeSignature", + "operation": "replace", + "field": "keyId", + "value": "f6fbe050defded18b50477ace38c9515fb61b8157e57b2f0e7e8ca69c862b6ca", + "expectedReason": "SIGNATURE_INVALID" + }, + { + "name": "pinned-root challenge carries only one trust link", + "scope": "challengeChain", + "operation": "keepFirst", + "expectedReason": "TRUST_CHAIN_INVALID" + }, + { + "name": "intermediate link formatVersion is unsupported", + "scope": "trustLink", + "index": 0, + "operation": "replace", + "field": "formatVersion", + "value": "rustfs.connect.offline.trustLink/2", + "expectedReason": "TRUST_CHAIN_INVALID" + }, + { + "name": "intermediate link protocolVersion is unsupported", + "scope": "trustLink", + "index": 0, + "operation": "replace", + "field": "protocolVersion", + "value": "v2", + "expectedReason": "TRUST_CHAIN_INVALID" + }, + { + "name": "intermediate link signature algorithm is not ES256", + "scope": "trustLinkSignature", + "index": 0, + "operation": "replace", + "field": "algorithm", + "value": "ES384", + "expectedReason": "TRUST_CHAIN_INVALID" + }, + { + "name": "intermediate link omits serial", + "scope": "trustLink", + "index": 0, + "operation": "remove", + "field": "serial", + "expectedReason": "TRUST_CHAIN_INVALID" + }, + { + "name": "first trust link carries signing role", + "scope": "trustLink", + "index": 0, + "operation": "replace", + "field": "role", + "value": "signing", + "expectedReason": "TRUST_CHAIN_INVALID" + } + ], + "linkValidityPolicyVectors": [ + { + "name": "intermediate link at its maximum validity", + "role": "intermediate", + "notBefore": "2026-01-01T00:00:00Z", + "notAfter": "2027-01-01T00:00:00Z", + "expectedReason": null + }, + { + "name": "intermediate link one second beyond its maximum validity", + "role": "intermediate", + "notBefore": "2026-01-01T00:00:00Z", + "notAfter": "2027-01-01T00:00:01Z", + "expectedReason": "TRUST_CHAIN_INVALID" + }, + { + "name": "signing link at its maximum validity", + "role": "signing", + "notBefore": "2026-08-01T00:00:00Z", + "notAfter": "2026-09-01T00:00:00Z", + "expectedReason": null + }, + { + "name": "signing link one second beyond its maximum validity", + "role": "signing", + "notBefore": "2026-08-01T00:00:00Z", + "notAfter": "2026-09-01T00:00:01Z", + "expectedReason": "TRUST_CHAIN_INVALID" + } + ], + "postSignaturePolicyVectors": [ + { + "name": "overlong challenge expires at the frozen seven-day boundary", + "issuedAt": "2026-08-01T00:00:00Z", + "declaredExpiresAt": "2026-08-09T00:00:00Z", + "evaluationTime": "2026-08-08T00:05:01Z", + "expectedEffectiveExpiresAt": "2026-08-08T00:00:00Z", + "expectedReason": "CHALLENGE_EXPIRED" + }, + { + "name": "overlong challenge remains valid at the frozen boundary plus tolerance", + "issuedAt": "2026-08-01T00:00:00Z", + "declaredExpiresAt": "2026-08-09T00:00:00Z", + "evaluationTime": "2026-08-08T00:05:00Z", + "expectedEffectiveExpiresAt": "2026-08-08T00:00:00Z", + "expectedReason": null + } + ] +} diff --git a/protocol/agent/v1/fixtures/offline-enrollment/error-codes.json b/protocol/agent/v1/fixtures/offline-enrollment/error-codes.json index cb7642702..1b7539996 100644 --- a/protocol/agent/v1/fixtures/offline-enrollment/error-codes.json +++ b/protocol/agent/v1/fixtures/offline-enrollment/error-codes.json @@ -11,6 +11,12 @@ "A rejection never reports which of several failed checks failed first beyond the single frozen reason." ], "reasons": [ + { + "reason": "DOCUMENT_MALFORMED", + "httpStatus": 400, + "status": "INVALID_ARGUMENT", + "meaning": "An enrollment challenge envelope or decoded challenge document cannot provide the padded-base64 JSON structure and pre-verification fields required to locate its trust chain and signing key. No partial challenge is processed." + }, { "reason": "UNSUPPORTED_PROTOCOL", "httpStatus": 400, @@ -27,7 +33,7 @@ "reason": "SIGNATURE_MALFORMED", "httpStatus": 400, "status": "INVALID_ARGUMENT", - "meaning": "The signature is not 64 octets of fixed-width r||s in unpadded base64url, or r or s is out of range." + "meaning": "A top-level challenge or response signature algorithm is not ES256, keyId is malformed, the value is not 64 octets of fixed-width r||s in unpadded base64url, or r or s is out of range. A trust-link signature failure is TRUST_CHAIN_INVALID instead." }, { "reason": "SIGNATURE_NOT_CANONICAL", @@ -51,7 +57,7 @@ "reason": "TRUST_CHAIN_INVALID", "httpStatus": 401, "status": "UNAUTHENTICATED", - "meaning": "A trust link failed its own signature check, named the wrong issuer, carried an unknown role, or was outside its validity at the challenge issuedAt." + "meaning": "The pinned-root chain has the wrong length or role order, or a trust link is structurally invalid, exceeds its role validity limit, fails its signature check, names the wrong issuer, or is outside its validity at the challenge issuedAt." }, { "reason": "CONNECT_KEY_UNCHAINED", @@ -75,7 +81,7 @@ "reason": "CHALLENGE_EXPIRED", "httpStatus": 401, "status": "UNAUTHENTICATED", - "meaning": "The evaluation time is more than the skew tolerance after expiresAt." + "meaning": "The evaluation time is more than the skew tolerance after effectiveExpiresAt, which never exceeds issuedAt plus the frozen maximum challenge lifetime." }, { "reason": "CHALLENGE_PROOF_INVALID", diff --git a/protocol/agent/v1/fixtures/offline-enrollment/trust-model.json b/protocol/agent/v1/fixtures/offline-enrollment/trust-model.json index ad674de6d..ff7ae0c16 100644 --- a/protocol/agent/v1/fixtures/offline-enrollment/trust-model.json +++ b/protocol/agent/v1/fixtures/offline-enrollment/trust-model.json @@ -22,7 +22,8 @@ "keyIdOver": "DER SubjectPublicKeyInfo", "keyIdEncoding": "lowercase-hex", "keyIdPattern": "^[0-9a-f]{64}$", - "documentTransferEncoding": "base64-padded" + "documentTransferEncoding": "base64-padded", + "documentTransferValidation": "strict standard base64 whose length is a multiple of four and whose decode then encode result is byte-identical to the received value" }, "domainSeparation": { "rule": "signatureInput = domainSeparationTag || 0x00 || the exact raw octets of the signed document as transmitted", @@ -39,6 +40,7 @@ } }, "verifierMustReject": [ + "An envelope or decoded document that is not valid JSON, bytes that are not strict padded base64, or a field required before signature verification that is absent or malformed.", "A signature that is not exactly 64 octets of fixed-width r||s.", "A DER or any other ASN.1 encoded signature, even when it decodes to the same r and s.", "A signature encoded with the standard base64 alphabet or with = padding.", @@ -89,25 +91,61 @@ ], "verificationOrder": { "principle": "Parse as late as the verification key allows, and treat anything read before the signature verified as untrusted routing information rather than as a fact.", + "enrollmentFailureReasons": { + "challengeDocumentMalformed": { + "appliesTo": "enrollmentChallenge", + "reason": "DOCUMENT_MALFORMED", + "covers": [ + "an envelope that is not a JSON object or lacks document.bytes or document.signature", + "document.bytes that is not strict standard padded base64", + "decoded document bytes that are not a JSON object", + "a missing or ill-typed challenge field required before signature verification", + "issuedAt that is not a real RFC 3339 UTC instant at second precision" + ], + "noPartialProcessing": true + }, + "artifactSignatureAlgorithm": { + "appliesTo": [ + "enrollmentChallenge", + "enrollmentResponse" + ], + "supported": "ES256", + "reason": "SIGNATURE_MALFORMED" + }, + "artifactSignatureKeyId": { + "malformedReason": "SIGNATURE_MALFORMED", + "challengeMismatchReason": "SIGNATURE_INVALID", + "responseMismatchReason": "DEVICE_PROOF_INVALID" + }, + "trustLink": { + "invalidFormatVersionReason": "TRUST_CHAIN_INVALID", + "invalidProtocolVersionReason": "TRUST_CHAIN_INVALID", + "missingRequiredFieldReason": "TRUST_CHAIN_INVALID", + "invalidRoleReason": "TRUST_CHAIN_INVALID", + "invalidSignatureAlgorithmReason": "TRUST_CHAIN_INVALID", + "excessiveValidityReason": "TRUST_CHAIN_INVALID" + } + }, "enrollmentChallenge": { - "note": "A challenge carries its own chain, so the CLI must read structure before it can verify anything. The pre-parse yields only trustChain, connectKeyId, and issuedAt, and none of them is believed: the chain has to close on a pinned root, and the challenge signature has to verify, before any other field is used.", + "note": "A challenge carries its own chain, so the CLI must read structure before it can verify anything. The pre-parse yields only trustChain, connectKeyId, and issuedAt, and none of them is believed. When a first link is present and readable, its issuer is checked against the pinned roots before chain length or role checks. This precedence deliberately makes an unpinned-root artifact ENROLLMENT_ROOT_UNKNOWN even when the rest of its chain is malformed.", "steps": [ - "check the signature encoding", - "pre-parse the untrusted document for trustChain, connectKeyId, and issuedAt", + "decode the envelope and document bytes, then pre-parse the untrusted document for trustChain, connectKeyId, and issuedAt; reject an unreadable value as DOCUMENT_MALFORMED", + "check signature.algorithm and the signature encoding; reject a non-ES256 algorithm or malformed encoding as SIGNATURE_MALFORMED", + "pre-parse the first trust link for an issuerKeyId matching the frozen keyId pattern; reject an unreadable or malformed value as DOCUMENT_MALFORMED", "reject unless trustChain[0].issuerKeyId is a pinned root", - "verify every trust link against its issuer and its validity at issuedAt", + "require exactly two links and the positional roles [intermediate, signing], then verify every required field, version, algorithm, signature, issuer binding, role validity limit, and validity at issuedAt; reject any failure as TRUST_CHAIN_INVALID", "reject unless connectKeyId is the subject of the last link", "verify the challenge signature over the received octets", - "only now read protocolVersion, then formatVersion", - "check the freshness window" + "only now read protocolVersion, then formatVersion, then validate the remaining required document fields", + "check the freshness window against effectiveExpiresAt = min(expiresAt, issuedAt + maxChallengeLifetimeSeconds)" ] }, "enrollmentResponse": { "note": "A response presents the device key it is enrolling, so Connect necessarily reads that key from the document. Proof of possession is what makes it safe: the presented key must be the key that signed the presenting document.", "steps": [ - "check the signature encoding", - "reject unless deviceKeyId is the fingerprint of devicePublicKey and the signature verifies under devicePublicKey", - "only now read protocolVersion, then formatVersion", + "check signature.algorithm and the signature encoding; reject a non-ES256 algorithm or malformed encoding as SIGNATURE_MALFORMED", + "pre-parse deviceKeyId and devicePublicKey, then reject an unreadable value or unless deviceKeyId is the fingerprint of devicePublicKey and the signature verifies under devicePublicKey as DEVICE_PROOF_INVALID", + "only now read protocolVersion, then formatVersion, then validate the remaining required document fields", "compare organization, then cluster, against the stored challenge", "compare challengeId, challengeNonce, and challengeProof against the stored challenge", "check the freshness window against producedAt, then against the receive time", @@ -146,6 +184,11 @@ "chainLinkCount": 2, "maxChainLinkCount": 2, "chainOrder": "index 0 is issued by a pinned root, index 1 is issued by the subject of index 0", + "chainRoles": [ + "intermediate", + "signing" + ], + "rootCheckPrecedesChainShape": true, "note": "Because no root is ever learned at runtime, an operator cannot be socially engineered into accepting an attacker root, and a stolen intermediate cannot mint its own root. The cost is that a root rollover requires redistributing the RustFS build, which is stated in rollover.root." }, "keyHierarchy": [ @@ -242,12 +285,14 @@ "clockSkew": { "toleranceSeconds": 300, "deviceClockAuthority": "advisory", - "challengeWindow": "accepted while verifierNow is within [issuedAt - 300, expiresAt + 300]", + "challengeWindow": "accepted while verifierNow is within [issuedAt - 300, effectiveExpiresAt + 300], where effectiveExpiresAt = min(expiresAt, issuedAt + maxChallengeLifetimeSeconds)", "chainLinkWindow": "each link must satisfy notBefore <= challenge.issuedAt <= notAfter, evaluated with no tolerance because the issuer controls both values", "maxChallengeLifetimeSeconds": 604800, + "excessiveChallengeLifetimePolicy": "clamp-effective-expiry", + "excessiveChallengeLifetimeReason": "CHALLENGE_EXPIRED", "maxManifestAgeSeconds": 2592000, "maxManifestFutureSkewSeconds": 300, - "responseWindow": "producedAt must fall within [challenge.issuedAt - 300, challenge.expiresAt + 300]", + "responseWindow": "producedAt must fall within [challenge.issuedAt - 300, effectiveExpiresAt + 300]", "note": "ADR 0003 already treats client clocks as advisory for heartbeat freshness. An air-gapped device is worse: it may have no synchronised clock at all. Every window is therefore evaluated against the Connect clock for artifacts Connect receives, and against the issuer-supplied issuedAt for the chain a device validates locally." }, "replay": { diff --git a/protocol/agent/v1/fixtures/registration/MANIFEST.sha256 b/protocol/agent/v1/fixtures/registration/MANIFEST.sha256 index 742f7165a..b54e04539 100644 --- a/protocol/agent/v1/fixtures/registration/MANIFEST.sha256 +++ b/protocol/agent/v1/fixtures/registration/MANIFEST.sha256 @@ -1,4 +1,4 @@ 812b0ba479a4c8d8eb9776e7bcb8d4c4d929bb83f372c03bec064472bca6155a accept-vectors.json -eb197077a2db61ae3114fa52cdeb32715f8060f6ce5f2b9bae7fe7e7f78b4981 error-codes.json -3940cc260b21a8655e5ebbdbeccd06a273d2299ce783eef92b22cacdaebc80e1 reject-vectors.json -58a7126cef796dd0631b2de8d31528267e6281566646a5662dd3ad555a530008 transcript.json +1c8cf2e5c7428dc1d41104aaf05855efc157d0a739e904ed8ba02146c711ee91 error-codes.json +09b849e3a7e4a9ee0829f44ae10318d5ea9c7f0196a60e8b122566e4201895b8 reject-vectors.json +1dea462159b23ce2640e8b0a68fb014d48dd3344ac24db55b35c1a1d82055ddc transcript.json diff --git a/protocol/agent/v1/fixtures/registration/error-codes.json b/protocol/agent/v1/fixtures/registration/error-codes.json index 4658cca26..2e9785c79 100644 --- a/protocol/agent/v1/fixtures/registration/error-codes.json +++ b/protocol/agent/v1/fixtures/registration/error-codes.json @@ -60,6 +60,14 @@ "definedBy": "protocol/agent/v1/registration-proof.md", "note": "Separate from CERTIFICATE_REQUEST_MALFORMED because the request is structurally fine and the refusal is a policy one: ADR 0008 fixes the device key and this surface may not widen it." }, + { + "reason": "CERTIFICATE_REQUEST_PROFILE_UNSUPPORTED", + "httpStatus": 400, + "status": "INVALID_ARGUMENT", + "meaning": "The CSR subject or a typed subject alternative name cannot round-trip through the pinned stock step-ca JWK authorization strings without changing its ASN.1 type.", + "definedBy": "protocol/agent/v1/registration-proof.md", + "note": "Stable and non-retryable. No SAN is valid and is the RustFS-generated profile. IP, absolute URI, email containing @, and remaining DNS strings are valid only in their matching GeneralName choice. These names authorize the CSR key only; the issued CN and sole URI SAN still come only from Connect's assigned device uid." + }, { "reason": "REGISTRATION_TOKEN_UNUSABLE", "httpStatus": 401, diff --git a/protocol/agent/v1/fixtures/registration/reject-vectors.json b/protocol/agent/v1/fixtures/registration/reject-vectors.json index a1f230acb..ba76757d5 100644 --- a/protocol/agent/v1/fixtures/registration/reject-vectors.json +++ b/protocol/agent/v1/fixtures/registration/reject-vectors.json @@ -809,6 +809,37 @@ "reason": "REGISTRATION_PROOF_INVALID", "verifiesMathematically": false } + }, + { + "name": "DNS subject alternative name whose value is classified as an IP address", + "stage": "certificateRequest", + "evaluatedAt": "2026-08-28T12:00:00Z", + "note": "The ASN.1 GeneralName is dNSName, but the pinned stock step-ca JWK path classifies the untyped authorization string 10.0.0.1 as an IP address before comparing it with the CSR. Connect refuses the mismatch as a stable protocol decision before asking the authority to guess.", + "tokenRecord": { + "registrationTokenUid": "0198f4b0-6f00-7b60-9271-7d8e9fa0b1c5", + "organizationUid": "0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70", + "clusterUid": "0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81", + "challengeNonce": "a3f1c07d9b2e4856af0c1d3b5e7f9012c4a6b8d0e2f4061738495a6b7c8d9e0f", + "expiresUnix": 1787228100, + "state": "ACTIVE" + }, + "request": { + "protocolVersion": "v1", + "requestId": "3f2a1c94-5b6d-4e8f-9a0b-1c2d3e4f5a6b", + "registrationTokenUid": "0198f4b0-6f00-7b60-9271-7d8e9fa0b1c5", + "certificateRequest": "MIIBETCBtwIBADAvMS0wKwYDVQQDDCQwMTk4ZjRiMC04YjAwLTdkODAtOTQ5MS05ZmEwYjFjMmQzZTcwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAR+TUddmyykdETq9JrvaQiHpzKCOGetvSPiPNKhP9ydmEYPt2k98OIkpzcIDHLbCE+HixdoZ1WWSqzOF9L3dRKboCYwJAYJKoZIhvcNAQkOMRcwFTATBgNVHREEDDAKgggxMC4wLjAuMTAKBggqhkjOPQQDAgNJADBGAiEA6q1VFU3kftE89kFMG0uEnPRSAs+GdTk+GB6q2xuR8Q8CIQCHvwA+1cR9xyJgtX+XUFJsNtMWsdjxajAxeHyTeFtOsA==", + "proof": { + "algorithm": "ES256", + "value": "iULWfq3BzJQ2mIqFkZomPHAXahWjbUP1ETO8KBIRr-s9RnUdst7MP_kuaizIZozfAmhaKIOImCejwpptE_9atQ" + } + }, + "serverTranscript": "RUSTFS-CONNECT-REGISTRATION-V1\n36:0198f4b0-6f00-7b60-9271-7d8e9fa0b1c5\n36:0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70\n36:0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81\n36:3f2a1c94-5b6d-4e8f-9a0b-1c2d3e4f5a6b\n64:a3f1c07d9b2e4856af0c1d3b5e7f9012c4a6b8d0e2f4061738495a6b7c8d9e0f\n10:1787228100\n43:l86VX044RFdnKAS-VCo2bfNDjJFiNsxkZWXQfsydc5c\n", + "serverTranscriptSha256": "6f944e6044aa001e68b56c2409da76c179f52dc4e6d171de45bee52e617c12d6", + "expected": { + "accepted": false, + "reason": "CERTIFICATE_REQUEST_PROFILE_UNSUPPORTED", + "verifiesMathematically": false + } } ] } diff --git a/protocol/agent/v1/fixtures/registration/transcript.json b/protocol/agent/v1/fixtures/registration/transcript.json index 067d9fdc0..771e3f154 100644 --- a/protocol/agent/v1/fixtures/registration/transcript.json +++ b/protocol/agent/v1/fixtures/registration/transcript.json @@ -228,10 +228,25 @@ "sanUsed": false, "extensionsUsed": false, "attributesUsed": false, + "authorizationCompatibility": { + "decision": "constrained CSR profile for the pinned stock step-ca JWK authorization path", + "reason": "The JWK token carries SANs as strings and step-ca classifies each string by content before comparing it with the typed PKCS#10 GeneralName.", + "noSubjectAlternativeNameAccepted": true, + "matchingTypes": { + "iPAddress": "a parsed IPv4 or IPv6 address", + "uniformResourceIdentifier": "an absolute URI with a conventional scheme, valid percent escapes, and no control octets", + "rfc822Name": "a non-IP, non-URI value containing @", + "dNSName": "every remaining non-empty string" + }, + "unsupportedGeneralNamesRejected": true, + "mismatchedTypeReason": "CERTIFICATE_REQUEST_PROFILE_UNSUPPORTED", + "retryable": false, + "identityBoundary": "The names are mirrored only so the authority can validate proof of possession. The issued CN and sole URI SAN are rendered from Connect's signed deviceUid claim and checked after issuance." + }, "claimedDeviceUidInFixtures": "0198f4b0-8b00-7d80-9491-9fa0b1c2d3e7", "claimedSubjectAlternativeNameInFixtures": "urn:rustfs:connect:device:0198f4b0-8b00-7d80-9491-9fa0b1c2d3e7", "claimedIdentityNote": "Every certificate request in this set carries the subject CN=0198f4b0-8b00-7d80-9491-9fa0b1c2d3e7 and the matching device URN as its only subject alternative name. Connect assigned no such device, and no vector references that uid anywhere else. A verifier that reads an identity out of a certificate request will visibly agree with a value nothing else in the exchange corroborates, which is easier to notice than an omission.", - "ignoredFieldsNote": "Connect consumes a certificate request for its SubjectPublicKeyInfo and its self-signature and for nothing else. The subject, the subject alternative names, any requested extensions, and any attributes are ignored and are never copied into the issued certificate. A device cannot name itself: ADR 0008 fixes the issued subject as CN= and the SAN as urn:rustfs:connect:device:, and Connect assigns that uid during this exchange. A device has no uid to put in a certificate request, which is the structural reason the request cannot be the source of its own identity.", + "ignoredFieldsNote": "subjectUsed and sanUsed mean used as identity or copied into the certificate; both are false. Connect mirrors compatible names into a minute-scale CA authorization token only so stock step-ca can compare them with this CSR. Requested extensions and attributes remain unused. A device cannot name itself: ADR 0008 fixes the issued subject as CN= and the SAN as urn:rustfs:connect:device:, and Connect assigns that uid during this exchange. A device has no uid to put in a certificate request, which is the structural reason the request cannot be the source of its own identity.", "selfSignatureAloneIsInsufficient": "A valid self-signature proves only that somebody holds the key in the request. It binds no token, no tenant, no cluster, and no attempt, so a verifier that stopped there would issue a device certificate to any key presented with any stolen token. reject-vectors.json publishes exactly that vector under \"accepted proof presented with a substituted certificate request\"." } }, @@ -245,6 +260,7 @@ "decode the certificate request, refuse anything that is not one well-formed PKCS#10 DER with no trailing octets with CERTIFICATE_REQUEST_MALFORMED", "refuse a SubjectPublicKeyInfo that is not an ECDSA key on P-256 with DEVICE_KEY_UNSUPPORTED", "refuse a certificate request whose ES256 self-signature does not verify under its own key with CERTIFICATE_REQUEST_MALFORMED", + "refuse a certificate request whose subject or typed SANs cannot round-trip through the pinned stock step-ca authorization strings with CERTIFICATE_REQUEST_PROFILE_UNSUPPORTED", "resolve the registration token by uid and secret digest and refuse anything not usable now with REGISTRATION_TOKEN_UNUSABLE", "rebuild the transcript from the resolved row plus requestId and the recomputed certificate request digest", "verify the proof over those octets under the certificate request key and refuse with REGISTRATION_PROOF_INVALID" @@ -255,6 +271,7 @@ "SIGNATURE_NOT_CANONICAL", "CERTIFICATE_REQUEST_MALFORMED", "DEVICE_KEY_UNSUPPORTED", + "CERTIFICATE_REQUEST_PROFILE_UNSUPPORTED", "REGISTRATION_PROOF_INVALID" ], "ownedElsewhere": [ diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index 3669f62eb..396e66187 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -67,6 +67,17 @@ use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; const DIAGNOSTIC_EXPORT_PREFIX: &str = "_diagnostic"; const DIAGNOSTIC_EXPORT_MANIFEST: &str = "_diagnostic-manifest.json"; +/// Archive entry naming the configurations a bucket stores but this build +/// could not read (rustfs/backlog#2309). +/// +/// The name deliberately sits outside the configuration-file namespace the +/// importers switch on — `ImportBucketMetadata` matches known configuration +/// names and ignores everything else — so no importer can mistake the marker +/// for a configuration. Ordinary exports carry it; diagnostic exports report +/// the same failures through [`DIAGNOSTIC_EXPORT_MANIFEST`] instead, which +/// deliberately withholds the parser detail this marker records. +const EXPORT_UNREADABLE_MANIFEST: &str = "rustfs-unreadable-configs.json"; + const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_BUCKET_META: &str = "bucket_meta"; const EVENT_ADMIN_BUCKET_META_STATE: &str = "admin_bucket_meta_state"; @@ -76,31 +87,82 @@ fn export_internal_error(message: impl Into) -> s3s::S3Error { s3_error!(InternalError, "{message}") } -fn checked_raw_xml(validated: &T, raw: Vec, parse: F) -> S3Result> +/// One configuration that is stored for a bucket but could not be exported. +#[derive(serde::Serialize)] +struct UnreadableExportEntry { + config: &'static str, + error: String, +} + +#[derive(serde::Serialize)] +struct UnreadableExportManifest<'a> { + bucket: &'a str, + unreadable: &'a [UnreadableExportEntry], +} + +/// Why one of a bucket's configurations could not be exported. +/// +/// The two variants are what an ordinary export dispatches on: a bucket whose +/// stored bytes this build cannot turn into a configuration is named and +/// skipped, while a failure of our own output machinery still fails the whole +/// export closed. +#[derive(Debug)] +enum ExportConfigError { + /// The configuration is stored but this build cannot read it: a + /// MinIO-origin or otherwise undecodable blob, or a revision that moved + /// underneath the export. No retry of ours turns those bytes into a + /// configuration, so one such bucket must not abort a whole-cluster export + /// (rustfs/backlog#2309). + Unreadable(String), + /// This build failed to produce its own output for a configuration it had + /// already decoded. Nothing about the stored bytes is in doubt, so the + /// export fails closed rather than reporting healthy metadata as + /// unreadable. + Internal(s3s::S3Error), +} + +impl ExportConfigError { + fn unreadable(message: impl Into) -> Self { + Self::Unreadable(message.into()) + } + + fn internal(message: impl Into) -> Self { + Self::Internal(export_internal_error(message)) + } +} + +fn checked_raw_xml(validated: &T, raw: Vec, parse: F) -> Result, ExportConfigError> where T: PartialEq, E: std::fmt::Display, F: FnOnce(&[u8]) -> Result, { - let selected = parse(&raw) - .map_err(|e| export_internal_error(format!("persisted bucket metadata changed to invalid XML during export: {e}")))?; + let selected = parse(&raw).map_err(|e| { + ExportConfigError::unreadable(format!("persisted bucket metadata changed to invalid XML during export: {e}")) + })?; if selected != *validated { - return Err(export_internal_error("bucket metadata changed during export")); + return Err(ExportConfigError::unreadable("bucket metadata changed during export")); } Ok(raw) } -fn checked_versioning_xml(validated: &VersioningConfiguration, raw: Vec) -> S3Result> { +fn checked_versioning_xml(validated: &VersioningConfiguration, raw: Vec) -> Result, ExportConfigError> { if raw.is_empty() { if *validated != VersioningConfiguration::default() { - return Err(export_internal_error("bucket metadata changed during export")); + return Err(ExportConfigError::unreadable("bucket metadata changed during export")); } - return serialize(validated).map_err(|e| export_internal_error(format!("serialize config failed: {e}"))); + return serialize(validated).map_err(|e| ExportConfigError::internal(format!("serialize config failed: {e}"))); } checked_raw_xml(validated, raw, deserialize::) } -async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result>> { +/// Bytes to export for one of a bucket's configurations. +/// +/// `Ok(None)` means the bucket has not configured it. An `Err` never becomes an +/// exported configuration — a fabricated default here would reach an importer +/// as a real one — and its variant tells the caller whether the failure belongs +/// to the stored bytes or to this build's own output; see [`ExportConfigError`]. +async fn exported_bucket_config(bucket: &str, conf: &str) -> Result>, ExportConfigError> { match conf { BUCKET_POLICY_CONFIG => { let config: BucketPolicy = match metadata_sys::get_bucket_policy(bucket).await { @@ -109,11 +171,11 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result { @@ -123,14 +185,14 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result return Ok(None), }; let raw_config = metadata_sys::get(bucket) .await - .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))? + .map_err(|e| ExportConfigError::unreadable(format!("get bucket metadata failed: {e}")))? .notification_config_xml .clone(); let config_xml = checked_raw_xml(&config, raw_config, deserialize::)?; @@ -144,12 +206,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result)?; @@ -163,12 +225,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result)?; @@ -182,11 +244,11 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result S3Result)?; @@ -216,12 +278,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result)?; @@ -235,12 +297,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result S3Result)?; @@ -273,12 +335,12 @@ async fn exported_bucket_config(bucket: &str, conf: &str) -> S3Result = Vec::new(); for &conf in confs.iter() { let conf_path = path_join_buf(&[bucket.name.as_str(), conf]); let config = match exported_bucket_config(&bucket.name, conf).await { Ok(Some(config)) => config, Ok(None) => continue, - Err(error) if !query.diagnostic => return Err(error), - Err(_) => { - errors.push(serde_json::json!({ - "bucket": bucket.name, - "config": conf, - "code": "configuration_unavailable", - })); - continue; + Err(error) => { + if query.diagnostic { + // A diagnostic archive names every configuration it + // could not export under one fixed code, carrying + // neither the payload nor the parser detail, so it + // stays shareable (rustfs/rustfs#7225). + errors.push(serde_json::json!({ + "bucket": bucket.name, + "config": conf, + "code": "configuration_unavailable", + })); + continue; + } + match error { + // One bucket's undecodable blob must not abort the + // whole-cluster export: record which configuration + // could not be read and keep going, so an operator + // migrating away still gets every readable + // configuration (rustfs/backlog#2309). + ExportConfigError::Unreadable(error) => { + warn!( + event = EVENT_ADMIN_BUCKET_META_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_BUCKET_META, + action = "export_bucket_metadata", + result = "config_unreadable", + bucket = %bucket.name, + config_name = %conf, + error = %error, + "admin bucket meta state" + ); + unreadable.push(UnreadableExportEntry { config: conf, error }); + continue; + } + // Our own encoder failed on a configuration this + // build had already decoded. The stored bytes are + // not in question, so fail the export instead of + // reporting readable metadata as unreadable. + ExportConfigError::Internal(error) => return Err(error), + } } }; let conf_path = if query.diagnostic { @@ -404,6 +499,23 @@ impl Operation for ExportBucketMetadata { .write_all(&config) .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?; } + + // Only reachable outside diagnostic mode, which reports the same + // failures through the archive-wide manifest instead. + if !unreadable.is_empty() { + let manifest = serde_json::to_vec(&UnreadableExportManifest { + bucket: bucket.name.as_str(), + unreadable: &unreadable, + }) + .map_err(|e| export_internal_error(format!("failed to serialize unreadable manifest: {e}")))?; + let manifest_path = path_join_buf(&[bucket.name.as_str(), EXPORT_UNREADABLE_MANIFEST]); + zip_writer + .start_file(manifest_path, SimpleFileOptions::default()) + .map_err(|e| s3_error!(InternalError, "failed to start archive entry: {e}"))?; + zip_writer + .write_all(&manifest) + .map_err(|e| s3_error!(InternalError, "failed to write archive entry: {e}"))?; + } } if query.diagnostic { @@ -1364,6 +1476,10 @@ mod backup_zip_compatibility_tests { const ROOT_ACCESS_KEY: &str = "BUCKETMETABACKUPROOT"; const ROOT_SECRET_KEY: &str = "bucketMetaBackupRootSecret123"; const BUCKET: &str = "backup-compatibility"; + const UNREADABLE_BUCKET: &str = "minio-origin-targets"; + /// The exact `BucketTargetsConfigJSON` payload carried by the MinIO + /// `.metadata.bin` fixture in `crates/ecstore/src/bucket/metadata_test.rs`. + const MINIO_ARRAY_TARGETS: &[u8] = br#"[{"endpoint":"http://target.example.com","targetBucket":"tb","region":"us-east-1"}]"#; const NOTIFICATION_XML: &[u8] = b"\n"; const LIFECYCLE_XML: &[u8] = b"\nexpireEnabledlogs/30\n"; const SSE_XML: &[u8] = b"\nAES256\n"; @@ -1495,14 +1611,63 @@ mod backup_zip_compatibility_tests { .expect("publish unreadable targets fixture"); assert!(metadata_sys::get_bucket_targets_config(UNREADABLE).await.is_err()); - let strict_error = ExportBucketMetadata {} + // rustfs/backlog#2309: an ordinary export no longer fails closed on + // a configuration that is stored but unreadable. It names that one + // configuration in the bucket's own marker entry and keeps every + // readable configuration of every bucket, so one MinIO-origin blob + // cannot cost an operator the whole-cluster backup. + let ordinary = ExportBucketMetadata {} .call( admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()), Params::new(), ) .await - .expect_err("a complete export must fail closed on unreadable targets"); - assert_eq!(*strict_error.code(), s3s::S3ErrorCode::InternalError); + .expect("one unreadable configuration must not abort the ordinary export"); + assert_eq!(ordinary.output.0, StatusCode::OK); + assert!(!ordinary.headers.contains_key("x-rustfs-bucket-metadata-export")); + let ordinary_bytes = ordinary.output.1.collect().await.expect("read ordinary archive").to_bytes(); + let mut ordinary_archive = ZipArchive::new(Cursor::new(&ordinary_bytes)).expect("open ordinary archive"); + assert!( + ordinary_archive + .by_name(&format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")) + .is_ok(), + "a healthy bucket must still export while another bucket is unreadable" + ); + assert!( + ordinary_archive + .by_name(&format!("{HEALTHY}/{EXPORT_UNREADABLE_MANIFEST}")) + .is_err(), + "a bucket whose configurations all read must carry no unreadable marker" + ); + assert!( + ordinary_archive + .by_name(&format!("{UNREADABLE}/{BUCKET_TARGETS_FILE}")) + .is_err(), + "an unreadable targets blob must never be exported as a configuration" + ); + let mut ordinary_marker = Vec::new(); + ordinary_archive + .by_name(&format!("{UNREADABLE}/{EXPORT_UNREADABLE_MANIFEST}")) + .expect("the ordinary export must name the configuration it could not read") + .read_to_end(&mut ordinary_marker) + .expect("read unreadable marker"); + assert!( + !ordinary_marker + .windows(SECRET.len()) + .any(|window| window == SECRET.as_bytes()) + ); + let ordinary_marker: serde_json::Value = serde_json::from_slice(&ordinary_marker).expect("the marker must be JSON"); + assert_eq!(ordinary_marker["bucket"], UNREADABLE); + assert_eq!( + ordinary_marker["unreadable"].as_array().map(Vec::len), + Some(1), + "only the configuration that could not be read may be marked: {ordinary_marker}" + ); + assert_eq!(ordinary_marker["unreadable"][0]["config"], BUCKET_TARGETS_FILE); + assert!( + ordinary_marker["unreadable"][0]["error"].is_string(), + "the marker must carry the reason an operator needs to repair the bucket" + ); let response = ExportBucketMetadata {} .call( @@ -1676,6 +1841,12 @@ mod backup_zip_compatibility_tests { assert!(archive.by_name(DIAGNOSTIC_EXPORT_MANIFEST).is_err()); assert!(archive.by_name(&format!("{HEALTHY}/{BUCKET_VERSIONING_CONFIG}")).is_ok()); assert!(archive.by_name(&format!("{UNREADABLE}/{BUCKET_TARGETS_FILE}")).is_ok()); + assert!( + archive + .by_name(&format!("{UNREADABLE}/{EXPORT_UNREADABLE_MANIFEST}")) + .is_err(), + "the marker must disappear once the configuration reads again" + ); } #[tokio::test] @@ -1818,6 +1989,89 @@ mod backup_zip_compatibility_tests { } let _: ReplicationConfiguration = deserialize(&restored.replication_config_xml).expect("old parser must read the newly exported archive payload"); + + // rustfs/backlog#2309: a MinIO-origin `.metadata.bin` stores its + // targets as a bare JSON array, which `BucketTargets` cannot decode. + // Since rustfs/rustfs#7172 that reads as "stored but unreadable" — + // which must mark one bucket's one configuration, not abort the + // whole-cluster export an operator needs to migrate away. + env.make_bucket(UNREADABLE_BUCKET, false).await; + metadata_sys::update(UNREADABLE_BUCKET, BUCKET_TARGETS_FILE, MINIO_ARRAY_TARGETS.to_vec()) + .await + .expect("persist the MinIO-shaped targets blob"); + metadata_sys::get_bucket_targets_config(UNREADABLE_BUCKET) + .await + .expect_err("a MinIO array-shaped targets blob must read as unreadable, not as an empty set"); + + let cluster_export = ExportBucketMetadata {} + .call( + admin_request(Method::GET, Uri::from_static("/rustfs/admin/v3/export-bucket-metadata"), Vec::new()), + Params::new(), + ) + .await + .expect("one bucket's unreadable configuration must not abort the whole-cluster export"); + assert_eq!(cluster_export.output.0, StatusCode::OK); + let cluster_archive = cluster_export + .output + .1 + .collect() + .await + .expect("read cluster archive body") + .to_bytes() + .to_vec(); + let mut archive = ZipArchive::new(Cursor::new(&cluster_archive)).expect("open cluster archive"); + + // Every readable configuration of every other bucket still exports. + for (config_file, payload) in persisted_xml_fixtures() { + let mut exported_payload = Vec::new(); + archive + .by_name(&format!("{BUCKET}/{config_file}")) + .unwrap_or_else(|_| panic!("cluster export must still contain {config_file}")) + .read_to_end(&mut exported_payload) + .unwrap_or_else(|_| panic!("read exported {config_file}")); + assert_eq!(exported_payload, payload, "one bad bucket must not change another bucket's export"); + } + assert!( + archive.by_name(&format!("{BUCKET}/{EXPORT_UNREADABLE_MANIFEST}")).is_err(), + "a bucket whose configurations all read must carry no unreadable marker" + ); + + // The unreadable configuration is named rather than fabricated: no + // targets entry is exported for it at all. + assert!( + archive + .by_name(&format!("{UNREADABLE_BUCKET}/{BUCKET_TARGETS_FILE}")) + .is_err(), + "an unreadable targets blob must never be exported as a configuration" + ); + let mut marker = Vec::new(); + archive + .by_name(&format!("{UNREADABLE_BUCKET}/{EXPORT_UNREADABLE_MANIFEST}")) + .expect("the export must name the configuration it could not read") + .read_to_end(&mut marker) + .expect("read unreadable marker"); + let marker: serde_json::Value = serde_json::from_slice(&marker).expect("the marker must be JSON"); + assert_eq!(marker["bucket"], UNREADABLE_BUCKET); + assert_eq!( + marker["unreadable"].as_array().map(Vec::len), + Some(1), + "only the configuration that could not be read may be marked: {marker}" + ); + assert_eq!(marker["unreadable"][0]["config"], BUCKET_TARGETS_FILE); + drop(archive); + + // The marker cannot be misread as a configuration on the way back in: + // the importer switches on configuration names and ignores everything + // else, so the bucket's stored bytes come through untouched and the + // operator still has to repair them explicitly. + import_archive(cluster_archive).await; + let after_round_trip = metadata_sys::get_config_from_disk(UNREADABLE_BUCKET) + .await + .expect("the marked bucket must still load after the round trip"); + assert_eq!( + after_round_trip.bucket_targets_config_json, MINIO_ARRAY_TARGETS, + "importing the marker must not overwrite or fabricate the bucket's targets configuration" + ); } } diff --git a/rustfs/src/admin/handlers/ilm_transition.rs b/rustfs/src/admin/handlers/ilm_transition.rs index c418af8b3..74788d0d9 100644 --- a/rustfs/src/admin/handlers/ilm_transition.rs +++ b/rustfs/src/admin/handlers/ilm_transition.rs @@ -18,18 +18,20 @@ use crate::admin::runtime_sources::object_store_from_extensions; use crate::admin::storage_api::bucket::is_reserved_or_invalid_bucket; use crate::admin::storage_api::error::StorageError; use crate::admin::storage_api::lifecycle::{ - ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink, - ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission, - ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError, - claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current, - delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped, - finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator, + IlmRecoveryClassification, IlmRecoveryProtocol, ManualTransitionCancelCheck, ManualTransitionJobRecord, + ManualTransitionJobState, ManualTransitionProgressSink, ManualTransitionQueueSnapshot, ManualTransitionRunOptions, + ManualTransitionRunReport, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim, + TransitionOperatorDeleteResult, TransitionOperatorError, claim_manual_transition_scope_admission, + delete_manual_transition_scope_admission_if_current, delete_transition_candidate_for_operator, + enqueue_transition_for_existing_objects_scoped, finalize_missing_transition_transaction_for_operator, + inspect_recovery_control, inspect_transition_transaction_for_operator, list_recovery_controls, load_manual_transition_job_record, load_manual_transition_scope_admission, manual_transition_job_lease_expired, manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired, persist_manual_transition_job_progress_if_owned, renew_manual_transition_job_lease_if_owned, request_manual_transition_job_cancel, save_manual_transition_job_record, update_manual_transition_job_record, }; use crate::admin::storage_api::runtime::ECStore; +use crate::admin::storage_api::s3::{S3ErrorCode as AdminS3ErrorCode, error as admin_s3_error}; use crate::admin::utils::json_response; use crate::server::{ADMIN_PREFIX, RemoteAddr}; use http::HeaderMap; @@ -230,9 +232,48 @@ pub fn register_ilm_transition_route(r: &mut S3Router) -> std::i format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(), AdminOperation(&TransitionReconcileApplyHandler {}), )?; + r.insert( + Method::GET, + format!("{ADMIN_PREFIX}/v3/ilm/recovery/records").as_str(), + AdminOperation(&IlmRecoveryControlListHandler {}), + )?; + r.insert( + Method::GET, + format!("{ADMIN_PREFIX}/v3/ilm/recovery/records/{{control_id}}").as_str(), + AdminOperation(&IlmRecoveryControlInspectHandler {}), + )?; Ok(()) } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct IlmRecoveryControlListQuery { + protocol: IlmRecoveryProtocol, + #[serde(default)] + classification: Option, + #[serde(default = "default_recovery_control_list_limit")] + limit: usize, + #[serde(default)] + marker: Option, +} + +const fn default_recovery_control_list_limit() -> usize { + 100 +} + +fn parse_recovery_control_list_query(query: Option<&str>) -> S3Result { + let query = query.ok_or_else(|| admin_s3_error(AdminS3ErrorCode::InvalidRequest, "protocol is required"))?; + let parsed: IlmRecoveryControlListQuery = serde_urlencoded::from_bytes(query.as_bytes()) + .map_err(|_| admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid ILM recovery control query"))?; + if !(1..=1_000).contains(&parsed.limit) { + return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "limit must be between 1 and 1000")); + } + if parsed.marker.as_ref().is_some_and(|marker| marker.is_empty()) { + return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "marker must not be empty")); + } + Ok(parsed) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ManualTransitionRunMode { EnqueueOnly, @@ -423,6 +464,26 @@ fn transition_transaction_id_from_params(params: &Params<'_, '_>) -> S3Result) -> S3Result { + let control_id = params.get("control_id").unwrap_or(""); + if control_id.len() != 64 + || !control_id + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err(admin_s3_error(AdminS3ErrorCode::InvalidArgument, "invalid ILM recovery control id")); + } + Ok(control_id.to_string()) +} + +fn map_recovery_control_error(err: StorageError) -> S3Error { + if err == StorageError::ConfigNotFound { + admin_s3_error(AdminS3ErrorCode::NoSuchKey, "ILM recovery control not found") + } else { + admin_s3_error(AdminS3ErrorCode::InternalError, "ILM recovery control request failed") + } +} + fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error { match err { TransitionOperatorError::NotFound => s3_error!(NoSuchKey, "transition transaction not found"), @@ -1031,6 +1092,40 @@ impl Operation for TransitionReconcileInspectHandler { } } +pub struct IlmRecoveryControlListHandler {} + +#[async_trait::async_trait] +impl Operation for IlmRecoveryControlListHandler { + async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { + authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?; + let query = parse_recovery_control_list_query(req.uri.query())?; + let Some(store) = object_store_from_extensions(&req.extensions) else { + return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized")); + }; + let page = list_recovery_controls(store, query.protocol, query.classification, query.limit, query.marker) + .await + .map_err(map_recovery_control_error)?; + json_response(StatusCode::OK, &page) + } +} + +pub struct IlmRecoveryControlInspectHandler {} + +#[async_trait::async_trait] +impl Operation for IlmRecoveryControlInspectHandler { + async fn call(&self, req: S3Request, params: Params<'_, '_>) -> S3Result> { + authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?; + let control_id = recovery_control_id_from_params(¶ms)?; + let Some(store) = object_store_from_extensions(&req.extensions) else { + return Err(admin_s3_error(AdminS3ErrorCode::InternalError, "object store is not initialized")); + }; + let control = inspect_recovery_control(store, &control_id) + .await + .map_err(map_recovery_control_error)?; + json_response(StatusCode::OK, &control) + } +} + pub struct TransitionReconcileApplyHandler {} #[async_trait::async_trait] @@ -1104,6 +1199,49 @@ mod tests { f(&matched.params) } + fn with_recovery_control_params(path: &str, f: impl FnOnce(&Params<'_, '_>) -> T) -> T { + let mut router = Router::new(); + router + .insert("/rustfs/admin/v3/ilm/recovery/records/{control_id}", ()) + .expect("route should insert"); + let matched = router.at(path).expect("route should match"); + f(&matched.params) + } + + #[test] + fn recovery_control_query_is_bounded_and_strict() { + let query = parse_recovery_control_list_query(Some("protocol=transition_transaction")) + .expect("minimal recovery query should parse"); + assert_eq!(query.protocol, IlmRecoveryProtocol::TransitionTransaction); + assert_eq!(query.classification, None); + assert_eq!(query.limit, 100); + + let filtered = parse_recovery_control_list_query(Some( + "protocol=tier_delete_journal&classification=retained_ambiguous&limit=1000&marker=opaque", + )) + .expect("bounded filtered query should parse"); + assert_eq!(filtered.protocol, IlmRecoveryProtocol::TierDeleteJournal); + assert_eq!(filtered.classification, Some(IlmRecoveryClassification::RetainedAmbiguous)); + assert_eq!(filtered.limit, 1000); + assert!(parse_recovery_control_list_query(None).is_err()); + assert!(parse_recovery_control_list_query(Some("protocol=transition_transaction&limit=0")).is_err()); + assert!(parse_recovery_control_list_query(Some("protocol=transition_transaction&limit=1001")).is_err()); + assert!(parse_recovery_control_list_query(Some("protocol=unknown")).is_err()); + assert!(parse_recovery_control_list_query(Some("protocol=transition_transaction&extra=true")).is_err()); + } + + #[test] + fn recovery_control_id_is_canonical_lowercase_sha256() { + let id = "ab".repeat(32); + with_recovery_control_params(&format!("/rustfs/admin/v3/ilm/recovery/records/{id}"), |params| { + assert_eq!(recovery_control_id_from_params(params).expect("control id should parse"), id); + }); + let uppercase = "AB".repeat(32); + with_recovery_control_params(&format!("/rustfs/admin/v3/ilm/recovery/records/{uppercase}"), |params| { + assert!(recovery_control_id_from_params(params).is_err()) + }); + } + fn manual_transition_job_request(method: Method, path: &'static str) -> S3Request { S3Request { input: Body::empty(), diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index b7bf5ef78..00a7ca6df 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -29,7 +29,7 @@ use crate::admin::storage_api::bucket::replication::{REMOTE_TARGET_READ_ONLY_HIS use crate::admin::storage_api::bucket::target::{ BucketTarget, BucketTargetType, Credentials as TargetCredentials, LatencyStat, duration_from_secs_or_nanos, }; -use crate::admin::storage_api::bucket::target_sys::{BucketTargetError, BucketTargetSys}; +use crate::admin::storage_api::bucket::target_sys::{BucketTargetError, BucketTargetSys, UnreadableTargetsPolicy}; use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions}; use crate::admin::storage_api::contract::list::ListOperations as _; use crate::admin::storage_api::error::StorageError; @@ -57,6 +57,20 @@ use url::Host; const SUPPORTED_REMOTE_TARGET_API: &str = "s3v4"; +const LOG_COMPONENT_ADMIN: &str = "admin"; +const LOG_SUBSYSTEM_REPLICATION: &str = "replication"; +const EVENT_ADMIN_REMOTE_TARGET_STATE: &str = "admin_remote_target_state"; + +/// `set-remote-target?replace-unreadable=true`: the operator's explicit +/// acknowledgement that this bucket's persisted target set cannot be decoded +/// and is to be discarded (rustfs/backlog#2309). +/// +/// Without it the refusal from rustfs/rustfs#7172 stands, which is what keeps +/// an unreadable set from being silently rewritten from a partial view. The +/// flag is deliberately absent from `PutBucketReplication`: targets are +/// repaired first, then the rule is set. +const REPLACE_UNREADABLE_TARGETS_PARAM: &str = "replace-unreadable"; + /// Field groups a `set-remote-target?update=true` request may modify, mirroring /// MinIO's `TargetUpdateType` / `GetTargetUpdateOps` query contract: the update /// overlays only the requested groups onto the stored target, so a client can @@ -541,6 +555,7 @@ impl Operation for SetRemoteTargetHandler { }; let update = queries.get("update").is_some_and(|v| v == "true"); + let replace_unreadable = queries.get(REPLACE_UNREADABLE_TARGETS_PARAM).is_some_and(|v| v == "true"); warn!("set remote target, bucket: {}, update: {}", bucket, update); @@ -708,10 +723,37 @@ impl Operation for SetRemoteTargetHandler { let arn = remote_target.arn.clone(); + let unreadable_policy = if replace_unreadable { + UnreadableTargetsPolicy::Replace + } else { + UnreadableTargetsPolicy::FailClosed + }; + let discarding_unreadable_targets = replace_unreadable + && matches!( + bucket_target_sys.list_bucket_targets(bucket).await, + Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) + ); + let targets = bucket_target_sys - .set_target(bucket, &remote_target, update) + .set_target(bucket, &remote_target, update, unreadable_policy) .await .map_err(map_bucket_target_error)?; + + // Audited only where the discard actually happened: the flag alone is + // not an event, on a readable set it changes nothing, and a refused + // write must not leave a record claiming the set was replaced. + if discarding_unreadable_targets { + warn!( + event = EVENT_ADMIN_REMOTE_TARGET_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_REPLICATION, + action = "set_remote_target", + result = "unreadable_targets_replaced", + bucket = %bucket, + arn = %remote_target.arn, + "admin remote target state" + ); + } let json_targets = serde_json::to_vec(&targets).map_err(|e| { error!("Serialization error: {}", e); S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize targets".to_string()) diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 310212cc4..b9bf3db67 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -613,6 +613,16 @@ impl Operation for RemoveTier { return if err.code == ERR_TIER_NOT_FOUND.code { Err(S3Error::with_message(S3ErrorCode::Custom("TierNotFound".into()), "tier not found")) } else if let Some(response) = tier_backend_error_response(&err) { + warn!( + event = EVENT_ADMIN_TIER_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_TIER, + action = "remove_tier", + tier_name = %tier_name, + result = "remove_blocked", + error = ?err, + "admin tier state" + ); Err(response) } else { warn!( diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index b18bd4012..69c61768f 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -495,6 +495,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[ SERVER_INFO, RouteRiskLevel::Sensitive, ), + admin( + HttpMethod::Get, + "/rustfs/admin/v3/ilm/recovery/records", + LIST_TIER, + RouteRiskLevel::Sensitive, + ), + admin( + HttpMethod::Get, + "/rustfs/admin/v3/ilm/recovery/records/{control_id}", + LIST_TIER, + RouteRiskLevel::Sensitive, + ), admin(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER, RouteRiskLevel::High), admin( HttpMethod::Get, @@ -2159,12 +2171,15 @@ mod tests { #[test] fn route_policy_uses_tier_actions_for_transition_routes() { + assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/recovery/records", LIST_TIER); + assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/recovery/records/{control_id}", LIST_TIER); assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER); assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER); assert_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER); assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER); assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER); assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SERVER_INFO); + assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/recovery/records", SERVER_INFO); assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO); assert_not_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO); assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER); diff --git a/rustfs/src/admin/route_registration_test.rs b/rustfs/src/admin/route_registration_test.rs index 5a8e65694..eec928968 100644 --- a/rustfs/src/admin/route_registration_test.rs +++ b/rustfs/src/admin/route_registration_test.rs @@ -209,6 +209,12 @@ fn expected_admin_route_matrix() -> Vec { admin_route_sample(Method::POST, "/v3/tier/{tiername}", "/v3/tier/HOT"), admin_route(Method::POST, "/v3/tier/clear"), admin_route(Method::GET, "/v3/ilm/expiry/status"), + admin_route(Method::GET, "/v3/ilm/recovery/records"), + admin_route_sample( + Method::GET, + "/v3/ilm/recovery/records/{control_id}", + "/v3/ilm/recovery/records/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ), admin_route(Method::POST, "/v3/ilm/transition/run"), admin_route_sample( Method::GET, @@ -930,6 +936,12 @@ fn test_register_routes_cover_representative_admin_paths() { assert_route(&router, Method::POST, &admin_path("/v3/scanner/cycle-state/reset")); assert_route(&router, Method::POST, &admin_path("/v3/scanner/usage-state/reset")); assert_route(&router, Method::GET, &admin_path("/v3/ilm/expiry/status")); + assert_route(&router, Method::GET, &admin_path("/v3/ilm/recovery/records")); + assert_route( + &router, + Method::GET, + &admin_path("/v3/ilm/recovery/records/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ); assert_route(&router, Method::POST, &admin_path("/v3/ilm/transition/run")); assert_route( &router, diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 01caaa95f..111fe52ce 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -212,6 +212,7 @@ pub(crate) mod bucket_target_sys { pub(crate) type S3ClientError = super::ecstore_bucket::bucket_target_sys::S3ClientError; pub(crate) type SsecPassthroughCapability = super::ecstore_bucket::bucket_target_sys::SsecPassthroughCapability; pub(crate) type TargetClient = super::ecstore_bucket::bucket_target_sys::TargetClient; + pub(crate) type UnreadableTargetsPolicy = super::ecstore_bucket::bucket_target_sys::UnreadableTargetsPolicy; } pub(crate) mod lifecycle { @@ -232,6 +233,9 @@ pub(crate) mod lifecycle { pub(crate) type ManualTransitionRunOptions = super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunOptions; pub(crate) type ManualTransitionRunReport = super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunReport; + pub(crate) use super::ecstore_bucket::lifecycle::recovery_control::{ + IlmRecoveryClassification, IlmRecoveryProtocol, inspect_recovery_control, list_recovery_controls, + }; pub(crate) use super::ecstore_bucket::lifecycle::transition_transaction::{ TransitionOperatorDeleteResult, TransitionOperatorError, delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator, diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 0459e2ea6..a596a5c70 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -27,8 +27,8 @@ use super::storage_api::bucket_usecase::bucket::target::BucketTarget; use super::storage_api::bucket_usecase::bucket::{ ObjectLockConfigExt as _, VersioningConfigExt as _, lifecycle::bucket_lifecycle_ops::{ - enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, run_stale_multipart_upload_cleanup_once, - validate_lifecycle_config, validate_transition_tier, + LIFECYCLE_MALFORMED_XML_ERROR_KIND, enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, + run_stale_multipart_upload_cleanup_once, validate_lifecycle_config, validate_transition_tier, }, metadata::{ BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, @@ -1188,6 +1188,21 @@ fn validate_lifecycle_rule_status(rules: &[LifecycleRule]) -> std::result::Resul Ok(()) } +/// Map a lifecycle validation failure onto the S3 error the client should see. +/// +/// The validator reports a schema-shape violation (a `Filter` with more than +/// one predicate, a one-member `And`) with +/// [`LIFECYCLE_MALFORMED_XML_ERROR_KIND`]; AWS answers those with +/// `MalformedXML`. Everything else is a value the schema allows but S3 refuses, +/// which stays `InvalidArgument` — the code this path has always returned +/// (backlog#2201). +fn lifecycle_validation_error(err: &std::io::Error) -> S3Error { + if err.kind() == LIFECYCLE_MALFORMED_XML_ERROR_KIND { + return S3Error::with_message(S3ErrorCode::MalformedXML, format!("Malformed XML: {err}")); + } + s3_error!(InvalidArgument, "{err}") +} + fn lifecycle_has_transition_rules(config: &BucketLifecycleConfiguration) -> bool { config.rules.iter().any(|rule| { rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED) @@ -2296,7 +2311,7 @@ impl DefaultBucketUsecase { }; if let Err(err) = validate_lifecycle_config(&input_cfg, &rcfg).await { - return Err(s3_error!(InvalidArgument, "{err}")); + return Err(lifecycle_validation_error(&err)); } if let Err(err) = validate_transition_tier(&input_cfg).await { @@ -4030,6 +4045,70 @@ mod tests { assert_eq!(rules[2].id.as_deref(), Some("rule-2")); } + #[tokio::test] + async fn put_bucket_lifecycle_validation_errors_keep_their_s3_code() { + // The PUT path answers a schema-shape violation with MalformedXML and a + // rejected value with InvalidArgument. Both categories are produced by + // the real validator here, so the mapping cannot drift from it + // (backlog#2201). + let malformed = validate_lifecycle_config( + &BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + days: Some(1), + ..Default::default() + }), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: Some(s3s::dto::LifecycleRuleFilter { + prefix: Some("logs/".to_string()), + tag: Some(s3s::dto::Tag { + key: Some("env".to_string()), + value: Some("prod".to_string()), + }), + ..Default::default() + }), + id: Some("two-predicates".to_string()), + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }, + &ObjectLockConfiguration::default(), + ) + .await + .expect_err("a Filter with two predicates is a schema violation"); + assert_eq!(*lifecycle_validation_error(&malformed).code(), S3ErrorCode::MalformedXML); + + let invalid_value = validate_lifecycle_config( + &BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: None, + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + id: Some("negative-count".to_string()), + noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration { + noncurrent_days: Some(30), + newer_noncurrent_versions: Some(-1), + }), + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }, + &ObjectLockConfiguration::default(), + ) + .await + .expect_err("a negative retention count is rejected"); + assert_eq!(*lifecycle_validation_error(&invalid_value).code(), S3ErrorCode::InvalidArgument); + } + #[test] fn validate_lifecycle_rule_status_rejects_invalid_status() { let rules = vec![LifecycleRule { diff --git a/rustfs/src/app/object/on_demand_migration_put.rs b/rustfs/src/app/object/on_demand_migration_put.rs index 86ba3972a..33069d3e3 100644 --- a/rustfs/src/app/object/on_demand_migration_put.rs +++ b/rustfs/src/app/object/on_demand_migration_put.rs @@ -692,9 +692,16 @@ mod tests { } } - #[tokio::test] + #[test] #[serial_test::serial] - async fn write_back_multipart_completion_preserves_a_client_put_after_staging() { + fn write_back_multipart_completion_preserves_a_client_put_after_staging() { + crate::app::gating_test_env::run_large_stack_test( + "odm-write-back-multipart-client-put-race", + write_back_multipart_completion_preserves_a_client_put_after_staging_inner, + ); + } + + async fn write_back_multipart_completion_preserves_a_client_put_after_staging_inner() { let (store, bucket) = write_back_test_bucket("odm-mpu-race", false).await; let write_back = OnDemandMigrationWriteBack::new(); let req = request( diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 4e5ae8cc4..2719ccb72 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -398,6 +398,12 @@ pub(crate) mod bucket { lc.validate(lock_config).await } + + /// The `std::io::ErrorKind` [`validate_lifecycle_config`] uses for a + /// lifecycle document that violates the published schema shape, which + /// the S3 boundary answers with `MalformedXML` (backlog#2201). + pub(crate) const LIFECYCLE_MALFORMED_XML_ERROR_KIND: std::io::ErrorKind = + crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::LIFECYCLE_MALFORMED_XML_ERROR_KIND; } pub(crate) mod lifecycle_contract { diff --git a/rustfs/src/connect/identity.rs b/rustfs/src/connect/identity.rs index 8ce7bb890..413b1643f 100644 --- a/rustfs/src/connect/identity.rs +++ b/rustfs/src/connect/identity.rs @@ -220,8 +220,10 @@ impl DeviceIdentity { /// Build the PKCS#10 certificate request Connect consumes. /// /// Connect reads the request for its SubjectPublicKeyInfo and its - /// self-signature and for nothing else: it assigns the device uid itself, - /// so the subject and SAN carried here name nothing Connect will honour. + /// self-signature. The generated profile deliberately has no subject + /// alternative name, so the stock CA authorization path cannot + /// reinterpret an untyped name as a different ASN.1 GeneralName. Connect + /// assigns the issued subject and device URI itself. pub fn certificate_request_der(&self) -> Result, IdentityError> { let pkcs8 = self.to_pkcs8_der()?; let key_pair = diff --git a/rustfs/src/connect/offline/enrollment.rs b/rustfs/src/connect/offline/enrollment.rs index 350c4ac28..ba25a92b4 100644 --- a/rustfs/src/connect/offline/enrollment.rs +++ b/rustfs/src/connect/offline/enrollment.rs @@ -110,6 +110,9 @@ const PUBLIC_KEY_CHARS: usize = 87; /// SEC1 tag of an uncompressed point. Compressed and hybrid forms are refused. const UNCOMPRESSED_POINT: u8 = 0x04; +const KEY_ID_CHARS: usize = 64; +const SERIAL_CHARS: usize = 32; + const TIMESTAMP_CHARS: usize = 20; /// The chain is exactly two links: a pinned root issues the intermediate, and @@ -117,6 +120,7 @@ const TIMESTAMP_CHARS: usize = 20; /// enumeration is closed. const CHAIN_LINK_COUNT: usize = 2; const CHAIN_ROLES: [&str; CHAIN_LINK_COUNT] = ["intermediate", "signing"]; +const CHAIN_MAX_VALIDITY_SECONDS: [i64; CHAIN_LINK_COUNT] = [31_536_000, 2_678_400]; /// Skew allowed on the challenge window. A device may have no synchronised /// clock at all, so its own reading of "now" is advisory. @@ -212,9 +216,7 @@ pub enum EnrollmentError { /// The artifact could not be read as a signed enrolment document at all: the /// envelope, the base64 of the signed octets, or a field the frozen order - /// reads before the signature verifies did not parse. The frozen reason set - /// has no code for a structurally unreadable document, so this variant maps - /// to none of them. + /// reads before the signature verifies did not parse. #[error("the offline enrollment document is not well formed")] MalformedDocument, @@ -253,7 +255,7 @@ impl EnrollmentError { Self::EnrollmentReplayed => "ENROLLMENT_REPLAYED", Self::OrganizationMismatch => "ORGANIZATION_MISMATCH", Self::ClusterMismatch => "CLUSTER_MISMATCH", - Self::MalformedDocument => "MALFORMED_DOCUMENT", + Self::MalformedDocument => "DOCUMENT_MALFORMED", Self::ResponseNotProduced => "RESPONSE_NOT_PRODUCED", } } @@ -269,6 +271,21 @@ struct SignedDocument { signature: DocumentSignature, } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct UnverifiedSignedDocument { + bytes: String, + signature: UnverifiedDocumentSignature, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct UnverifiedDocumentSignature { + algorithm: Option, + key_id: Option, + value: Option, +} + #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] struct DocumentSignature { @@ -284,7 +301,13 @@ struct DocumentSignature { struct ChallengeRouting { connect_key_id: String, issued_at: String, - trust_chain: Vec, + trust_chain: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct TrustLinkRouting { + issuer_key_id: String, } #[derive(Deserialize)] @@ -306,6 +329,7 @@ struct ChallengeDocument { struct TrustLink { format_version: String, protocol_version: String, + serial: String, role: String, issuer_key_id: String, subject_key_id: String, @@ -355,30 +379,67 @@ impl OfflineEnrollment { now_unix: i64, root: EnrollmentRoot, ) -> Result { - let envelope: SignedDocument = serde_json::from_slice(document).map_err(|_| EnrollmentError::MalformedDocument)?; + let envelope: UnverifiedSignedDocument = + serde_json::from_slice(document).map_err(|_| EnrollmentError::MalformedDocument)?; - // Step 1: the encoding is checked before anything is decoded from it, so - // a DER, padded, truncated, out-of-range, or high-S signature is refused - // on its spelling rather than handed to a library that would accept it. - let signature = decode_signature(&envelope.signature)?; + // Decode the exact transmitted octets before reading any routing field. + // Standard padded base64 is canonical in this protocol: accepting an + // alternate spelling would give one signed document multiple artifact + // identities. + let bytes = decode_document_bytes(&envelope.bytes)?; - // The octets that were transmitted. They are never re-serialised: every - // later step signs and parses this same buffer. - let bytes = BASE64_STANDARD - .decode_to_vec(envelope.bytes.as_bytes()) - .map_err(|_| EnrollmentError::MalformedDocument)?; - - // Step 2: routing only. + // Routing only. These values are still untrusted, but they must be + // structurally usable before the signature and trust decisions can be + // made in their frozen order. let routing: ChallengeRouting = serde_json::from_slice(&bytes).map_err(|_| EnrollmentError::MalformedDocument)?; + if !is_key_id(&routing.connect_key_id) { + return Err(EnrollmentError::MalformedDocument); + } let issued_at = parse_timestamp(&routing.issued_at)?; - // Steps 3 to 5. - let connect_key = verify_trust_chain(&routing.trust_chain, &routing.connect_key_id, issued_at, root)?; + // The frozen decision order classifies the top-level signature before + // parsing any trust-link envelope or routing fields. Otherwise a + // malformed link could mask a malformed artifact signature with + // DOCUMENT_MALFORMED. + let signature_member = |value: Option| { + value + .and_then(|value| value.as_str().map(str::to_owned)) + .ok_or(EnrollmentError::SignatureMalformed) + }; + let signature_document = DocumentSignature { + algorithm: signature_member(envelope.signature.algorithm)?, + key_id: signature_member(envelope.signature.key_id)?, + value: signature_member(envelope.signature.value)?, + }; + let signature = decode_signature(&signature_document)?; - // Step 6. The verification key comes from the chain, so `signature.keyId` - // is a label rather than an input: a value naming some other key simply - // fails to verify here. - if !verifies(&connect_key, TAG_CHALLENGE, &bytes, &signature) { + let first = routing.trust_chain.first().ok_or(EnrollmentError::MalformedDocument)?; + let first_bytes = first + .get("bytes") + .and_then(serde_json::Value::as_str) + .ok_or(EnrollmentError::MalformedDocument) + .and_then(decode_document_bytes)?; + let first_routing: TrustLinkRouting = + serde_json::from_slice(&first_bytes).map_err(|_| EnrollmentError::MalformedDocument)?; + if !is_key_id(&first_routing.issuer_key_id) { + return Err(EnrollmentError::MalformedDocument); + } + + // The pinned-root decision precedes full chain-envelope validation. + if first_routing.issuer_key_id != root.key_id { + return Err(EnrollmentError::EnrollmentRootUnknown); + } + let trust_chain: Vec = serde_json::from_value(serde_json::Value::Array(routing.trust_chain)) + .map_err(|_| EnrollmentError::TrustChainInvalid)?; + + // Steps 3 to 5. + let connect_key = + verify_trust_chain(&trust_chain, &routing.connect_key_id, &first_routing.issuer_key_id, issued_at, root)?; + + // Step 6. The detached signature must name the same chained key whose + // public key verifies it. A different well-formed key id is a signature + // failure, not an opportunity to ignore the binding. + if signature_document.key_id != routing.connect_key_id || !verifies(&connect_key, TAG_CHALLENGE, &bytes, &signature) { return Err(EnrollmentError::SignatureInvalid); } @@ -403,7 +464,7 @@ impl OfflineEnrollment { issued_at: challenge.issued_at, expires_at: challenge.expires_at, connect_key_id: challenge.connect_key_id, - challenge_proof: envelope.signature.value, + challenge_proof: signature_document.value, }) } @@ -478,6 +539,7 @@ fn e2e_root() -> Result { fn verify_trust_chain( chain: &[SignedDocument], connect_key_id: &str, + first_issuer_key_id: &str, challenge_issued_at: i64, root: EnrollmentRoot, ) -> Result { @@ -486,10 +548,10 @@ fn verify_trust_chain( // trust on first use would have accepted — is refused for its root rather // than for its length. let first = chain.first().ok_or(EnrollmentError::EnrollmentRootUnknown)?; - let first_link = decode_trust_link(first)?; - if first_link.0.issuer_key_id != root.key_id { + if first_issuer_key_id != root.key_id { return Err(EnrollmentError::EnrollmentRootUnknown); } + let first_link = decode_trust_link(first)?; let [_, second] = chain else { return Err(EnrollmentError::TrustChainInvalid); @@ -502,8 +564,11 @@ fn verify_trust_chain( for (index, ((link, link_bytes), entry)) in links.iter().zip(chain).enumerate() { if link.format_version != FORMAT_TRUST_LINK || link.protocol_version != PROTOCOL_VERSION + || !is_serial(&link.serial) || link.role != CHAIN_ROLES[index] || link.issuer_key_id != issuer_key_id + || !is_key_id(&link.issuer_key_id) + || !is_key_id(&link.subject_key_id) // A link that names itself as its own issuer would let a stolen // intermediate mint its own root. || link.subject_key_id == link.issuer_key_id @@ -517,8 +582,8 @@ fn verify_trust_chain( return Err(EnrollmentError::TrustChainInvalid); } - let signature = decode_signature(&entry.signature)?; - if !verifies(&issuer_key, TAG_TRUST_LINK, link_bytes, &signature) { + let signature = decode_signature(&entry.signature).map_err(|_| EnrollmentError::TrustChainInvalid)?; + if entry.signature.key_id != link.issuer_key_id || !verifies(&issuer_key, TAG_TRUST_LINK, link_bytes, &signature) { return Err(EnrollmentError::TrustChainInvalid); } @@ -526,9 +591,12 @@ fn verify_trust_chain( // with no skew tolerance, and against the challenge's issuedAt rather // than against the device clock: a challenge carries the chain that was // valid when it was issued. - let not_before = parse_timestamp(&link.not_before)?; - let not_after = parse_timestamp(&link.not_after)?; - if challenge_issued_at < not_before || challenge_issued_at > not_after { + let not_before = parse_timestamp(&link.not_before).map_err(|_| EnrollmentError::TrustChainInvalid)?; + let not_after = parse_timestamp(&link.not_after).map_err(|_| EnrollmentError::TrustChainInvalid)?; + if !link_validity_allowed(not_before, not_after, CHAIN_MAX_VALIDITY_SECONDS[index]) + || challenge_issued_at < not_before + || challenge_issued_at > not_after + { return Err(EnrollmentError::TrustChainInvalid); } @@ -546,20 +614,33 @@ fn verify_trust_chain( /// Decode a link and keep the octets it was signed over: the signature is /// checked against these, never against a re-encoding of the parsed link. fn decode_trust_link(entry: &SignedDocument) -> Result<(TrustLink, Vec), EnrollmentError> { - let bytes = BASE64_STANDARD - .decode_to_vec(entry.bytes.as_bytes()) - .map_err(|_| EnrollmentError::MalformedDocument)?; + let bytes = decode_document_bytes(&entry.bytes).map_err(|_| EnrollmentError::TrustChainInvalid)?; let link = serde_json::from_slice(&bytes).map_err(|_| EnrollmentError::TrustChainInvalid)?; Ok((link, bytes)) } +fn decode_document_bytes(value: &str) -> Result, EnrollmentError> { + if !value.len().is_multiple_of(4) { + return Err(EnrollmentError::MalformedDocument); + } + + let decoded = BASE64_STANDARD + .decode_to_vec(value.as_bytes()) + .map_err(|_| EnrollmentError::MalformedDocument)?; + if BASE64_STANDARD.encode_to_string(&decoded) != value { + return Err(EnrollmentError::MalformedDocument); + } + + Ok(decoded) +} + /// Check a signature's spelling and range, then admit it. /// /// `r` and `s` are compared against the group order here rather than left to /// the ECDSA library, because a library that accepts high-S — every library /// does — would let a malleated copy of an artifact pass as a second artifact. fn decode_signature(signature: &DocumentSignature) -> Result { - if signature.algorithm != SIGNATURE_ALGORITHM { + if signature.algorithm != SIGNATURE_ALGORITHM || !is_key_id(&signature.key_id) { return Err(EnrollmentError::SignatureMalformed); } @@ -590,6 +671,24 @@ fn decode_signature(signature: &DocumentSignature) -> Result bool { + value.len() == KEY_ID_CHARS + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn is_serial(value: &str) -> bool { + value.len() == SERIAL_CHARS + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn link_validity_allowed(not_before: i64, not_after: i64, maximum: i64) -> bool { + not_after > not_before && not_after - not_before <= maximum +} + fn verifies(key: &VerifyingKey, tag: &[u8], bytes: &[u8], signature: &Signature) -> bool { key.verify(&signature_input(tag, bytes), signature).is_ok() } @@ -723,3 +822,41 @@ fn check_challenge_window(issued_at: i64, expires_at: i64, at: i64) -> Result<() Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn trust_link_validity_limits_match_the_frozen_boundary_vectors() { + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../../protocol/agent/v1/fixtures/offline-enrollment/boundary-vectors.json" + )) + .expect("boundary vectors parse"); + let vectors = fixture["linkValidityPolicyVectors"] + .as_array() + .expect("link validity policy vectors are a list"); + + for vector in vectors { + let name = vector["name"].as_str().expect("vector has a name"); + let role = vector["role"].as_str().expect("vector has a role"); + let index = CHAIN_ROLES + .iter() + .position(|candidate| *candidate == role) + .expect("known chain role"); + let not_before = + parse_timestamp(vector["notBefore"].as_str().expect("notBefore is a string")).expect("notBefore is an instant"); + let not_after = + parse_timestamp(vector["notAfter"].as_str().expect("notAfter is a string")).expect("notAfter is an instant"); + let expected = vector["expectedReason"].is_null(); + + assert_eq!( + link_validity_allowed(not_before, not_after, CHAIN_MAX_VALIDITY_SECONDS[index]), + expected, + "link validity policy vector '{name}'" + ); + } + + assert_eq!(vectors.len(), 4, "boundary-vectors.json publishes four link-validity vectors"); + } +} diff --git a/rustfs/src/on_demand_migration/gcs.rs b/rustfs/src/on_demand_migration/gcs.rs index bce647b97..5b43ca6ec 100644 --- a/rustfs/src/on_demand_migration/gcs.rs +++ b/rustfs/src/on_demand_migration/gcs.rs @@ -561,4 +561,41 @@ mod tests { } } } + + /// GCS states its error code in the response body, which this backend never + /// reads, so every class must follow from the status alone. The classes are + /// what the runtime acts on: only `NotFound` is negative-cached, and only a + /// retryable class may be re-sent rather than counted against the breaker. + /// The 404 row scripts the readable-bucket probe as well, because a GCS + /// object 404 is only a key miss once the bucket has answered + /// (`object_404_requires_a_readable_source_bucket` pins that rule). + #[tokio::test] + async fn gcs_statuses_map_onto_the_shared_error_classes() { + for method in [Method::HEAD, Method::GET] { + for (status, expected, retryable) in [ + (404_u16, "not_found", false), + (403, "access_denied", false), + (401, "access_denied", false), + (429, "throttled", true), + (503, "throttled", true), + (500, "server_error", true), + (502, "server_error", true), + ] { + let mut script = vec![ScriptedResponse::new(status, Vec::new(), String::new())]; + if status == 404 { + script.push(ScriptedResponse::new(200, Vec::new(), "{}".to_string())); + } + let (endpoint, _) = scripted_server(script).await; + let backend = backend(&endpoint); + let result = if method == Method::HEAD { + backend.head("a.txt").await.map(|_| ()) + } else { + backend.get("a.txt", None).await.map(|_| ()) + }; + let err = result.expect_err("a non-2xx status must fail"); + assert_eq!(err.class_label(), expected, "{method} HTTP {status}: {err:?}"); + assert_eq!(err.is_retryable(), retryable, "{method} HTTP {status}: {err:?}"); + } + } + } } diff --git a/rustfs/tests/connect_identity.rs b/rustfs/tests/connect_identity.rs index 2c4ed14ef..c76dc19ba 100644 --- a/rustfs/tests/connect_identity.rs +++ b/rustfs/tests/connect_identity.rs @@ -23,6 +23,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use base64_simd::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD; use rustfs::connect::identity::{DeviceIdentity, IdentityError, RegistrationTranscript}; use rustfs::connect::identity_store::{IdentityStore, StoreError}; +use x509_parser::prelude::{FromDer as _, X509CertificationRequest}; fn transcript_fixture() -> serde_json::Value { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../protocol/agent/v1/fixtures/registration/transcript.json"); @@ -34,6 +35,11 @@ fn accept_vectors() -> serde_json::Value { serde_json::from_slice(&fs::read(path).expect("read accept-vectors.json")).expect("accept-vectors.json parses") } +fn reject_vectors() -> serde_json::Value { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../protocol/agent/v1/fixtures/registration/reject-vectors.json"); + serde_json::from_slice(&fs::read(path).expect("read reject-vectors.json")).expect("reject-vectors.json parses") +} + /// Extract the SubjectPublicKeyInfo from a PKCS#10 request. /// /// The protocol freezes the DER prefix of a P-256 SubjectPublicKeyInfo, and the @@ -359,6 +365,33 @@ fn certificate_request_presents_a_p256_key() { assert_eq!(der[0], 0x30, "a PKCS#10 request is a DER SEQUENCE"); } +#[test] +fn certificate_request_uses_the_frozen_no_san_authorization_profile() { + let profile = &transcript_fixture()["request"]["certificateRequestProfile"]["authorizationCompatibility"]; + assert_eq!(profile["noSubjectAlternativeNameAccepted"].as_bool(), Some(true)); + assert_eq!(profile["mismatchedTypeReason"].as_str(), Some("CERTIFICATE_REQUEST_PROFILE_UNSUPPORTED")); + + let identity = DeviceIdentity::generate(); + let der = identity.certificate_request_der().expect("certificate request builds"); + let (remaining, request) = X509CertificationRequest::from_der(&der).expect("certificate request parses"); + assert!(remaining.is_empty(), "the certificate request has no trailing octets"); + assert!( + request + .requested_extensions() + .is_none_or(|mut extensions| extensions.next().is_none()), + "RustFS must not send a subject alternative name in its registration request" + ); + + let profile_vector = reject_vectors()["vectors"] + .as_array() + .expect("reject vectors are a list") + .iter() + .find(|vector| vector["expected"]["reason"] == "CERTIFICATE_REQUEST_PROFILE_UNSUPPORTED") + .expect("the frozen profile rejection vector exists") + .clone(); + assert_eq!(profile_vector["stage"].as_str(), Some("certificateRequest")); +} + fn hex_to_bytes(hex: &str) -> Vec { (0..hex.len()) .step_by(2) diff --git a/rustfs/tests/connect_offline_enrollment.rs b/rustfs/tests/connect_offline_enrollment.rs index 0e3810a5f..24e5dd891 100644 --- a/rustfs/tests/connect_offline_enrollment.rs +++ b/rustfs/tests/connect_offline_enrollment.rs @@ -28,6 +28,7 @@ use std::fs; use std::path::PathBuf; +use std::sync::RwLock; use base64_simd::STANDARD as BASE64_STANDARD; use base64_simd::URL_SAFE_NO_PAD as BASE64_URL_NO_PAD; @@ -45,6 +46,11 @@ const SPKI_PREFIX_HEX: &str = "3059301306072a8648ce3d020106082a8648ce3d030107034 /// `clockSkew.toleranceSeconds` in `trust-model.json`. const SKEW_TOLERANCE_SECONDS: i64 = 300; +/// Fixture reads use real descriptors, while the offline invariant below +/// snapshots the process descriptor table. A write guard around that snapshot +/// keeps parallel test fixture I/O from masquerading as network activity. +static FIXTURE_ACCESS: RwLock<()> = RwLock::new(()); + // --------------------------------------------------------------------------- // Fixture access // --------------------------------------------------------------------------- @@ -73,6 +79,7 @@ fn sha256_hex(bytes: &[u8]) -> String { /// redefining what conformance means, which is the failure mode a /// fixture-driven suite is otherwise blind to. fn read_fixture(name: &str) -> Vec { + let _fixture_guard = FIXTURE_ACCESS.read().expect("fixture access lock"); let dir = fixture_dir(); let manifest = fs::read_to_string(dir.join("MANIFEST.sha256")).expect("read MANIFEST.sha256"); @@ -108,6 +115,10 @@ fn trust_model() -> Value { fixture_json("trust-model.json") } +fn boundary_vectors() -> Value { + fixture_json("boundary-vectors.json") +} + fn vector_list(fixture: &Value) -> Vec { fixture["vectors"].as_array().expect("fixture carries a vector list").clone() } @@ -142,6 +153,75 @@ fn signed_document(document: &Value) -> Value { serde_json::from_slice(&signed_octets(document)).expect("signed document parses") } +fn encoded_document(document: &Value) -> String { + BASE64_STANDARD.encode_to_string(serde_json::to_vec(document).expect("document serialises")) +} + +fn apply_object_mutation(target: &mut Value, mutation: &Value) { + let object = target.as_object_mut().expect("mutation target is an object"); + let name = field(mutation, "field"); + + match mutation["operation"].as_str().expect("mutation carries an operation") { + "remove" => { + object.remove(name); + } + "replace" => { + object.insert(name.to_string(), mutation["value"].clone()); + } + operation => panic!("unsupported object mutation {operation}"), + } +} + +fn boundary_artifact(source: &Value, mutation: &Value) -> Vec { + if field(mutation, "scope") == "serializedEnvelope" { + return field(mutation, "value").as_bytes().to_vec(); + } + + let mut result = source["document"].clone(); + let scope = field(mutation, "scope"); + if scope == "envelopeBytes" { + result["bytes"] = mutation["value"].clone(); + return envelope(&result); + } + if scope == "envelopeSignature" { + apply_object_mutation(&mut result["signature"], mutation); + return envelope(&result); + } + + let mut challenge = signed_document(&result); + match scope { + "challenge" => apply_object_mutation(&mut challenge, mutation), + "challengeChain" => { + let chain = challenge["trustChain"].as_array().expect("challenge carries a chain"); + challenge["trustChain"] = match mutation["operation"].as_str().expect("chain mutation carries an operation") { + "keepFirst" => Value::Array(vec![chain[0].clone()]), + "objectWithFirst" => { + let mut object = serde_json::Map::new(); + object.insert("first".to_string(), chain[0].clone()); + Value::Object(object) + } + operation => panic!("unsupported chain mutation {operation}"), + }; + } + "trustLink" | "trustLinkSignature" => { + let index = mutation["index"].as_u64().expect("trust-link mutation carries an index") as usize; + let chain = challenge["trustChain"].as_array_mut().expect("challenge carries a chain"); + let link_envelope = &mut chain[index]; + if scope == "trustLinkSignature" { + apply_object_mutation(&mut link_envelope["signature"], mutation); + } else { + let mut link = signed_document(link_envelope); + apply_object_mutation(&mut link, mutation); + link_envelope["bytes"] = Value::String(encoded_document(&link)); + } + } + other => panic!("unsupported boundary scope {other}"), + } + + result["bytes"] = Value::String(encoded_document(&challenge)); + envelope(&result) +} + fn unix(rfc3339: &str) -> i64 { chrono::DateTime::parse_from_rfc3339(rfc3339) .unwrap_or_else(|error| panic!("'{rfc3339}' is not RFC 3339: {error}")) @@ -277,6 +357,41 @@ fn every_challenge_accept_vector_verifies_and_exposes_the_signed_fields() { ); } +#[test] +fn malformed_top_level_signature_members_keep_the_frozen_reason() { + let vector = accept_vector_named("challenge signed by a chained signing key under the pinned root"); + let now = unix(field(&vector, "evaluationTime")); + + for (member, malformed) in [ + ("algorithm", serde_json::json!(1)), + ("keyId", serde_json::Value::Null), + ("value", serde_json::json!([])), + ] { + let mut document = vector["document"].clone(); + document["signature"][member] = malformed; + + let error = OfflineEnrollment::verify_challenge(&envelope(&document), now) + .expect_err(&format!("a malformed top-level signature {member} must be refused")); + assert_eq!( + error.reason(), + "SIGNATURE_MALFORMED", + "a malformed top-level signature {member} keeps the frozen classifier" + ); + } +} + +#[test] +fn duplicate_top_level_signature_members_are_refused() { + let vector = accept_vector_named("challenge signed by a chained signing key under the pinned root"); + let document = String::from_utf8(envelope(&vector["document"])).expect("envelope is UTF-8 JSON"); + let duplicate = document.replacen("\"algorithm\":\"ES256\"", "\"algorithm\":\"ES384\",\"algorithm\":\"ES256\"", 1); + assert_ne!(duplicate, document, "the accepted vector carries the expected algorithm"); + + let error = OfflineEnrollment::verify_challenge(duplicate.as_bytes(), unix(field(&vector, "evaluationTime"))) + .expect_err("a duplicate top-level signature member must be refused"); + assert_eq!(error.reason(), "DOCUMENT_MALFORMED"); +} + #[cfg(feature = "offline-enrollment-e2e-root")] #[test] fn e2e_root_is_fixed_and_disjoint_from_the_hosted_root() { @@ -472,6 +587,124 @@ fn every_challenge_reject_vector_fails_with_its_frozen_reason() { ); } +#[test] +fn every_challenge_boundary_mutation_fails_with_its_frozen_reason() { + let boundaries = boundary_vectors(); + let source = accept_vector_named(field(&boundaries, "sourceVector")); + let now = unix(field(&source, "evaluationTime")); + let mut covered = 0usize; + + for group in ["preparseMutations", "verificationMutations"] { + for mutation in boundaries[group].as_array().expect("boundary group is a list") { + let name = field(mutation, "name"); + let expected = field(mutation, "expectedReason"); + let error = match OfflineEnrollment::verify_challenge(&boundary_artifact(&source, mutation), now) { + Err(error) => error, + Ok(_) => panic!("boundary mutation '{name}' must fail"), + }; + assert_eq!(error.reason(), expected, "boundary mutation '{name}'"); + covered += 1; + } + } + + assert_eq!(covered, 16, "boundary-vectors.json publishes sixteen executable challenge mutations"); +} + +#[test] +fn malformed_top_level_signature_precedes_malformed_first_link_routing() { + let source = accept_vector_named("challenge signed by a chained signing key under the pinned root"); + let now = unix(field(&source, "evaluationTime")); + let mut challenge_envelope = source["document"].clone(); + challenge_envelope["signature"]["algorithm"] = Value::String("ES384".to_string()); + + let mut challenge = signed_document(&challenge_envelope); + let first_envelope = &mut challenge["trustChain"].as_array_mut().expect("challenge carries a chain")[0]; + let mut first_link = signed_document(first_envelope); + first_link["issuerKeyId"] = Value::String("not-a-key-id".to_string()); + first_envelope["bytes"] = Value::String(encoded_document(&first_link)); + challenge_envelope["bytes"] = Value::String(encoded_document(&challenge)); + + let error = OfflineEnrollment::verify_challenge(&envelope(&challenge_envelope), now) + .expect_err("a malformed top-level signature and first-link issuer must not verify"); + assert_eq!(error.reason(), "SIGNATURE_MALFORMED"); +} + +#[test] +fn malformed_top_level_signature_precedes_malformed_second_link_envelope() { + let source = accept_vector_named("challenge signed by a chained signing key under the pinned root"); + let now = unix(field(&source, "evaluationTime")); + let mut challenge_envelope = source["document"].clone(); + challenge_envelope["signature"]["algorithm"] = Value::String("ES384".to_string()); + + let mut challenge = signed_document(&challenge_envelope); + challenge["trustChain"].as_array_mut().expect("challenge carries a chain")[1] + .as_object_mut() + .expect("trust link envelope is an object") + .remove("signature"); + challenge_envelope["bytes"] = Value::String(encoded_document(&challenge)); + + let error = OfflineEnrollment::verify_challenge(&envelope(&challenge_envelope), now) + .expect_err("a malformed top-level signature and second-link envelope must not verify"); + assert_eq!(error.reason(), "SIGNATURE_MALFORMED"); +} + +#[test] +fn unpinned_root_precedes_a_malformed_chain_shape() { + let source = accept_vector_named("challenge signed by a chained signing key under the pinned root"); + let now = unix(field(&source, "evaluationTime")); + let mut challenge_envelope = source["document"].clone(); + let mut challenge = signed_document(&challenge_envelope); + let first_envelope = &mut challenge["trustChain"].as_array_mut().expect("challenge carries a chain")[0]; + let mut first_link = signed_document(first_envelope); + first_link.as_object_mut().expect("trust link is an object").remove("serial"); + first_link["issuerKeyId"] = Value::String("5ff37910aa4d69949e2c488f98d6072f10a3c3e73d776698963872582644f731".to_string()); + first_envelope["bytes"] = Value::String(encoded_document(&first_link)); + challenge_envelope["bytes"] = Value::String(encoded_document(&challenge)); + + let error = + OfflineEnrollment::verify_challenge(&envelope(&challenge_envelope), now).expect_err("an unpinned root must never verify"); + assert_eq!( + error.reason(), + "ENROLLMENT_ROOT_UNKNOWN", + "the pinned-root decision must precede the rest of the attacker-controlled chain shape" + ); +} + +#[test] +fn response_production_honours_the_frozen_effective_challenge_expiry() { + let key = DeviceIdentity::generate(); + let boundaries = boundary_vectors(); + let mut covered = 0usize; + + for vector in boundaries["postSignaturePolicyVectors"] + .as_array() + .expect("post-signature policy vectors are a list") + { + let name = field(vector, "name"); + let challenge = VerifiedChallenge { + challenge_id: "018f7e6d-9d6a-7d93-8f64-8b20b3384712".to_string(), + organization_name: "organizations/01HZXQ9J2XW6R7V8T9Y0Z1A2B3".to_string(), + cluster_name: "organizations/01HZXQ9J2XW6R7V8T9Y0Z1A2B3/clusters/01HZXQ9J2XW6R7V8T9Y0Z1A2B4".to_string(), + nonce: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + issued_at: field(vector, "issuedAt").to_string(), + expires_at: field(vector, "declaredExpiresAt").to_string(), + connect_key_id: "08e7295c8f9d043e22b2b80fdb1480b0bec060dacbce7de9dd2e3d583f93d7e8".to_string(), + challenge_proof: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + }; + let outcome = OfflineEnrollment::build_response(&challenge, &key, &[0x5a; 32], unix(field(vector, "evaluationTime"))); + + match vector["expectedReason"].as_str() { + Some(expected) => assert_eq!(outcome.unwrap_err().reason(), expected, "post-signature policy vector '{name}'"), + None => { + outcome.unwrap_or_else(|error| panic!("post-signature policy vector '{name}' must pass: {}", error.reason())); + } + } + covered += 1; + } + + assert_eq!(covered, 2, "boundary-vectors.json publishes two effective-expiry vectors"); +} + /// The response reject vectors are artifacts Connect refuses. This side never /// verifies a response, so the device-side statement is the stronger one: given /// the challenge each vector answers, `build_response` must not be capable of @@ -949,6 +1182,7 @@ fn enrollment_opens_no_descriptor_and_is_a_pure_byte_transform() { let document = envelope(&vector["document"]); let now = unix(field(&vector, "evaluationTime")); let key = DeviceIdentity::generate(); + let _fixture_guard = FIXTURE_ACCESS.write().expect("fixture access lock"); // Warm anything the test harness itself lazily opens before the baseline. let _ = open_descriptors(); diff --git a/scripts/README.md b/scripts/README.md index da2bd119c..25c45c863 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -46,6 +46,8 @@ their issue closes. | Entry | Status | Purpose | Wiring / docs | |---|---|---|---| +| `diagnose_scanner_enumeration_restart.py` | dev-tool | Strict fixed raw-entry-budget scanner-worker restart diagnostic | [Checkpoint fixture](../docs/testing/scanner-checkpoint-fixture.md) | +| `test_diagnose_scanner_enumeration_restart.py` | dev-tool | Driver report validation and positive convergence oracle tests | Python unittest; same guide | | `e2e-run.sh` | ci-gate | Boots a rustfs server and runs the `s3s-e2e` black-box conformance tool against it | ci.yml `e2e-tests` jobs; `docs/testing/README.md` | | `run_ecstore_validation_suite.sh` | dev-tool | ecstore black-box validation suite (`quick`/`full`/`destructive`/`fuzz` profiles) | `docs/testing/README.md`, `docs/testing/ecstore-validation-suite-design.md` | | `run_e2e_tests.sh` | dev-tool | Local `e2e_test` crate runner (starts a server, applies filters, cleans up) | `crates/e2e_test/README.md` | @@ -54,6 +56,8 @@ their issue closes. | `probe.sh` | dev-tool | Probe-style e2e run | `make probe-e2e` | | `run_scanner_validation_harness.sh` | dev-tool | Scanner validation harness | `docs/operations/scanner-benchmark-runbook.md` | | `test_scanner_validation_harness.sh` | dev-tool | Self-test for the scanner validation harness | — | +| `scanner_abba.py` | dev-tool | Scanner/heal ABBA orchestration and evidence gates via `run_scanner_validation_harness.sh --abba` | `docs/operations/scanner-benchmark-runbook.md` | +| `test_scanner_abba.py` | dev-tool | Synthetic ABBA adapter and failure-path tests | `test_scanner_validation_harness.sh` | | `test_build_rustfs_options.sh` | dev-tool | Shell test for rustfs build-option wiring | `make test` (script-tests) | | `test_entrypoint_credentials.sh` | dev-tool | Container entrypoint credential-handling test | `make test` (script-tests) | | `test_helm_chart_version.sh` | dev-tool | Test for `helm_chart_version.sh` | — | diff --git a/scripts/check_migration_gate_count.sh b/scripts/check_migration_gate_count.sh index 0e3bf0a6c..7ee471ed1 100755 --- a/scripts/check_migration_gate_count.sh +++ b/scripts/check_migration_gate_count.sh @@ -33,8 +33,11 @@ set -euo pipefail cd "$(dirname "$0")/.." -# Single source of truth for the migration-gate filter. ci.yml must invoke -# this script instead of inlining the expression. +# Single source of truth for the migration-gate target and filter. The +# test-util feature activates migration-critical tests that otherwise leave +# their shared fixtures compiled but unused. ci.yml must invoke this script +# instead of inlining either selection. +MIGRATION_GATE_TARGET_ARGS=(-p rustfs-ecstore --lib --features test-util) MIGRATION_GATE_FILTER='test(data_movement) or test(rebalance) or test(decommission) or test(source_cleanup) or test(delete_marker)' FLOOR_FILE=".config/migration-gate-floor.txt" @@ -60,7 +63,7 @@ if [[ "$mode" == "all" || "$mode" == "check" ]]; then # count to 0 and failing every PR). A nextest failure aborts via set -e # with its stderr visible; a JSON schema change makes jq fail loudly # rather than silently passing. - count="$(cargo nextest list -p rustfs-ecstore --lib -E "$MIGRATION_GATE_FILTER" --message-format json \ + count="$(cargo nextest list "${MIGRATION_GATE_TARGET_ARGS[@]}" -E "$MIGRATION_GATE_FILTER" --message-format json \ | jq '[."rust-suites"[].testcases[] | select(."filter-match".status == "matches")] | length')" if ! [[ "$count" =~ ^[0-9]+$ ]]; then echo "error: could not parse nextest JSON listing (got count: '$count')" >&2 @@ -81,5 +84,5 @@ if [[ "$mode" == "all" || "$mode" == "check" ]]; then fi if [[ "$mode" == "all" || "$mode" == "run" ]]; then - cargo nextest run -p rustfs-ecstore --lib -E "$MIGRATION_GATE_FILTER" + cargo nextest run "${MIGRATION_GATE_TARGET_ARGS[@]}" -E "$MIGRATION_GATE_FILTER" fi diff --git a/scripts/check_s3s_footprint.sh b/scripts/check_s3s_footprint.sh index ae14199b5..3bbebafb3 100755 --- a/scripts/check_s3s_footprint.sh +++ b/scripts/check_s3s_footprint.sh @@ -50,8 +50,13 @@ cd "$(dirname "$0")/.." # s3_error! stays flat at 1616. # 1616 → 1613 on 2026-09-02: dependency refresh verified the current tree has # already shed three s3_error! invocation lines; retighten the line counter. +# 1613 → 1589 on 2026-09-06: rustfs/backlog#2309 and rustfs/rustfs#7225 both +# folded the ten per-config arms of ExportBucketMetadata into one helper, which +# now reports an unreadable configuration as a plain string instead of raising +# an S3 error per arm (24 invocation lines removed from +# rustfs/src/admin/handlers/bucket_meta.rs; measured after merging the two). S3S_IMPORT_FILES_BASELINE=213 -S3_ERROR_LINES_BASELINE=1613 +S3_ERROR_LINES_BASELINE=1589 # ecstore-scoped ratchet (rustfs/backlog#1842): the storage engine must not # know S3 wire/DTO types (ARCHITECTURE.md invariant 4). The S3-*consuming* # client was extracted to crates/s3-client, where s3s usage is legitimate; diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index ef3f15107..3415167a9 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -12,11 +12,15 @@ import sys import tempfile import tomllib import unittest +import uuid +import xml.etree.ElementTree as ET from datetime import datetime, timezone from unittest import mock from pathlib import Path from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from scanner_abba import MAX_JSON_BYTES, digest, number, read_json, require, sha, write_json + ROOT = Path(__file__).resolve().parents[1] SCHEDULED_ALERT_WORKFLOWS = tuple( @@ -873,6 +877,187 @@ def check_core_listing(root: Path, listing: Path) -> list[str]: return [f"cannot read core nextest listing: {error}"] +def evidence_integer(value: object, name: str, minimum: int, maximum: int) -> int: + require(type(value) is int and minimum <= value <= maximum, f"invalid integer {name}") + return value + + +def begin_scanner_heal_receipt(root: Path, directory: Path, binary: Path, test_binary: Path) -> None: + """Record an existing build; this command never builds or runs a test.""" + require(not directory.exists(), "scanner/heal run directory must be new") + require(not subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=no"], cwd=root, text=True).strip(), + "commit tracked source changes before creating evidence") + builds = {} + for label, path in (("binary", binary), ("test_binary", test_binary)): + path = path.resolve(strict=True) + require(path.is_file() and os.access(path, os.X_OK), f"missing executable {label}") + builds[label] = {"path": str(path), "sha256": digest(path)} + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + require(re.fullmatch(r"[0-9a-f]{40}", revision), "invalid source revision") + version = subprocess.check_output([builds["binary"]["path"], "--version"], text=True, timeout=30) + embedded_revision = re.search(r"^git commit\s*:\s*([0-9a-f]{40})\s*$", version, re.MULTILINE) + embedded_status = re.search(r"^git status\s*:\s*(.*)\Z", version, re.MULTILINE | re.DOTALL) + require(embedded_revision is not None and embedded_revision[1] == revision, "server binary source revision mismatch") + require(embedded_status is not None and not embedded_status[1].strip(), "server binary was built from dirty/unknown source") + lock_blob = subprocess.check_output(["git", "hash-object", "Cargo.lock"], cwd=root, text=True).strip() + require(re.fullmatch(r"[0-9a-f]{40}", lock_blob), "invalid Cargo.lock identity") + features = os.environ.get("RUSTFS_E2E_EXPECTED_FEATURES") + require(features is not None, "set RUSTFS_E2E_EXPECTED_FEATURES to the compiled e2e crate feature set") + features = ",".join(sorted(set(filter(None, (feature.strip() for feature in features.split(",")))))) + require(all(re.fullmatch(r"[a-z0-9-]+", feature) for feature in features.split(",") if feature), "invalid expected features") + directory.mkdir(parents=True) + write_json(directory / "run.json", {"schema": 1, "run_id": uuid.uuid4().hex, + "source_revision": revision, + "binary_source_revision": embedded_revision[1], + "test_build": {"source_revision": revision, "dirty": False, + "lock_blob": lock_blob, "features": features}, + "started_at": datetime.now(timezone.utc).timestamp(), **builds}) + + +def finish_scanner_heal_receipt(directory: Path, exit_code: int) -> None: + require(type(exit_code) is int and 0 <= exit_code <= 255, "invalid test exit code") + require(not (directory / "execution.json").exists(), "execution receipt already exists") + run = read_json(directory / "run.json") + artifacts = {} + for name in ("listing.json", "junit.xml", "background-target-restart.json"): + path = directory / name + if exit_code != 0 and not path.exists(): + continue + require(path.is_file() and 0 < path.stat().st_size <= MAX_JSON_BYTES, f"missing/oversized {name}") + require(path.stat().st_mtime >= run["started_at"], f"stale {name}") + artifacts[name] = digest(path) + write_json(directory / "execution.json", {"run_id": run["run_id"], "exit_code": exit_code, + "finished_at": datetime.now(timezone.utc).timestamp(), + "artifacts": artifacts}) + + +def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> list[str]: + """Validate one actual case, or fail the release while required lanes are pending.""" + try: + registry = read_json(root / ".config/scanner-heal-required-tests.json") + evidence_integer(registry.get("schema"), "registry schema", 1, 1) + require(registry.get("cases"), "invalid scanner/heal registry") + selected = registry["cases"] if case_id == "release" else {case_id: registry["cases"][case_id]} + run = read_json(directory / "run.json") + execution = read_json(directory / "execution.json") + evidence_integer(run.get("schema"), "run schema", 1, 1) + require(re.fullmatch(r"[0-9a-f]{32}", run["run_id"]), "invalid run identity") + require(re.fullmatch(r"[0-9a-f]{40}", run["source_revision"]), "invalid source revision") + require(run.get("binary_source_revision") == run["source_revision"], "server source provenance missing") + expected_build = run["test_build"] + require(expected_build["source_revision"] == run["source_revision"] and expected_build["dirty"] is False, + "test source provenance missing") + require(re.fullmatch(r"[0-9a-f]{40}", expected_build["lock_blob"]), "invalid test lockfile identity") + require(isinstance(expected_build["features"], str), "missing test features") + require(execution.get("run_id") == run["run_id"], "execution belongs to another run") + require(type(execution.get("exit_code")) is int and execution["exit_code"] == 0, "test command failed or did not run") + number(run["started_at"], "started_at", 1) + number(execution["finished_at"], "finished_at", run["started_at"]) + for label in ("binary", "test_binary"): + require(sha(run[label]["sha256"]) and digest(Path(run[label]["path"])) == run[label]["sha256"], + f"{label} changed or missing") + for name in ("listing.json", "junit.xml"): + path = directory / name + require(0 < path.stat().st_size <= MAX_JSON_BYTES, f"missing/oversized {name}") + require(run["started_at"] <= path.stat().st_mtime <= execution["finished_at"], f"{name} outside run window") + require(digest(path) == execution["artifacts"][name], f"{name} hash mismatch") + suites = read_json(directory / "listing.json")["rust-suites"] + xml = (directory / "junit.xml").read_bytes() + require(b" 0, "missing expected bytes") + require(type(obj["actual_bytes"]) is int and obj["actual_bytes"] == obj["expected_bytes"], "S3 body length mismatch") + require(sha(obj["expected_sha256"]) and obj["actual_sha256"] == obj["expected_sha256"], "S3 body digest mismatch") + physical = obj["physical"] + if obj["expected_physical"] is not None: + require(physical == obj["expected_physical"], "target shard differs from pre-fault manifest") + for geometry in [physical] + ([obj["expected_physical"]] if obj["expected_physical"] is not None else []): + data = evidence_integer(geometry["data_blocks"], "EC data blocks", 1, 16) + parity = evidence_integer(geometry["parity_blocks"], "EC parity blocks", 1, 16) + require(data + parity == oracle["topology"]["nodes"] * oracle["topology"]["drives_per_node"], + "EC geometry differs from this case's single set") + evidence_integer(geometry["erasure_index"], "target erasure index", 1, data + parity) + require(physical["has_xl_meta"] is True and physical["version_id"] is None, "missing target metadata") + parts = physical["expected_part_numbers"] + require(isinstance(parts, list) and 0 < len(parts) <= 10000, "no physical part coverage") + require(all(type(part) is int and part > 0 for part in parts) and len(set(parts)) == len(parts), + "invalid physical part identity") + require({str(part) for part in parts} == set(physical["present_part_fingerprints"]), "target shard parts missing") + for part in physical["present_part_fingerprints"].values(): + require(type(part["size"]) is int and part["size"] > 0 and sha(part["sha256"]), "invalid target shard fingerprint") + node_listings = oracle["node_listings"] + require(isinstance(node_listings, list) and len(node_listings) == requirement["topology"]["nodes"], + "missing per-node S3 listing") + require(all(keys == sorted(obj["key"] for obj in objects) for keys in node_listings), + "S3 listing differs from object oracle") + if case_id == "release": + errors.extend(f"pending {gate}: {reason}" for gate, reason in registry["release_pending"].items()) + return errors + except (OSError, KeyError, TypeError, ValueError, ET.ParseError) as error: + return [f"scanner/heal evidence rejected: {error}"] + + def validate(root: Path) -> list[str]: errors: list[str] = [] errors.extend(check_core_fixtures(root)) @@ -1022,6 +1207,213 @@ class SelfTests(unittest.TestCase): with mock.patch(__name__ + ".check_quick_checks", return_value=[error]): self.assertIn(error, validate(ROOT)) + def scanner_heal_fixture(self, directory: Path) -> tuple[Path, Path]: + """Parser fixtures only; these files are never runtime evidence.""" + root, run_dir = directory / "repo", directory / "run" + (root / ".config").mkdir(parents=True) + run_dir.mkdir() + registry = read_json(ROOT / ".config/scanner-heal-required-tests.json") + write_json(root / ".config/scanner-heal-required-tests.json", registry) + requirement = registry["cases"]["background-target-restart"] + binary = directory / "fake-binary" + binary.write_bytes(b"parser fixture, not a real build") + binary.chmod(0o700) + build = {"path": str(binary), "sha256": digest(binary)} + write_json(run_dir / "run.json", {"schema": 1, "run_id": "a" * 32, "source_revision": "b" * 40, + "binary_source_revision": "b" * 40, + "test_build": {"source_revision": "b" * 40, "dirty": False, + "lock_blob": "c" * 40, "features": "default"}, + "started_at": datetime.now(timezone.utc).timestamp() - 1, + "binary": build, "test_binary": build}) + write_json(run_dir / "listing.json", {"rust-suites": {requirement["suite"]: { + "binary-id": requirement["suite"], "binary-path": str(binary), "package-name": "e2e_test", "build-platform": "target", + "testcases": { + requirement["name"]: {"ignored": False, "filter-match": {"status": "matches"}} + }}}}) + (run_dir / "junit.xml").write_text( + f'') + physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation", + "erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1], + "present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}}, + "inline_data_fingerprint": None} + obj = {"key": "object", "version_id": None, "expected_bytes": 16, "actual_bytes": 16, + "expected_sha256": "d" * 64, "actual_sha256": "d" * 64, + "expected_physical": physical, "physical": physical} + objects = [dict(obj, key=f"object-{index}") for index in range(9)] + objects[-1] = dict(objects[-1], expected_physical=None) + write_json(run_dir / "background-target-restart.json", { + "schema": 1, "evidence": "process-restart", "case": "background-target-restart", + "run_id": "a" * 32, "source_revision": "b" * 40, + "test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40, + "features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""}, + "binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"], + "topology": {"nodes": 4, "drives_per_node": 1}, "pid_before": 10, "pid_after": 11, + "objects": objects, "node_listings": [[item["key"] for item in objects]] * 4, + }) + finish_scanner_heal_receipt(run_dir, 0) + return root, run_dir + + def test_scanner_heal_case_does_not_approve_pending_release(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + self.assertEqual(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), []) + errors = check_scanner_heal_evidence(root, run_dir, "release") + self.assertEqual(len(errors), 21) + self.assertTrue(any(error.startswith("pending R-E:") for error in errors)) + self.assertTrue(any(error.startswith("pending R-D:") for error in errors)) + self.assertTrue(any(error.startswith("pending R-L:") for error in errors)) + + def test_scanner_heal_rejects_broken_execution_and_artifacts(self) -> None: + for fault in ("exit", "missing", "zero", "skipped", "failed", "retry", "filtered", "ignored", "stale", + "hash", "binary", "synthetic", "wrong-run", "same-pid", "body", "parts", "listing", "topology"): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + path = run_dir / "background-target-restart.json" + oracle = read_json(path) + if fault == "exit": + receipt = read_json(run_dir / "execution.json") + receipt["exit_code"] = 42 + write_json(run_dir / "execution.json", receipt) + elif fault == "missing": + path.unlink() + elif fault == "zero": + (run_dir / "junit.xml").write_text("") + elif fault in ("skipped", "failed", "retry"): + junit = run_dir / "junit.xml" + tag = {"skipped": "skipped", "failed": "failure", "retry": "rerunFailure"}[fault] + junit.write_text(junit.read_text().replace("/>", f"><{tag}/>")) + elif fault in ("filtered", "ignored"): + listing = read_json(run_dir / "listing.json") + case = next(iter(listing["rust-suites"]["e2e_test"]["testcases"].values())) + case["ignored"] = fault == "ignored" + case["filter-match"]["status"] = "mismatch" if fault == "filtered" else "matches" + write_json(run_dir / "listing.json", listing) + elif fault == "stale": + os.utime(path, (1, 1)) + elif fault == "hash": + path.write_text(path.read_text() + " ") + elif fault == "binary": + Path(read_json(run_dir / "run.json")["binary"]["path"]).write_bytes(b"another build") + else: + if fault == "synthetic": + oracle["evidence"] = "synthetic" + elif fault == "wrong-run": + oracle["run_id"] = "f" * 32 + elif fault == "same-pid": + oracle["pid_after"] = oracle["pid_before"] + elif fault == "body": + oracle["objects"][0]["actual_sha256"] = "e" * 64 + elif fault == "parts": + oracle["objects"][0]["physical"]["present_part_fingerprints"] = {} + elif fault == "listing": + oracle["node_listings"][0] = [] + elif fault == "topology": + oracle["topology"] = {"nodes": 3, "drives_per_node": 4} + write_json(path, oracle) + if fault not in ("exit", "missing", "stale", "hash", "binary"): + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0) + self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), fault) + + def test_scanner_heal_receipts_reject_reuse_and_missing_builds(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + with self.assertRaisesRegex(ValueError, "already exists"): + finish_scanner_heal_receipt(run_dir, 0) + with self.assertRaisesRegex(ValueError, "must be new"): + begin_scanner_heal_receipt(root, run_dir, Path("missing"), Path("missing")) + with mock.patch("subprocess.check_output", side_effect=["", "b" * 40]): + with self.assertRaises(FileNotFoundError): + begin_scanner_heal_receipt(root, Path(tmp) / "new-run", Path(tmp) / "missing", Path(tmp) / "missing") + + def test_scanner_heal_begin_requires_embedded_source_provenance(self) -> None: + for kind in ("current", "stale", "dirty", "unknown"): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp: + root, _ = self.scanner_heal_fixture(Path(tmp)) + sources = root / "crates/e2e_test/src" + sources.mkdir(parents=True) + (sources / "heal_erasure_disk_rebuild_test.rs").write_bytes(b"oracle source") + (sources / "chaos.rs").write_bytes(b"census source") + revision = "c" * 40 if kind == "stale" else "b" * 40 + version = f"rustfs\ngit commit : {revision}\ngit status :\n" + if kind == "dirty": + version += "modified source\n" + if kind == "unknown": + version = "rustfs without build provenance" + with mock.patch("subprocess.check_output", side_effect=["", "b" * 40, version, "c" * 40]), \ + mock.patch.dict(os.environ, {"RUSTFS_E2E_EXPECTED_FEATURES": "default"}): + directory = Path(tmp) / "fresh" + binary = Path(tmp) / "fake-binary" + if kind == "current": + begin_scanner_heal_receipt(root, directory, binary, binary) + self.assertEqual(read_json(directory / "run.json")["binary_source_revision"], "b" * 40) + else: + with self.assertRaisesRegex(ValueError, "server binary"): + begin_scanner_heal_receipt(root, directory, binary, binary) + self.assertFalse(directory.exists()) + + def test_scanner_heal_rejects_copied_junit_and_wrong_suite_build(self) -> None: + for fault in ("old-junit", "missing-time", "wrong-binary", "common-source", "lockfile", "features", "dirty-build"): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + if fault in ("old-junit", "missing-time"): + path = run_dir / "junit.xml" + xml = ET.fromstring(path.read_bytes()) + testcase = next(xml.iter("testcase")) + if fault == "old-junit": + testcase.set("timestamp", "2000-01-01T00:00:00.000Z") + else: + del testcase.attrib["timestamp"] + # Rewriting/copying gives an old execution a fresh mtime. + path.write_bytes(ET.tostring(xml)) + elif fault == "wrong-binary": + path = run_dir / "listing.json" + listing = read_json(path) + another = Path(tmp) / "another-binary" + another.write_bytes(Path(tmp, "fake-binary").read_bytes()) + listing["rust-suites"]["e2e_test"]["binary-path"] = str(another) + write_json(path, listing) + else: + path = run_dir / "background-target-restart.json" + oracle = read_json(path) + key, value = {"common-source": ("source_revision", "f" * 40), "lockfile": ("lock_blob", "f" * 40), + "features": ("features", "default,sftp"), "dirty-build": ("dirty", True)}[fault] + oracle["test_build"][key] = value + write_json(path, oracle) + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0) + self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart"), fault) + + def test_scanner_heal_rejects_boolean_fractional_and_out_of_geometry_integers(self) -> None: + valid = {"schema": 1, "nodes": 4, "drives_per_node": 1, "pid_before": 10, "pid_after": 11, + "erasure_index": 1, "data_blocks": 2, "parity_blocks": 2} + cases = [(field, value) for field, correct in valid.items() for value in (True, float(correct))] + cases += [("erasure_index", 5), ("erasure_index", 0), ("pid_after", -1)] + for field, value in cases: + with self.subTest(field=field, value=value), tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + path = run_dir / "background-target-restart.json" + oracle = read_json(path) + if field in ("nodes", "drives_per_node"): + oracle["topology"][field] = value + elif field in ("erasure_index", "data_blocks", "parity_blocks"): + oracle["objects"][-1]["physical"][field] = value + else: + oracle[field] = value + write_json(path, oracle) + (run_dir / "execution.json").unlink() + finish_scanner_heal_receipt(run_dir, 0) + self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart")) + for filename in ("run.json", ".config/scanner-heal-required-tests.json"): + with self.subTest(filename=filename), tempfile.TemporaryDirectory() as tmp: + root, run_dir = self.scanner_heal_fixture(Path(tmp)) + path = (root if filename.startswith(".config") else run_dir) / filename + content = read_json(path) + content["schema"] = True + write_json(path, content) + self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-restart")) + def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -1664,6 +2056,25 @@ def main() -> int: if sys.argv[1:] == ["--self-test"]: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) return 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1 + if sys.argv[1:2] in (["--begin-scanner-heal"], ["--finish-scanner-heal"], ["--check-scanner-heal"]): + try: + if len(sys.argv) == 5 and sys.argv[1] == "--begin-scanner-heal": + begin_scanner_heal_receipt(ROOT, Path(sys.argv[2]), Path(sys.argv[3]), Path(sys.argv[4])) + return 0 + if len(sys.argv) == 4 and sys.argv[1] == "--finish-scanner-heal": + finish_scanner_heal_receipt(Path(sys.argv[2]), int(sys.argv[3])) + return 0 + if len(sys.argv) == 4 and sys.argv[1] == "--check-scanner-heal": + errors = check_scanner_heal_evidence(ROOT, Path(sys.argv[2]), sys.argv[3]) + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + if not errors: + print(f"Case evidence verified: {sys.argv[3]}; this does not approve release") + return 1 if errors else 0 + raise ValueError("expected --begin-scanner-heal DIR BINARY TEST_BINARY, --finish-scanner-heal DIR EXIT, or --check-scanner-heal DIR CASE|release") + except (OSError, KeyError, TypeError, ValueError, subprocess.SubprocessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if len(sys.argv) == 3 and sys.argv[1] == "--check-core": errors = check_core_listing(ROOT, Path(sys.argv[2])) for error in errors: diff --git a/scripts/diagnose_scanner_enumeration_restart.py b/scripts/diagnose_scanner_enumeration_restart.py new file mode 100644 index 000000000..951c164a7 --- /dev/null +++ b/scripts/diagnose_scanner_enumeration_restart.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Strict restart diagnostic using the real scanner libtest worker, not a walker model.""" + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys + +WORKER = "scanner_folder::tests::enumeration_restart::enumeration_restart_worker" +MAX_REPORT_BYTES = 16384 + + +def bounded_int(low, high): + def parse(value): + number = int(value) + if not low <= number <= high: + raise argparse.ArgumentTypeError(f"must be between {low} and {high}") + return number + return parse + + +def validate_report(report, *, round_number, pid, objects, budget): + if not isinstance(report, dict): + raise ValueError("worker report must be an object") + expected = {"schema": 1, "round": round_number, "pid": pid, + "objects_expected": objects, "raw_entry_budget": budget} + for key, value in expected.items(): + if type(report.get(key)) is not int or report[key] != value: + raise ValueError(f"worker report mismatch: {key}") + for key in ("raw_entries", "raw_name_bytes", "objects_before", "objects_retained", + "versions_retained", "bytes_retained", "objects_processed"): + if type(report.get(key)) is not int or not 0 <= report[key] <= 1048576: + raise ValueError(f"invalid bounded counter: {key}") + if report["raw_entries"] == 0: + raise ValueError("nonempty fixture must observe raw entries; budget hook may not have run") + if report["raw_entries"] > budget: + raise ValueError("raw-entry budget exceeded; no unbudgeted tail is permitted") + if type(report.get("snapshot_complete")) is not bool: + raise ValueError("missing explicit completeness") + if report.get("outcome") not in ("complete", "partial", "cancelled_without_cache"): + raise ValueError("unexpected scanner outcome") + + +def converged(report, objects): + return (report["snapshot_complete"] and report["outcome"] == "complete" + and all(report[key] == objects for key in + ("objects_retained", "versions_retained", "bytes_retained"))) + + +def run(args): + binary = args.test_binary.resolve(strict=True) + listed = subprocess.run([str(binary), WORKER, "--exact", "--list"], + check=True, capture_output=True, text=True, timeout=30) + if f"{WORKER}: test" not in listed.stdout.splitlines(): + raise ValueError("binary does not contain the exact scanner worker test") + workspace = args.output.resolve() + workspace.mkdir() # Refuse reuse/overwrite of previous evidence or customer data. + reports = [] + for round_number in range(args.rounds): + request = {"workspace": str(workspace), "objects": args.objects, + "raw_entry_budget": args.raw_entry_budget, "round": round_number} + request_path = workspace / "request.json" + request_path.write_text(json.dumps(request), encoding="utf-8") + env = dict(os.environ, RUSTFS_ENUMERATION_REQUEST=str(request_path), + RUST_MIN_STACK="4194304", NO_PROXY="localhost,127.0.0.1,::1", + no_proxy="localhost,127.0.0.1,::1") + with subprocess.Popen([str(binary), WORKER, "--exact", "--test-threads=1"], + env=env, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) as worker: + try: + status = worker.wait(timeout=args.timeout) + except subprocess.TimeoutExpired: + worker.kill() + worker.wait() + raise ValueError(f"worker round {round_number} timed out") from None + if status: + raise ValueError(f"real scanner worker round {round_number} exited {status}") + report_path = workspace / f"round-{round_number}.json" + with report_path.open("rb") as handle: + raw = handle.read(MAX_REPORT_BYTES + 1) + if len(raw) > MAX_REPORT_BYTES: + raise ValueError("oversized worker report") + report = json.loads(raw) + validate_report(report, round_number=round_number, pid=worker.pid, + objects=args.objects, budget=args.raw_entry_budget) + if reports and report["objects_before"] != reports[-1]["objects_retained"]: + raise ValueError("cache coverage did not survive the process boundary") + reports.append(report) + print(json.dumps(report, sort_keys=True), flush=True) + if converged(report, args.objects): + print("PASS: bounded scanner-worker restart convergence for this fixture only") + return 0 + print("FAIL: fixed-budget restart convergence not established; R-E gate remains unmet", + file=sys.stderr) + return 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--test-binary", type=Path, required=True, + help="compiled rustfs-scanner libtest executable") + parser.add_argument("--output", type=Path, required=True, help="new evidence directory (must not exist)") + parser.add_argument("--objects", type=bounded_int(1, 1024), default=128) + parser.add_argument("--raw-entry-budget", type=bounded_int(1, 4096), default=8) + parser.add_argument("--rounds", type=bounded_int(1, 64), default=8) + parser.add_argument("--timeout", type=bounded_int(1, 120), default=60, + help="per-worker watchdog seconds, not the scan work budget") + args = parser.parse_args() + try: + return run(args) + except (OSError, ValueError, subprocess.SubprocessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_scanner_validation_harness.sh b/scripts/run_scanner_validation_harness.sh index 12c962203..5822cd757 100755 --- a/scripts/run_scanner_validation_harness.sh +++ b/scripts/run_scanner_validation_harness.sh @@ -1,6 +1,12 @@ #!/usr/bin/env bash set -euo pipefail +if [[ "${1:-}" == "--abba" ]]; then + shift + SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + exec "$SCRIPT_DIR/python_bin.sh" "$SCRIPT_DIR/scanner_abba.py" "$@" +fi + ALIAS="" ENDPOINT="" ACCESS_KEY="${RUSTFS_ACCESS_KEY:-}" @@ -23,6 +29,7 @@ TELEMETRY_PIDS=() usage() { cat <<'USAGE' Usage: + scripts/run_scanner_validation_harness.sh --abba --help scripts/run_scanner_validation_harness.sh --alias \ --endpoint [options] diff --git a/scripts/scanner_abba.py b/scripts/scanner_abba.py new file mode 100644 index 000000000..e728dc965 --- /dev/null +++ b/scripts/scanner_abba.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +"""Run isolated scanner/heal ABBA cells through a deployment-specific adapter.""" + +import argparse +from decimal import Decimal +import hashlib +import json +import math +import os +from pathlib import Path +import select +import shutil +import signal +import subprocess +import sys +import time + +SCENARIOS = ("cold-hot", "fresh-hot", "multi-hot-new", "running-heal", "mrf-replay") +LEGS = ("A1", "B1", "B2", "A2") +MAX_JSON_BYTES = 1024 * 1024 +METRICS = ( + "p99_ms", "throughput_ops", "rss_bytes", "cpu_seconds", "iops", "rpc_count", + "cache_clone_bytes", "encode_bytes", "save_bytes", "oldest_age_seconds", + "walk_objects", "cold_walk_objects", "healed_objects", "errors", "requests", +) +REPEATABILITY_LIMIT = Decimal("0.05") +P2_WORK_MULTIPLE_LIMIT = Decimal("1.2") + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def number(value, name, minimum=0): + require(type(value) in (float, int) and math.isfinite(value) and value >= minimum, + f"invalid {name}") + return value + + +def decimal_number(value, name, minimum=0): + if isinstance(value, Decimal): + require(value.is_finite() and value >= Decimal(str(minimum)), f"invalid {name}") + return value + number(value, name, minimum) + return Decimal(str(value)) + + +def ratio(numerator, denominator, name): + denominator = decimal_number(denominator, f"{name} denominator") + require(denominator > 0, f"invalid {name} denominator") + return decimal_number(numerator, name) / denominator + + +def relative_change(current, baseline, name): + return ratio(current, baseline, name) - Decimal("1") + + +def repeatability_change(first, second, name): + first = decimal_number(first, name) + second = decimal_number(second, name) + if first == 0 and second == 0: + return Decimal("0") + if first == 0 or second == 0: + return Decimal("Infinity") + return abs(second / first - Decimal("1")) + + +def report_number(value): + return None if value.is_infinite() else float(value) + + +def digest(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def read_json(path): + require(path.stat().st_size <= MAX_JSON_BYTES, f"oversized JSON: {path.name}") + with path.open() as stream: + value = json.load(stream) + require(isinstance(value, dict), f"expected JSON object: {path.name}") + return value + + +def write_json(path, value): + data = json.dumps(value, indent=2, allow_nan=False) + "\n" + require(len(data.encode()) <= MAX_JSON_BYTES, "oversized result") + path.write_text(data) + + +def sha(value): + return isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value) + + +def validate_manifest(manifest): + require(manifest.get("schema") == 1, "unsupported manifest schema") + require(manifest.get("evidence") in ("synthetic", "measured"), "missing evidence type") + fixed = manifest["fixed"] + for key in ("config_sha256", "dataset_sha256"): + require(sha(fixed.get(key)), f"invalid fixed.{key}") + for key in ("release_flags", "durability", "disk_type", "cache_state", "load_command", "resource_isolation"): + require(isinstance(fixed.get(key), str) and fixed[key].strip(), f"missing fixed.{key}") + require(fixed.get("topology") == "EC8+4", "formal matrix requires EC8+4") + number(fixed.get("offered_load_ops"), "offered load", 1) + require(type(manifest.get("rounds")) is int and 3 <= manifest["rounds"] <= 10, + "rounds must be 3..10") + minimum = 900 if manifest["evidence"] == "measured" else 1 + require(type(manifest.get("duration_seconds")) is int and + minimum <= manifest["duration_seconds"] <= 86400, "invalid duration_seconds") + number(manifest.get("min_free_bytes"), "min_free_bytes", 1) + for phase in ("baseline", "candidate"): + build = manifest[phase] + path = Path(build["binary"]).resolve(strict=True) + require(path.is_file() and os.access(path, os.X_OK), f"missing executable {phase} build") + require(sha(build.get("sha256")) and digest(path) == build["sha256"], f"{phase} binary hash mismatch") + require(isinstance(build.get("revision"), str) and len(build["revision"]) == 40 and + all(c in "0123456789abcdef" for c in build["revision"]), f"invalid {phase} revision") + build["binary"] = str(path) + for scenario in SCENARIOS: + expected = manifest["oracles"][scenario] + for key in ("objects", "versions", "bytes"): + require(type(expected.get(key)) is int and expected[key] > 0, f"missing {scenario} oracle {key}") + require(sha(expected.get("sha256")), f"missing {scenario} content/version digest") + number(manifest["expected_healed_objects"].get(scenario), f"{scenario} expected repairs") + if scenario in ("running-heal", "mrf-replay"): + require(manifest["expected_healed_objects"][scenario] > 0, f"{scenario} requires repairs") + + +class OwnedCommand: + """Keep the session leader unreaped until its group's last signal is sent.""" + + def __init__(self, args, log): + require(sys.platform == "darwin" or hasattr(os, "waitid"), "non-reaping child observation is unavailable") + self.args, self.status = args, None + self.queue = select.kqueue() if sys.platform == "darwin" else None + self.process = None + read_gate, write_gate = os.pipe() + try: + # The shell has already exec'd when Popen returns. Gate the target + # until kqueue is registered; preexec_fn would deadlock Popen here. + gate = f'read -r _scanner_gate <&{read_gate} || exit 125; exec {read_gate}<&-; exec "$@"' + self.process = subprocess.Popen(["bash", "-c", gate, "scanner-abba", *args], pass_fds=(read_gate,), + stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + if self.queue is not None: + # Darwin NOTE_EXITSTATUS is not exposed by Python's select constants. + event = select.kevent(self.process.pid, filter=select.KQ_FILTER_PROC, + flags=select.KQ_EV_ADD | select.KQ_EV_ONESHOT, + fflags=select.KQ_NOTE_EXIT | 0x04000000) + self.queue.control([event], 0, 0) + os.write(write_gate, b"\n") + except BaseException: + try: + if self.process is not None: + try: + self._signal_group(signal.SIGKILL) + finally: + self.process.wait(timeout=10) + finally: + if self.queue is not None: + self.queue.close() + raise + finally: + os.close(read_gate) + os.close(write_gate) + + def wait(self, timeout): + if self.status is not None: + return self.status + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired(self.args, timeout) + if self.queue is not None: + events = self.queue.control(None, 1, remaining) + if events: + self.status = os.waitstatus_to_exitcode(events[0].data) + return self.status + else: + result = os.waitid(os.P_PID, self.process.pid, os.WEXITED | os.WNOWAIT | os.WNOHANG) + if result is not None: + self.status = result.si_status if result.si_code == os.CLD_EXITED else -result.si_status + return self.status + time.sleep(min(0.05, remaining)) + + def _signal_group(self, sig): + try: + os.killpg(self.process.pid, sig) + return True + except ProcessLookupError: + return False + + def finish(self, terminate=False): + if self.process.returncode is not None: + return self.process.returncode + try: + if terminate: + try: + self._signal_group(signal.SIGTERM) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline and self._signal_group(0): + time.sleep(0.05) + finally: + # Keep the PID reserved through the last group signal, even + # when the cleanup grace period itself is interrupted. + self._signal_group(signal.SIGKILL) + finally: + try: + returncode = self.process.wait(timeout=10) + finally: + if self.queue is not None: + self.queue.close() + return returncode + + +def invoke(adapter, action, request, timeout): + """The adapter writes bounded JSON separately; stderr/stdout remain raw evidence.""" + output = request.parent / f"{action}.json" + with (request.parent / f"{action}.log").open("wb") as log: + process = OwnedCommand([str(adapter), action, str(request), str(output)], log) + try: + returncode = process.wait(timeout) + if returncode: + raise subprocess.CalledProcessError(returncode, [str(adapter), action]) + result = read_json(output) + except BaseException: + process.finish(terminate=True) + raise + else: + # Successful prepare may intentionally leave adapter-owned services. + process.finish() + return result + + +def validate_result(result, request, expected): + require(result.get("evidence") == request["evidence"], "adapter evidence type mismatch") + require(result.get("fixed") == request["fixed"], "offered load/config/cache/durability drift") + require(result.get("build") == request["build"], "deployed build provenance mismatch") + require(result.get("data_dir") == request["data_dir"], "adapter data isolation mismatch") + require(result.get("background") == request["background"], "background mode mismatch") + require(type(result.get("sample_count")) is int and 1 <= result["sample_count"] <= 3600, + "sample_count must be 1..3600") + number(result.get("elapsed_seconds"), "elapsed_seconds", request["duration_seconds"]) + metrics = result["metrics"] + for key in METRICS: + number(metrics.get(key), key) + for key in ("requests", "p99_ms", "throughput_ops"): + require(metrics[key] > 0, f"zero {key}") + require(metrics["errors"] == 0, "workload request errors") + require(metrics["cold_walk_objects"] <= metrics["walk_objects"], "cold walk exceeds total walk") + require(result.get("oracle") == expected, "object/version/byte oracle mismatch") + if request["background"] == "on": + require(metrics["walk_objects"] > 0, "zero background walk") + require(metrics["healed_objects"] == request["expected_healed_objects"], "incomplete repair oracle") + if request["scenario"] in ("running-heal", "mrf-replay"): + require(metrics["healed_objects"] > 0, "zero completed repairs") + return result + + +def convergence(result): + window = result.get("convergence") + if not window or window.get("writes_stopped") is not True or window.get("last_mutation_observed") is not True or window.get("first_complete_publication") is not True: + return None + for key in ("last_mutation_time", "last_mutation_observed_time", "writes_stopped_time", "window_start", "window_end", "walk_objects", "full_walk_objects", "budget_available_seconds"): + number(window.get(key), f"convergence.{key}") + require(window["last_mutation_time"] <= window["writes_stopped_time"] <= window["window_start"] < window["window_end"], + "invalid post-mutation convergence window") + require(window["last_mutation_time"] <= window["last_mutation_observed_time"] <= window["window_start"], + "convergence started before last mutation was observed") + require(window["full_walk_objects"] > 0, "zero full walk reference") + require(0 < window["budget_available_seconds"] <= window["window_end"] - window["window_start"], + "invalid convergence budget window") + return ratio(window["walk_objects"], window["full_walk_objects"], "convergence work") + + +def evaluate(cells): + comparisons = [] + inconclusive = False + failed = False + for offset in range(0, len(cells), 4): + group = cells[offset:offset + 4] + require([cell["leg"] for cell in group] == list(LEGS), "incomplete ABBA group") + a1, b1, b2, a2 = (cell["result"]["metrics"] for cell in group) + control = group[0]["comparison"] == "background" + drift = max(abs(relative_change(a2[k], a1[k], k)) for k in ("p99_ms", "throughput_ops")) + repeat_drift = max(abs(relative_change(b2[k], b1[k], k)) for k in ("p99_ms", "throughput_ops")) + noise = max(drift, repeat_drift) > REPEATABILITY_LIMIT + a = {key: (decimal_number(a1[key], key) + decimal_number(a2[key], key)) / Decimal("2") for key in METRICS} + b = {key: (decimal_number(b1[key], key) + decimal_number(b2[key], key)) / Decimal("2") for key in METRICS} + p99 = relative_change(b["p99_ms"], a["p99_ms"], "p99_ms") + throughput = relative_change(b["throughput_ops"], a["throughput_ops"], "throughput_ops") + thresholds = {"p99_regression": Decimal("0.10") if control else Decimal("0.05"), + "throughput_loss": Decimal("0.05") if control else Decimal("0.03")} + passed = p99 <= thresholds["p99_regression"] and throughput >= -thresholds["throughput_loss"] + p1 = None + work_drift = None + if not control: + if group[0]["scenario"] == "cold-hot": + require(a["cold_walk_objects"] > 0, "cold-hot baseline has no cold walk samples") + work_drift = max(repeatability_change(a1[key], a2[key], key) for key in ("walk_objects", "cold_walk_objects")) + work_drift = max(work_drift, *(repeatability_change(b1[key], b2[key], key) for key in ("walk_objects", "cold_walk_objects"))) + noise |= work_drift > REPEATABILITY_LIMIT + required = ratio(a["cold_walk_objects"], a["walk_objects"], "cold walk baseline") * Decimal("0.80") + reduction = Decimal("1") - ratio(b["walk_objects"], a["walk_objects"], "walk reduction") + p1 = {"required_reduction": float(required), "observed_reduction": float(reduction), + "repeatability_drift": report_number(work_drift)} + if group[0]["scenario"] == "cold-hot": + # Compare counts before division can round repeating decimal ratios. + passed &= a["walk_objects"] - b["walk_objects"] >= a["cold_walk_objects"] * Decimal("0.80") + p2 = [convergence(cell["result"]) if cell["background"] == "on" else None for cell in group] + candidate_p2 = [value for cell, value in zip(group, p2) if cell["leg"].startswith("B")] + p2_pending = any(value is None for value in candidate_p2) + passed &= all(ratio(value, 1, "p2 work multiple") <= P2_WORK_MULTIPLE_LIMIT for value in candidate_p2 if value is not None) + p2_report = [None if value is None else float(value) for value in p2] + inconclusive |= noise or p2_pending + if not noise and not passed: + failed = True + comparisons.append({"scenario": group[0]["scenario"], "comparison": group[0]["comparison"], + "round": group[0]["round"], "status": "inconclusive" if noise else ("fail" if not passed else "inconclusive" if p2_pending else "pass"), + "a2_a1_drift": report_number(drift), "b2_b1_drift": report_number(repeat_drift), + "p99_regression": float(p99), "throughput_change": float(throughput), + "thresholds": {key: float(value) for key, value in thresholds.items()}, + "p1": p1, "p2_max_work_multiple": float(P2_WORK_MULTIPLE_LIMIT), + "p2_post_stop_work_multiples": p2_report}) + return ("fail" if failed else "inconclusive" if inconclusive else "pass"), comparisons + + +def collect_live(prepared, request, request_path, adapter): + collector = Path(__file__).with_name("run_scanner_validation_harness.sh") + # Only allow connection fields here; the runner owns cadence and output paths. + connection = prepared["collector"] + require(set(connection) == {"alias", "endpoint", "metrics_endpoints"}, "invalid collector connection") + require(all(isinstance(value, str) and value for value in connection.values()), "missing collector endpoint") + output = request_path.parent / "telemetry" + args = ["bash", str(collector), "--alias", connection["alias"], "--endpoint", connection["endpoint"], + "--metrics-endpoints", connection["metrics_endpoints"], "--deployment", "distributed", + "--samples", str(request["duration_seconds"] // 60 + 1), "--interval-secs", "60", + "--out-dir", str(output)] + with (request_path.parent / "collector.log").open("wb") as log: + process = OwnedCommand(args, log) + try: + started = time.monotonic() + result = invoke(adapter, "measure", request_path, request["duration_seconds"] + 300) + require(time.monotonic() - started >= request["duration_seconds"], "measurement ended before required window") + require(process.wait(120) == 0, "scanner collector failed") + require(output.joinpath("scanner-summary.csv").stat().st_size > 0, "missing collector samples") + samples = list((output / "status").glob("scanner-status.*.json")) + require(len(samples) == request["duration_seconds"] // 60 + 1, "missing scanner samples") + for sample in samples: + status = read_json(sample) + require(isinstance(status.get("metrics"), dict) and status["metrics"], "invalid scanner status response") + heals = list((output / "heal").glob("background-heal-status.*.json")) + require(bool(heals), "missing heal samples") + for sample in heals: + status = read_json(sample) + require(isinstance(status.get("healOperations"), dict) and status["healOperations"], "invalid heal status response") + metrics = list((output / "metrics").glob("admin-metrics.*.ndjson")) + endpoints = [endpoint for endpoint in connection["metrics_endpoints"].split(",") if endpoint] + require(metrics and len(metrics) == len(endpoints) * len(samples), "missing distributed metrics samples") + for sample in metrics: + # The collector requests n=1, so each file contains one final JSON record. + status = read_json(sample) + require(status.get("errors") == [], "distributed metrics errors") + require(status.get("final") is True, "incomplete distributed metrics") + hosts = status.get("by_host") + require(isinstance(hosts, dict) and hosts, "missing by-host metrics") + for host in hosts.values(): + require(isinstance(host, dict) and isinstance(host.get("scanner"), dict) and host["scanner"], + "missing per-host scanner metrics") + return result + finally: + process.finish(terminate=True) + + +def run(manifest, adapter, output, data_root): + validate_manifest(manifest) + require(adapter.is_file() and os.access(adapter, os.X_OK), "missing executable adapter") + require(not output.exists() and not data_root.exists(), "output/data root must be new; existing data is preserved") + require(output != data_root and output not in data_root.parents and data_root not in output.parents, + "output and data roots must not overlap") + output.mkdir(parents=True) + data_root.mkdir(parents=True) + require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space") + manifest["adapter_sha256"] = digest(adapter) + manifest["collector_sha256"] = digest(Path(__file__).with_name("run_scanner_validation_harness.sh")) + write_json(output / "manifest.json", manifest) + cells = [] + write_json(output / "report.json", {"status": "incomplete", "performance": "pending"}) + try: + for scenario in SCENARIOS: + for comparison in ("build", "background"): + for round_id in range(1, manifest["rounds"] + 1): + for leg in LEGS: + phase = "baseline" if comparison == "build" and leg.startswith("A") else "candidate" + background = "off" if comparison == "background" and leg.startswith("A") else "on" + name = f"{scenario}-{comparison}-{round_id}-{leg}" + cell_dir = output / name + cell_dir.mkdir() + data_dir = data_root / name + data_dir.mkdir() + request = {"schema": 1, "scenario": scenario, "comparison": comparison, "round": round_id, + "leg": leg, "background": background, "build": manifest[phase], + "evidence": manifest["evidence"], "fixed": manifest["fixed"], + "duration_seconds": manifest["duration_seconds"], "data_dir": str(data_dir), + "expected_healed_objects": manifest["expected_healed_objects"][scenario], + "expected_oracle": manifest["oracles"][scenario]} + require(digest(Path(request["build"]["binary"])) == request["build"]["sha256"], "binary changed during run") + require(digest(adapter) == manifest["adapter_sha256"], "adapter changed during run") + require(shutil.disk_usage(data_root).free >= manifest["min_free_bytes"], "insufficient free disk space") + request_path = cell_dir / "request.json" + write_json(request_path, request) + print(name, flush=True) + try: + prepared = invoke(adapter, "prepare", request_path, 300) + require(prepared.get("ready") is True, "deployment not ready") + if manifest["evidence"] == "measured": + result = collect_live(prepared, request, request_path, adapter) + else: + result = invoke(adapter, "measure", request_path, 300) + # An independent operation must enumerate all object versions and bytes. + oracle = invoke(adapter, "oracle", request_path, 300) + require(oracle.get("complete") is True and oracle.get("errors") == 0, "correctness oracle failed") + require(type(oracle.get("errors")) is int, "invalid oracle error count") + result["oracle"] = oracle["actual"] + validate_result(result, request, request["expected_oracle"]) + cells.append({**request, "result": result}) + finally: + stopped = invoke(adapter, "stop", request_path, 300) + require(stopped.get("stopped") is True, "adapter failed to stop deployment") + status, comparisons = evaluate(cells) + synthetic = manifest["evidence"] == "synthetic" + report = {"status": "synthetic_validated" if synthetic and status == "pass" else status, + "evidence": manifest["evidence"], "performance": "pending" if synthetic else status, + "cells": len(cells), "comparisons": comparisons} + write_json(output / "report.json", report) + return 0 if status == "pass" else 3 if status == "inconclusive" else 1 + except (ValueError, KeyError, OSError, subprocess.SubprocessError) as error: + write_json(output / "report.json", {"status": "failed", "performance": "pending", + "completed_cells": len(cells), "error": str(error)}) + raise + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--adapter", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--data-root", type=Path, required=True) + args = parser.parse_args() + try: + return run(read_json(args.manifest), args.adapter.resolve(), args.out_dir.resolve(), args.data_root.resolve()) + except (ValueError, KeyError, OSError, subprocess.SubprocessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_diagnose_scanner_enumeration_restart.py b/scripts/test_diagnose_scanner_enumeration_restart.py new file mode 100644 index 000000000..efe5f2ddd --- /dev/null +++ b/scripts/test_diagnose_scanner_enumeration_restart.py @@ -0,0 +1,73 @@ +"""Driver contract tests; these do not replace the real scanner diagnostic.""" + +import unittest + +from diagnose_scanner_enumeration_restart import converged, validate_report + + +class ReportTests(unittest.TestCase): + def report(self): + return dict(schema=1, round=0, pid=123, objects_expected=4, raw_entry_budget=16, + raw_entries=8, raw_name_bytes=64, objects_before=0, objects_retained=4, + versions_retained=4, bytes_retained=4, objects_processed=4, + snapshot_complete=True, outcome="complete") + + def validate(self, report): + validate_report(report, round_number=0, pid=123, objects=4, budget=16) + + def test_complete_exact_coverage_satisfies_oracle(self): + report = self.report() + self.validate(report) + self.assertTrue(converged(report, 4)) + + def test_incomplete_or_inexact_coverage_cannot_pass(self): + for key, value in (("snapshot_complete", False), ("objects_retained", 3), + ("versions_retained", 3), ("bytes_retained", 3), ("outcome", "partial")): + with self.subTest(key=key): + report = self.report() + report[key] = value + self.assertFalse(converged(report, 4)) + + def test_wrong_process_or_round_rejected(self): + for key in ("pid", "round", "schema", "raw_entry_budget", "objects_expected"): + with self.subTest(key=key): + report = self.report() + report[key] += 1 + with self.assertRaises(ValueError): + self.validate(report) + + def test_unbudgeted_tail_rejected(self): + report = self.report() + report["raw_entries"] = 17 + with self.assertRaises(ValueError): + self.validate(report) + + def test_complete_coverage_without_entry_observation_rejected(self): + report = self.report() + report["raw_entries"] = 0 + with self.assertRaises(ValueError): + self.validate(report) + + def test_missing_wrong_type_and_negative_counter_rejected(self): + for value in (None, True, -1, "8", 1048577): + with self.subTest(value=value): + report = self.report() + report["raw_entries"] = value + with self.assertRaises(ValueError): + self.validate(report) + + def test_missing_completeness_or_unknown_outcome_rejected(self): + for key in ("snapshot_complete", "outcome"): + report = self.report() + del report[key] + with self.assertRaises(ValueError): + self.validate(report) + + def test_non_object_report_rejected(self): + for report in (None, [], "report"): + with self.assertRaises(ValueError): + self.validate(report) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_scanner_abba.py b/scripts/test_scanner_abba.py new file mode 100755 index 000000000..5408eb44c --- /dev/null +++ b/scripts/test_scanner_abba.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python3 +"""Synthetic adapter and failure-propagation tests; never start a RustFS server.""" + +import contextlib +import copy +import fcntl +import io +import json +import os +from pathlib import Path +import shlex +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest.mock import Mock, patch + +import scanner_abba as harness + + +def fake_adapter(): + action, request_path, output_path = sys.argv[1:] + request = harness.read_json(Path(request_path)) + fault = os.environ.get("SCANNER_ABBA_TEST_FAULT", "") + if fault == "stubborn-child" and action in ("prepare", "measure"): + marker = Path(request_path).parent / "stubborn.pid" + if os.fork() == 0: + os.execv(sys.executable, [sys.executable, str(Path(__file__).resolve()), "--stubborn-worker", str(marker)]) + wait_for_marker(marker) + if action == "measure": + time.sleep(60) + if action == "prepare": + result = {"ready": True} + elif action == "stop": + if fault == "stubborn-child": + reap_fixture(Path(request_path).parent / "stubborn.pid") + result = {"stopped": True} + elif action == "oracle": + if fault == "oracle-exit": + return 42 + if fault == "missing-oracle": + return 0 + result = {"complete": True, "errors": 0, "actual": request["expected_oracle"]} + if fault == "oracle-mismatch": + result["actual"]["bytes"] += 1 + else: + if fault == "measure-exit": + return 42 + result = {key: request[key] for key in ("evidence", "fixed", "build", "data_dir", "background")} + result.update({"sample_count": 10, "elapsed_seconds": request["duration_seconds"], + "metrics": dict.fromkeys(harness.METRICS, 10)}) + baseline = request["comparison"] == "build" and request["leg"].startswith("A") + result["metrics"].update(p99_ms=10, throughput_ops=100, errors=0, requests=100, + walk_objects=100 if baseline else 20, cold_walk_objects=100 if baseline else 0, + healed_objects=request["expected_healed_objects"]) + result["convergence"] = {"writes_stopped": True, "last_mutation_observed": True, + "first_complete_publication": True, "last_mutation_time": 1, + "last_mutation_observed_time": 2, + "writes_stopped_time": 2, "window_start": 2, "window_end": 3, + "budget_available_seconds": 1, "walk_objects": 110, "full_walk_objects": 100} + if fault == "zero-samples": + result["sample_count"] = 0 + elif fault == "request-errors": + result["metrics"]["errors"] = 1 + elif fault == "load-drift": + result["fixed"]["offered_load_ops"] += 1 + elif fault == "noise" and request["leg"] == "A2": + result["metrics"]["p99_ms"] = 20 + elif fault == "zero-requests": + result["metrics"]["requests"] = 0 + elif fault == "no-publication": + result["convergence"]["first_complete_publication"] = False + elif fault == "p2-regression": + result["convergence"]["walk_objects"] = 121 + elif fault == "latency-regression" and request["leg"].startswith("B"): + result["metrics"]["p99_ms"] = 12 + elif fault == "exact-thresholds" and request["leg"].startswith("B"): + result["metrics"].update(p99_ms=10.5, throughput_ops=97) + elif fault == "just-over-threshold" and request["comparison"] == "build" and request["leg"].startswith("B"): + result["metrics"]["p99_ms"] = 10.500001 + elif fault == "p1-regression" and not baseline: + result["metrics"]["walk_objects"] = 30 + elif fault in ("p1-exact-fraction", "p1-over-fraction"): + result["metrics"].update(walk_objects=9 if baseline else 5 + (fault == "p1-over-fraction"), + cold_walk_objects=5 if baseline else 0) + elif fault == "unstable-p1-control" and request["comparison"] == "build": + if request["leg"] == "A1": + result["metrics"].update(walk_objects=1000, cold_walk_objects=1000) + elif request["leg"] == "A2": + result["metrics"].update(walk_objects=10, cold_walk_objects=10) + elif request["leg"].startswith("B"): + result["metrics"].update(walk_objects=100, cold_walk_objects=0) + elif fault == "missing-metric": + del result["metrics"]["save_bytes"] + elif fault == "incomplete-repair": + result["metrics"]["healed_objects"] = 0 + harness.write_json(Path(output_path), result) + return 0 + + +def wait_for_marker(marker): + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + if marker.exists() and marker.stat().st_size: + return + time.sleep(0.01) + raise AssertionError("fixture child did not become ready") + + +def child_released(marker, timeout=1): + deadline = time.monotonic() + timeout + with marker.open("r+") as stream: + while True: + try: + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + return True + except BlockingIOError: + if time.monotonic() >= deadline: + return False + time.sleep(0.01) + + +def reap_fixture(marker): + if marker.exists() and marker.stat().st_size and not child_released(marker, timeout=0): + # The unique file lock proves the original fixture process still owns this PID. + os.kill(int(marker.read_text()), signal.SIGKILL) + if not child_released(marker, timeout=5): + raise AssertionError("fixture child did not release its process-owned lock") + + +class ScannerAbbaTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.binary = Path(sys.executable).resolve() + self.adapter = Path(__file__).resolve() + self.manifest = { + "schema": 1, "evidence": "synthetic", "rounds": 3, "duration_seconds": 1, "min_free_bytes": 1, + "fixed": {"config_sha256": "1" * 64, "dataset_sha256": "2" * 64, + "release_flags": "--release", "durability": "drive-sync=on", + "disk_type": "synthetic", "cache_state": "cold", "load_command": "fake", + "topology": "EC8+4", "offered_load_ops": 100, "resource_isolation": "synthetic"}, + "oracles": {s: {"objects": 10, "versions": 20, "bytes": 30, "sha256": "3" * 64} for s in harness.SCENARIOS}, + "expected_healed_objects": {s: 10 for s in harness.SCENARIOS}, + } + build = {"binary": str(self.binary), "sha256": harness.digest(self.binary), "revision": "a" * 40} + self.manifest.update(baseline=build.copy(), candidate=build.copy()) + + def run_harness(self, fault=""): + with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": fault}), contextlib.redirect_stdout(io.StringIO()): + return harness.run(copy.deepcopy(self.manifest), self.adapter, self.root / "out", self.root / "data") + + def test_adapter_timeout_reaps_group_after_parent_exits_on_term(self): + request = self.root / "request.json" + harness.write_json(request, {}) + marker = self.root / "stubborn.pid" + try: + with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}): + with self.assertRaises(subprocess.TimeoutExpired): + harness.invoke(self.adapter, "measure", request, 3) + wait_for_marker(marker) + self.assertTrue(child_released(marker), "TERM-exited parent left its TERM-ignoring child alive") + finally: + reap_fixture(marker) + + def test_collector_failure_reaps_group_after_parent_exits_on_term(self): + request = self.root / "request.json" + harness.write_json(request, {}) + marker = self.root / "stubborn.pid" + collector = self.root / "run_scanner_validation_harness.sh" + command = [sys.executable, str(self.adapter), "measure", str(request), str(self.root / "unused.json")] + collector.write_text("#!/usr/bin/env bash\nexec " + shlex.join(command) + "\n") + + def failed_measure(*_): + wait_for_marker(marker) + raise ValueError("injected measurement failure") + + try: + with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}), \ + patch.object(harness, "__file__", str(self.root / "scanner_abba.py")), \ + patch.object(harness, "invoke", side_effect=failed_measure): + with self.assertRaisesRegex(ValueError, "injected measurement failure"): + harness.collect_live({"collector": {"alias": "fixture", "endpoint": "fixture", "metrics_endpoints": "fixture"}}, + {"duration_seconds": 900}, request, self.adapter) + self.assertTrue(child_released(marker), "collector parent exit did not end its telemetry child") + finally: + reap_fixture(marker) + + def test_successful_prepare_keeps_service_alive(self): + request = self.root / "request.json" + harness.write_json(request, {}) + marker = self.root / "stubborn.pid" + try: + with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}): + self.assertEqual(harness.invoke(self.adapter, "prepare", request, 5), {"ready": True}) + self.assertFalse(child_released(marker, timeout=0), "successful prepare must preserve its service") + self.assertEqual(harness.invoke(self.adapter, "stop", request, 5), {"stopped": True}) + self.assertTrue(child_released(marker), "adapter stop must release its service") + finally: + reap_fixture(marker) + + def test_reaped_owner_never_signals_a_reused_process_group(self): + with (self.root / "owner.log").open("wb") as log: + owner = harness.OwnedCommand([sys.executable, "-c", "pass"], log) + self.assertEqual(owner.wait(5), 0) + self.assertEqual(owner.finish(), 0) + with patch.object(harness.os, "killpg", side_effect=AssertionError("released PGID must not be signalled")): + self.assertEqual(owner.finish(terminate=True), 0) + + def test_cleanup_interruption_still_kills_group_and_reaps_leader(self): + request = self.root / "request.json" + harness.write_json(request, {}) + marker = self.root / "stubborn.pid" + original_sleep = time.sleep + interrupted = False + + def interrupt_once(delay): + nonlocal interrupted + if not interrupted: + interrupted = True + raise KeyboardInterrupt + original_sleep(delay) + + with (self.root / "interrupted.log").open("wb") as log: + with patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}): + owner = harness.OwnedCommand([str(self.adapter), "measure", str(request), str(self.root / "unused.json")], log) + try: + wait_for_marker(marker) + with patch.object(harness.time, "sleep", side_effect=interrupt_once): + with self.assertRaises(KeyboardInterrupt): + owner.finish(terminate=True) + self.assertTrue(child_released(marker), "cleanup cancellation left its child alive") + self.assertIsNotNone(owner.process.returncode, "cleanup cancellation must reap its leader") + finally: + reap_fixture(marker) + owner.process.wait(timeout=5) + + def test_constructor_failure_after_gate_release_kills_group(self): + request = self.root / "request.json" + harness.write_json(request, {}) + marker = self.root / "stubborn.pid" + original_write = os.write + + def release_then_fail(fd, data): + original_write(fd, data) + wait_for_marker(marker) + raise OSError("injected failure after gate release") + + try: + with (self.root / "construction.log").open("wb") as log, \ + patch.dict(os.environ, {"SCANNER_ABBA_TEST_FAULT": "stubborn-child"}), \ + patch.object(harness.os, "write", side_effect=release_then_fail): + with self.assertRaisesRegex(OSError, "injected failure after gate release"): + harness.OwnedCommand([str(self.adapter), "measure", str(request), str(self.root / "unused.json")], log) + self.assertTrue(child_released(marker), "initialization failure left its child alive") + finally: + reap_fixture(marker) + + def test_complete_synthetic_matrix_is_not_performance_evidence(self): + self.assertEqual(self.run_harness(), 0) + report = harness.read_json(self.root / "out/report.json") + self.assertEqual((report["status"], report["performance"], report["cells"]), ("synthetic_validated", "pending", 120)) + requests = [harness.read_json(path) for path in (self.root / "out").glob("*/request.json")] + self.assertEqual(len({r["data_dir"] for r in requests}), 120) + for scenario in harness.SCENARIOS: + for comparison in ("build", "background"): + for round_id in (1, 2, 3): + legs = [r for r in requests if (r["scenario"], r["comparison"], r["round"]) == (scenario, comparison, round_id)] + self.assertEqual({r["leg"] for r in legs}, set(harness.LEGS)) + self.assertTrue(all(c["p2_max_work_multiple"] == 1.2 for c in report["comparisons"])) + + def test_fail_closed_adapter_and_data_errors(self): + for fault in ("measure-exit", "oracle-exit", "missing-oracle", "oracle-mismatch", "zero-samples", + "zero-requests", "request-errors", "load-drift", "missing-metric", "incomplete-repair"): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory: + self.root = Path(directory) + with self.assertRaises((ValueError, OSError, subprocess.SubprocessError)): + self.run_harness(fault) + report = harness.read_json(self.root / "out/report.json") + self.assertEqual(report["status"], "failed") + self.assertTrue(list((self.root / "out").glob("*/stop.json"))) + + def test_noise_is_inconclusive_and_nonzero(self): + with patch.object(harness, "SCENARIOS", ("cold-hot",)): + self.assertEqual(self.run_harness("noise"), 3) + self.assertEqual(harness.read_json(self.root / "out/report.json")["status"], "inconclusive") + + def test_missing_first_publication_is_inconclusive(self): + with patch.object(harness, "SCENARIOS", ("cold-hot",)): + self.assertEqual(self.run_harness("no-publication"), 3) + + def test_performance_regressions_fail(self): + for fault in ("p1-regression", "p2-regression", "latency-regression"): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory: + self.root = Path(directory) + with patch.object(harness, "SCENARIOS", ("cold-hot",)): + self.assertEqual(self.run_harness(fault), 1) + + def test_exact_threshold_boundaries_pass(self): + with patch.object(harness, "SCENARIOS", ("cold-hot",)): + self.assertEqual(self.run_harness("exact-thresholds"), 0) + + def test_just_over_threshold_fails(self): + with patch.object(harness, "SCENARIOS", ("cold-hot",)): + self.assertEqual(self.run_harness("just-over-threshold"), 1) + + def test_p1_fractional_boundary(self): + for fault, expected in (("p1-exact-fraction", 0), ("p1-over-fraction", 1)): + with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory: + self.root = Path(directory) + with patch.object(harness, "SCENARIOS", ("cold-hot",)): + self.assertEqual(self.run_harness(fault), expected) + + def test_live_collector_rejects_missing_or_failed_node_metrics(self): + telemetry = self.root / "telemetry" + for name in ("status", "heal", "metrics"): + (telemetry / name).mkdir(parents=True) + (telemetry / "scanner-summary.csv").write_text("timestamp\n") + valid = {"errors": [], "final": True, "by_host": {"node-b:9000": {"scanner": {"objects": 10}}}} + for index in range(16): + harness.write_json(telemetry / f"status/scanner-status.{index}.json", {"metrics": {"objects": 10}}) + for node in ("node-a", "node-b"): + harness.write_json(telemetry / f"heal/background-heal-status.{node}.{index}.json", + {"healOperations": {"queueLength": 0}}) + harness.write_json(telemetry / f"metrics/admin-metrics.{node}.{index}.ndjson", + {**valid, "by_host": {f"{node}:9000": {"scanner": {"objects": 10}}}}) + sample = telemetry / "metrics/admin-metrics.node-b.15.ndjson" + prepared = {"collector": {"alias": "test", "endpoint": "http://node-a:9000", + "metrics_endpoints": "http://node-a:9000,http://node-b:9000"}} + cases = ( + ("valid", valid, None), + ("missing", None, "missing distributed metrics samples"), + ("empty", "", "Expecting value"), + ("http-error", {"Code": "AccessDenied"}, "distributed metrics errors"), + ("partial-error", {**valid, "errors": ["node unavailable"]}, "distributed metrics errors"), + ("unfinished", {**valid, "final": False}, "incomplete distributed metrics"), + ("missing-host", {**valid, "by_host": {}}, "missing by-host metrics"), + ("missing-scanner", {**valid, "by_host": {"node-a:9000": {}}}, "missing per-host scanner metrics"), + ("collector-exit", valid, "scanner collector failed"), + ) + for name, payload, error in cases: + with self.subTest(fault=name): + if payload is None: + sample.unlink() + elif isinstance(payload, str): + sample.write_text(payload) + else: + harness.write_json(sample, payload) + process = Mock(pid=123, wait=Mock(return_value=1 if name == "collector-exit" else 0)) + with patch.object(harness, "OwnedCommand", return_value=process), \ + patch.object(harness, "invoke", return_value={"sample_count": 10}), \ + patch.object(harness.time, "monotonic", side_effect=(0, 900)): + if error: + with self.assertRaisesRegex(ValueError, error): + harness.collect_live(prepared, {"duration_seconds": 900}, self.root / "request.json", self.adapter) + else: + self.assertEqual(harness.collect_live(prepared, {"duration_seconds": 900}, + self.root / "request.json", self.adapter), {"sample_count": 10}) + + process.finish.assert_called_once_with(terminate=True) + + def test_unstable_p1_work_control_is_inconclusive(self): + with patch.object(harness, "SCENARIOS", ("cold-hot",)): + self.assertEqual(self.run_harness("unstable-p1-control"), 3) + comparison = harness.read_json(self.root / "out/report.json")["comparisons"][0] + self.assertEqual(comparison["status"], "inconclusive") + self.assertGreater(comparison["p1"]["repeatability_drift"], 0.05) + + def test_manifest_rejects_missing_build_or_oracle(self): + for section, key in (("baseline", "binary"), ("oracles", "cold-hot")): + manifest = copy.deepcopy(self.manifest) + del manifest[section][key] + with self.subTest(section=section), self.assertRaises((ValueError, KeyError)): + harness.validate_manifest(manifest) + + def test_short_measured_window_and_fewer_rounds_rejected(self): + self.manifest["evidence"] = "measured" + with self.assertRaisesRegex(ValueError, "duration_seconds"): + harness.validate_manifest(self.manifest) + self.manifest["duration_seconds"] = 900 + self.manifest["rounds"] = 2 + with self.assertRaisesRegex(ValueError, "rounds"): + harness.validate_manifest(self.manifest) + + def test_existing_data_preserved(self): + (self.root / "data").mkdir() + marker = self.root / "data/keep" + marker.write_text("existing") + with self.assertRaisesRegex(ValueError, "preserved"): + self.run_harness() + self.assertEqual(marker.read_text(), "existing") + + def test_invalid_or_live_write_window_does_not_claim_p2(self): + self.assertIsNone(harness.convergence({"convergence": {"writes_stopped": False}})) + with self.assertRaises(ValueError): + harness.convergence({"convergence": {"writes_stopped": True, "last_mutation_observed": True, + "first_complete_publication": True}}) + + def test_nan_and_oversized_samples_rejected(self): + with self.assertRaises(ValueError): + harness.number(float("nan"), "latency") + path = self.root / "oversized.json" + path.write_bytes(b" " * (harness.MAX_JSON_BYTES + 1)) + with self.assertRaisesRegex(ValueError, "oversized"): + harness.read_json(path) + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--stubborn-worker": + signal.signal(signal.SIGTERM, signal.SIG_IGN) + with Path(sys.argv[2]).open("w+") as marker: + fcntl.flock(marker, fcntl.LOCK_EX) + marker.write(str(os.getpid())) + marker.flush() + while True: + time.sleep(1) + if len(sys.argv) == 4 and sys.argv[1] in ("prepare", "measure", "oracle", "stop"): + sys.exit(fake_adapter()) + unittest.main() diff --git a/scripts/test_scanner_validation_harness.sh b/scripts/test_scanner_validation_harness.sh index 9107926b1..d360097b8 100755 --- a/scripts/test_scanner_validation_harness.sh +++ b/scripts/test_scanner_validation_harness.sh @@ -312,3 +312,5 @@ if PATH="$BIN_DIR:$PATH" "$SCRIPT" --secret-key rustfsadmin >"$secret_arg_log" 2 fi grep -q -- 'unknown arg: --secret-key' "$secret_arg_log" + +"$ROOT_DIR/scripts/python_bin.sh" "$ROOT_DIR/scripts/test_scanner_abba.py" diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index 23268bf73..ba483571f 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -110,6 +110,7 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase): self.env = { **os.environ, "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name, "TMPDIR": self.temp.name, + "LOG_FILE": str(self.directory / "suite.log"), } for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"): self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]