gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-05 19:55:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into docs/scanner-heal-v2-plan

Browse Source
This commit is contained in:
houseme
2026-09-05 18:43:47 +08:00
committed by GitHub
parent a46e030420 7ba5cd6888
commit c448afe4a3
69 changed files with 8627 additions and 1217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/make/tests.mak
+1
View File
@@ -40,6 +40,7 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
.github/workflows/ci-docs-only.yml
+1
View File
@@ -129,6 +129,7 @@ jobs:
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
.github/workflows/ci.yml
+1
View File
@@ -167,6 +167,7 @@ jobs:
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
.github/workflows/rustfs-security-test.yml
+55 -28
View File
@@ -74,10 +74,23 @@ env:
jobs:
security-test:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Initialize security evidence
id: evidence
run: |
set -euo pipefail
umask 077
SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${SECURITY_ARTIFACTS_DIR}"
printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -98,11 +111,6 @@ jobs:
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
exit 1
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Show environment
run: |
uname -a
@@ -135,7 +143,8 @@ jobs:
id: test
continue-on-error: true
env:
REPORT_FILE: /tmp/rustfs-security-report.md
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
run: |
set -euo pipefail
@@ -159,29 +168,48 @@ jobs:
else
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
fi
./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
- name: Generate report
if: always()
id: report
if: ${{ always() && steps.evidence.outcome == 'success' }}
env:
TEST_OUTCOME: ${{ steps.test.outcome }}
run: |
set -euo pipefail
if [ ! -f /tmp/rustfs-security-report.md ]; then
{
echo "# RustFS security test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Test Step Outcome: failure (suite did not produce a report)"
} > /tmp/rustfs-security-report.md
RESULT=failure
if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
RESULT=success
fi
cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}"
{
echo "# RustFS security test report"
echo ""
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${TEST_OUTCOME}"
echo ""
# The dashboard prioritizes case rows over the step outcome.
# Keep partial case results in the artifact when the suite fails.
if [ "${RESULT}" = "success" ]; then
cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics."
else
echo "The suite did not produce a non-empty report."
fi
} > "${SECURITY_ARTIFACTS_DIR}/report.md"
cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: /tmp/rustfs-security-report.md
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
SUITE: security
run: |
set -euo pipefail
@@ -210,8 +238,9 @@ jobs:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
SUITE: 'security'
SUITE_LABEL: 'Security'
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-security-report.md'
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
LOG_FILE: ''
run: |
set -euo pipefail
@@ -245,7 +274,7 @@ jobs:
echo ""
echo "## Report (errors and symptoms)"
echo ""
if [ -s "${REPORT_FILE}" ]; then
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
@@ -263,14 +292,12 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-security-test-${{ github.run_id }}
path: |
/tmp/rustfs-security-report.md
/tmp/rustfs-security.*/*
if-no-files-found: ignore
name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
if-no-files-found: error
retention-days: 3
- name: Cleanup environment (after)
.github/workflows/scheduled-validation-freshness.yml
+1
View File
@@ -42,6 +42,7 @@ jobs:
- name: Check latest scheduled runs
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: |
set +e
python3 scripts/check_scheduled_validation_freshness.py \
.gitignore
+1
View File
@@ -33,6 +33,7 @@ profile.json
*.zst
.secrets
*.go
!crates/zip/tests/fixtures/snowball/**/generate/*.go
*.pb
*.svg
deploy/logs/*.log.*
Cargo.lock
Generated
+220 -9
View File
@@ -164,6 +164,12 @@ version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "ambient-authority"
version = "0.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b"
[[package]]
name = "amq-protocol"
version = "10.6.3"
@@ -330,6 +336,19 @@ dependencies = [
"rustversion",
]
[[package]]
name = "archive-trait"
version = "0.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6080ea14ccf9019d7ce572c319e581c20a805c9bdc6dbfb9b988da003cbd1a3"
dependencies = [
"cap-std",
"thiserror 2.0.20",
"tokio",
"walkdir",
"windows-sys 0.60.2",
]
[[package]]
name = "arcstr"
version = "1.2.0"
@@ -1873,6 +1892,36 @@ dependencies = [
"serde_core",
]
[[package]]
name = "cap-primitives"
version = "4.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b5f74729fd2f44701d1a8eb47e906cdb3ccd9ec0f02baad85a744b791940b18"
dependencies = [
"ambient-authority",
"fs-set-times",
"io-extras",
"io-lifetimes 3.0.1",
"ipnet",
"maybe-owned",
"rustix",
"rustix-linux-procfs",
"windows-sys 0.61.2",
"winx",
]
[[package]]
name = "cap-std"
version = "4.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8"
dependencies = [
"cap-primitives",
"io-extras",
"io-lifetimes 3.0.1",
"rustix",
]
[[package]]
name = "cargo-platform"
version = "0.3.3"
@@ -4442,6 +4491,17 @@ dependencies = [
"pe-unwind-info",
]
[[package]]
name = "fs-set-times"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
dependencies = [
"io-lifetimes 2.0.4",
"rustix",
"windows-sys 0.52.0",
]
[[package]]
name = "fs_extra"
version = "1.3.0"
@@ -5626,6 +5686,28 @@ dependencies = [
"tempfile",
]
[[package]]
name = "io-extras"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
dependencies = [
"io-lifetimes 3.0.1",
"windows-sys 0.52.0",
]
[[package]]
name = "io-lifetimes"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983"
[[package]]
name = "io-lifetimes"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96"
[[package]]
name = "io-uring"
version = "0.7.14"
@@ -6328,6 +6410,12 @@ version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608"
[[package]]
name = "maybe-owned"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4"
[[package]]
name = "md-5"
version = "0.10.6"
@@ -10904,9 +10992,16 @@ dependencies = [
name = "rustfs-zip"
version = "1.0.0-rc.5"
dependencies = [
"astral-tokio-tar",
"async-compression",
"futures",
"hotpath",
"rustfs-rio",
"serde",
"serde_json",
"sha2 0.11.0",
"tar-codec",
"tar-framing",
"thiserror 2.0.20",
"tokio",
]
@@ -10967,6 +11062,16 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "rustix-linux-procfs"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056"
dependencies = [
"once_cell",
"rustix",
]
[[package]]
name = "rustls"
version = "0.23.43"
@@ -12242,6 +12347,28 @@ dependencies = [
"xattr",
]
[[package]]
name = "tar-codec"
version = "0.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ea42eb144d30fcbf32c26dfea8959175bb8335bfb7b18d20c6ed32edecf0551"
dependencies = [
"archive-trait",
"tar-framing",
"thiserror 2.0.20",
"tokio",
]
[[package]]
name = "tar-framing"
version = "0.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "783223a7a6590be4227cb821e7ce80372575511f67c824921aba0752d8ad5573"
dependencies = [
"thiserror 2.0.20",
"tokio",
]
[[package]]
name = "tcp-stream"
version = "0.34.14"
@@ -13521,7 +13648,16 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
]
[[package]]
@@ -13539,14 +13675,31 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
]
[[package]]
@@ -13564,48 +13717,96 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "1.0.4"
@@ -13615,6 +13816,16 @@ dependencies = [
"memchr",
]
[[package]]
name = "winx"
version = "0.36.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
dependencies = [
"bitflags 2.13.1",
"windows-sys 0.52.0",
]
[[package]]
name = "wit-bindgen"
version = "0.57.1"
Cargo.toml
+4 -1
View File
@@ -234,8 +234,11 @@ tokio-postgres-rustls = "0.14.0"
# Utilities and Tools
anyhow = "1.0.104"
arc-swap = "1.9.2"
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams.
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork.
astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
tar-codec = "0.0.14"
tar-framing = "0.0.14"
atoi = "3.1.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.12.0" }
crates/e2e_test/src/replication_extension_test.rs
+93
View File
@@ -6743,6 +6743,99 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua
Ok(())
}
#[tokio::test]
async fn test_site_replication_replays_bucket_created_during_peer_outage_real_dual_node() -> TestResult {
init_logging();
// Keep compilation outside the scenario timeout. Recovery itself waits
// for the production 30-second lightweight retry tick.
let _rustfs_binary = rustfs_binary_path();
match timeout(Duration::from_secs(150), async {
let mut site_env = replication_fast_env();
site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
let mut site_a_env = RustFSTestEnvironment::new().await?;
site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
let mut site_b_env = RustFSTestEnvironment::new().await?;
site_b_env.start_rustfs_server_without_cleanup_with_env(&site_env).await?;
let site_a_client = site_a_env.create_s3_client();
let site_b_client = site_b_env.create_s3_client();
let bucket = "site-repl-peer-outage";
let key = "after-recovery.txt";
let payload = b"site replication recovered the missed bucket".to_vec();
let add_status = site_replication_add(
&site_a_env,
&[
PeerSite {
name: "outage-site-a".to_string(),
endpoint: site_a_env.url.clone(),
access_key: site_a_env.access_key.clone(),
secret_key: site_a_env.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "outage-site-b".to_string(),
endpoint: site_b_env.url.clone(),
access_key: site_b_env.access_key.clone(),
secret_key: site_b_env.secret_key.clone(),
..Default::default()
},
],
)
.await?;
assert!(add_status.success, "unexpected site add result: {add_status:?}");
wait_for_site_replication_enabled(&site_a_env, 2).await?;
wait_for_site_replication_enabled(&site_b_env, 2).await?;
site_b_env.stop_server();
site_a_client.create_bucket().bucket(bucket).send().await?;
site_a_client.head_bucket().bucket(bucket).send().await?;
let queued = site_replication_info(&site_a_env)
.await?
.retry_stats
.ok_or("peer outage did not persist a site replication retry event")?;
assert!(queued.pending + queued.failed > 0, "peer outage retry queue was unexpectedly empty");
site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
let recovery_deadline = tokio::time::Instant::now() + Duration::from_secs(75);
loop {
let bucket_recovered = site_b_client.head_bucket().bucket(bucket).send().await.is_ok();
let queue_empty = site_replication_info(&site_a_env).await?.retry_stats.is_none();
if bucket_recovered && queue_empty {
break;
}
if tokio::time::Instant::now() >= recovery_deadline {
return Err(format!(
"site replication retry did not settle after peer recovery; bucket_recovered={bucket_recovered}, queue_empty={queue_empty}"
)
.into());
}
sleep(Duration::from_millis(250)).await;
}
site_a_client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from(payload.clone()))
.send()
.await?;
assert_eq!(wait_for_object_on_target(&site_b_client, bucket, key).await?, payload);
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err("site replication peer-outage recovery timed out after 150 seconds".into()),
}
}
/// Re-applying a site's own replication config must not disable the peer's reverse direction.
///
/// `PutBucketReplication` broadcasts the config to every peer — the console's replication
crates/ecstore/src/api/mod.rs
+10 -9
View File
@@ -196,15 +196,16 @@ pub mod bucket {
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
pub use crate::bucket::metadata_sys::{
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config,
get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config,
get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata,
remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock,
acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
update_under_transaction_lock,
};
}
crates/ecstore/src/bucket/metadata_sys.rs
+114 -7
View File
@@ -655,6 +655,12 @@ pub struct BucketMetadataMutationGuard {
}
impl BucketMetadataMutationGuard {
/// Returns the storage-verified identity while both incarnation fences remain valid.
pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
self.ensure_valid(&self.bucket)?;
Ok((&self.bucket, self.incarnation_id))
}
fn ensure_valid(&self, bucket: &str) -> Result<()> {
if self.bucket != bucket {
return Err(Error::other("bucket metadata mutation guard does not match bucket"));
@@ -674,6 +680,29 @@ async fn acquire_config_write_guard_for_incarnation(
sys: Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
expected_incarnation_id: Option<Uuid>,
) -> Result<BucketMetadataMutationGuard> {
acquire_config_write_guard_with_migration(sys, bucket, expected_incarnation_id, true).await
}
/// Scanner probes must not create an incarnation to make a capability available.
pub async fn acquire_scanner_bucket_incarnation_fence(
bucket: &str,
expected_incarnation_id: Uuid,
expected_owner_id: Uuid,
) -> Result<BucketMetadataMutationGuard> {
super::utils::check_valid_bucket_name(bucket)?;
let sys = get_bucket_metadata_sys()?;
if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() {
return Err(Error::other("scanner bucket incarnation owner does not match"));
}
acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
}
async fn acquire_config_write_guard_with_migration(
sys: Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
expected_incarnation_id: Option<Uuid>,
migrate: bool,
) -> Result<BucketMetadataMutationGuard> {
let metadata_sys = sys.read().await.clone();
let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?;
@@ -681,13 +710,15 @@ async fn acquire_config_write_guard_for_incarnation(
// Legacy buckets are migrated while the lifecycle fence prevents a
// same-name replacement. The second read under the write transaction is
// the CAS source of truth for the actual rewrite.
await_bucket_namespace_operation(
Some(&lifecycle_guard),
bucket,
"bucket config incarnation migration",
metadata_sys.get_bucket_incarnation_id(bucket),
)
.await?;
if migrate {
await_bucket_namespace_operation(
Some(&lifecycle_guard),
bucket,
"bucket config incarnation migration",
metadata_sys.get_bucket_incarnation_id(bucket),
)
.await?;
}
let transaction_guard = await_bucket_namespace_operation(
Some(&lifecycle_guard),
bucket,
@@ -3176,6 +3207,82 @@ mod tests {
);
}
#[tokio::test]
async fn scoped_dirty_usage_incarnation_probe_does_not_migrate_legacy_metadata() {
let (dirs, store) = isolated_store_over_temp_disks().await;
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
let bucket = "scoped-ack-legacy";
for dir in &dirs {
std::fs::create_dir_all(dir.path().join(bucket)).expect("create legacy bucket");
}
let mut metadata = BucketMetadata::new(bucket);
metadata.bucket_incarnation_id = Uuid::nil();
sys.read()
.await
.persist_and_set(metadata)
.await
.expect("persist legacy metadata");
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(Uuid::new_v4()), false)
.await
.is_err()
);
assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
assert!(
sys.read()
.await
.get_config_from_disk(bucket)
.await
.expect("read metadata")
.bucket_incarnation_id
.is_nil()
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn scoped_dirty_usage_incarnation_rejects_deleted_and_recreated_bucket() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
let bucket = "scoped-ack-recreated";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let old = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
let guard = acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.expect("trusted incarnation fence");
assert_eq!(guard.checked_bucket_incarnation().expect("valid fences"), (bucket, old));
drop(guard);
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete bucket");
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.is_err()
);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("recreate bucket");
let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation");
assert_ne!(old, new);
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.is_err()
);
assert!(
acquire_config_write_guard_with_migration(sys, bucket, Some(new), false)
.await
.is_ok()
);
}
#[tokio::test]
async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
crates/ecstore/src/cluster/rpc/client.rs
+19
View File
@@ -30,6 +30,7 @@ use rustfs_protos::{
ChannelClass, create_new_channel, get_channel_for_class,
proto_gen::node_service::{
heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient,
scanner_control_service_client::ScannerControlServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
},
};
@@ -60,6 +61,24 @@ pub async fn node_service_time_out_client(
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
}
pub(crate) async fn scanner_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> crate::error::Result<ScannerControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr)
.await
.map_err(|err| crate::error::Error::other(err.to_string()))?,
};
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(limit)
.max_encoding_message_size(limit))
}
pub async fn heal_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+47
View File
@@ -2050,6 +2050,53 @@ impl PeerRestClient {
.await
}
/// Probe only: scoped ACK production requires a durable per-bucket proof.
pub async fn scanner_scoped_dirty_usage_capability(
&self,
owner_id: String,
instance_id: String,
entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
) -> Result<bool> {
use rustfs_protos::scoped_dirty_usage::*;
let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
owner_id,
instance_id,
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
probe_only: true,
entries,
};
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
self.finalize_result(
async {
let mut client = super::client::scanner_control_time_out_client(
&self.grid_host,
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
)
.await?;
let mut request = Request::new(payload.clone());
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|| response.owner_id != payload.owner_id
|| response.instance_id != payload.instance_id
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|| response.cleared != 0
{
return Err(Error::other("scoped dirty usage capability response does not match request"));
}
Ok(response.supported)
}
.await,
)
.await
}
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
let result = self
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
crates/ecstore/src/disk/local.rs
+997
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/set_disk/core/io_primitives.rs
+21 -369
View File
@@ -59,27 +59,34 @@ use super::super::{
send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
};
#[cfg(test)]
pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision;
pub(in crate::set_disk) use super::metadata_quorum::{
MetadataQuorumAccumulator, is_metadata_fanout_ignored_error, metadata_early_stop_candidate_matches,
};
#[cfg(test)]
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
#[cfg(test)]
use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH;
#[cfg(test)]
use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD;
use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY,
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING,
GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT,
GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled,
record_get_stage_duration_if_enabled,
};
#[cfg(test)]
use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR,
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND,
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META,
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE,
GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN,
GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, record_get_stage_duration_if_enabled,
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
};
#[cfg(test)]
use crate::disk::CHECK_PART_FILE_NOT_FOUND;
@@ -690,324 +697,6 @@ impl MetadataFanoutDiagnostics {
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
pub(in crate::set_disk) reason: &'static str,
}
#[derive(Clone, Debug)]
pub(in crate::set_disk) struct MetadataQuorumAccumulator {
pub(in crate::set_disk) total_disks: usize,
pub(in crate::set_disk) default_parity_count: usize,
pub(in crate::set_disk) allow_early_stop: bool,
pub(in crate::set_disk) valid_responses: usize,
pub(in crate::set_disk) not_found_responses: usize,
pub(in crate::set_disk) version_not_found_responses: usize,
pub(in crate::set_disk) ignored_errors: usize,
pub(in crate::set_disk) hard_errors: usize,
pub(in crate::set_disk) candidate: Option<FileInfo>,
pub(in crate::set_disk) candidate_votes: usize,
// Bitset of shard indexes whose metadata matches the candidate. Erasure
// layouts are capped at 16 shards, so this stays allocation-free on the
// GET metadata hot path.
candidate_shard_mask: u16,
pub(in crate::set_disk) conflicting_metadata: bool,
pub(in crate::set_disk) delete_marker_seen: bool,
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
pub(in crate::set_disk) delete_marker_votes: usize,
pub(in crate::set_disk) requested_version_id: String,
pub(in crate::set_disk) matching_version_votes: usize,
}
impl MetadataQuorumAccumulator {
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
Self {
total_disks,
default_parity_count,
allow_early_stop,
valid_responses: 0,
not_found_responses: 0,
version_not_found_responses: 0,
ignored_errors: 0,
hard_errors: 0,
candidate: None,
candidate_votes: 0,
candidate_shard_mask: 0,
conflicting_metadata: false,
delete_marker_seen: false,
delete_marker_candidates: Vec::new(),
delete_marker_votes: 0,
requested_version_id: String::new(),
matching_version_votes: 0,
}
}
pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
self.requested_version_id = version_id.to_string();
self
}
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
self.observe_file_info_with_index(None, file_info);
}
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
self.observe_file_info_with_index(Some(disk_index), file_info);
}
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
if !file_info_is_valid_for_metadata(file_info) {
self.hard_errors = self.hard_errors.saturating_add(1);
return;
}
self.valid_responses = self.valid_responses.saturating_add(1);
// Track version match for versioned requests
if !self.requested_version_id.is_empty()
&& let Some(ref vid) = file_info.version_id
&& vid.to_string() == self.requested_version_id
{
self.matching_version_votes = self.matching_version_votes.saturating_add(1);
}
if file_info.is_canonical_delete_marker() {
self.delete_marker_seen = true;
if let Some((_, votes)) = self
.delete_marker_candidates
.iter_mut()
.find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
{
*votes = votes.saturating_add(1);
} else {
self.delete_marker_candidates.push((file_info.clone(), 1));
}
self.delete_marker_votes = self
.delete_marker_candidates
.iter()
.map(|(_, votes)| *votes)
.max()
.unwrap_or_default();
self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
return;
}
match &self.candidate {
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
self.candidate_votes = self.candidate_votes.saturating_add(1);
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
Some(_) => {
self.conflicting_metadata = true;
}
None => {
self.candidate = Some(file_info.clone());
self.candidate_votes = 1;
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
}
}
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
return None;
}
Some(1u16 << (erasure_index - 1))
}
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
self.candidate_read_reserve_target()
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
}
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
let candidate = self.candidate.as_ref()?;
Some(
candidate
.erasure
.data_blocks
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
)
}
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
match err {
DiskError::FileNotFound | DiskError::VolumeNotFound => {
self.not_found_responses = self.not_found_responses.saturating_add(1);
}
DiskError::FileVersionNotFound => {
self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
}
_ if is_metadata_fanout_ignored_error(err) => {
self.ignored_errors = self.ignored_errors.saturating_add(1);
}
_ => {
self.hard_errors = self.hard_errors.saturating_add(1);
}
}
}
pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.delete_marker_votes >= self.default_write_quorum() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
});
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self
.candidate
.as_ref()
.and_then(|candidate| self.candidate_latest_quorum(candidate))
.is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
{
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
});
}
None
}
/// Check if a versioned request can early-stop because the requested
/// version_id has reached quorum across disks.
pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.requested_version_id.is_empty() {
return None;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self.matching_version_votes >= self.read_quorum_for_version() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
});
}
None
}
pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
if !self.allow_early_stop {
return false;
}
if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
return true;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return false;
}
if !self.requested_version_id.is_empty()
&& self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
{
return true;
}
match &self.candidate {
Some(candidate) => self
.candidate_latest_quorum(candidate)
.is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
None => pending >= self.default_write_quorum(),
}
}
/// Compute the read quorum threshold for version-aware early-stop.
/// Uses `total_disks / 2` (like `missing_response_quorum`) when
/// `default_parity_count` is set, otherwise requires all disks.
pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
self.missing_response_quorum()
}
pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
if !self.allow_early_stop {
return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
}
if self.conflicting_metadata {
return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
}
if self.delete_marker_seen {
return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
}
let missing_response_quorum = self.missing_response_quorum();
if self.version_not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
}
if self.not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
}
if self.hard_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_ERROR;
}
if self.ignored_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
}
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
}
pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
if self.default_parity_count == 0 {
return Some(self.total_disks);
}
if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
return None;
}
let data_blocks = candidate.erasure.data_blocks;
Some(if data_blocks == candidate.erasure.parity_blocks {
data_blocks.saturating_add(1)
} else {
data_blocks
})
}
pub(crate) fn default_write_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
return self.total_disks;
}
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
if data_blocks == self.default_parity_count {
data_blocks.saturating_add(1)
} else {
data_blocks
}
}
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
self.total_disks
} else {
self.total_disks / 2
}
}
}
#[derive(Clone, Debug)]
pub(in crate::set_disk) enum MetadataCacheLookup {
Hit(Arc<GetObjectMetadataCacheEntry>),
@@ -1015,39 +704,6 @@ pub(in crate::set_disk) enum MetadataCacheLookup {
RejectedInsufficientQuorum,
}
pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
left.volume == right.volume
&& left.name == right.name
&& left.version_id == right.version_id
&& left.is_latest == right.is_latest
&& left.deleted == right.deleted
&& left.mark_deleted == right.mark_deleted
&& left.transition_status == right.transition_status
&& left.transitioned_objname == right.transitioned_objname
&& left.transition_tier == right.transition_tier
&& left.transition_version_id == right.transition_version_id
&& left.transition_version == right.transition_version
&& left.transition_version_state == right.transition_version_state
&& left.expire_restored == right.expire_restored
&& left.size == right.size
&& left.mod_time == right.mod_time
&& left.mode == right.mode
&& left.written_by_version == right.written_by_version
&& left.metadata == right.metadata
&& left.replication_state_internal == right.replication_state_internal
&& left.parts == right.parts
&& left.checksum == right.checksum
&& left.versioned == right.versioned
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.data_dir == right.data_dir
&& left.erasure.algorithm == right.erasure.algorithm
&& left.erasure.data_blocks == right.erasure.data_blocks
&& left.erasure.parity_blocks == right.erasure.parity_blocks
&& left.erasure.block_size == right.erasure.block_size
&& left.erasure.distribution == right.erasure.distribution
}
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
bucket: &str,
object: &str,
@@ -1247,10 +903,6 @@ pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) ->
}
}
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
}
pub(in crate::set_disk) fn is_confirmed_missing_part_error(err: Option<&str>) -> bool {
let Some(err) = err else {
return false;
crates/ecstore/src/set_disk/core/metadata_quorum.rs
+385
View File
@@ -0,0 +1,385 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Pure metadata quorum and early-stop decisions for `SetDisks` reads.
//!
//! Disk scheduling, coalescing, cancellation, and late shard materialization
//! remain with their existing owners; this module only classifies observations.
use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
};
use crate::disk::error::DiskError;
use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS;
use crate::set_disk::file_info_is_valid_for_metadata;
use rustfs_filemeta::FileInfo;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
pub(in crate::set_disk) reason: &'static str,
}
#[derive(Clone, Debug)]
pub(in crate::set_disk) struct MetadataQuorumAccumulator {
pub(in crate::set_disk) total_disks: usize,
pub(in crate::set_disk) default_parity_count: usize,
pub(in crate::set_disk) allow_early_stop: bool,
pub(in crate::set_disk) valid_responses: usize,
pub(in crate::set_disk) not_found_responses: usize,
pub(in crate::set_disk) version_not_found_responses: usize,
pub(in crate::set_disk) ignored_errors: usize,
pub(in crate::set_disk) hard_errors: usize,
pub(in crate::set_disk) candidate: Option<FileInfo>,
pub(in crate::set_disk) candidate_votes: usize,
// Bitset of shard indexes whose metadata matches the candidate. Erasure
// layouts are capped at 16 shards, so this stays allocation-free on the
// GET metadata hot path.
candidate_shard_mask: u16,
pub(in crate::set_disk) conflicting_metadata: bool,
pub(in crate::set_disk) delete_marker_seen: bool,
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
pub(in crate::set_disk) delete_marker_votes: usize,
pub(in crate::set_disk) requested_version_id: String,
pub(in crate::set_disk) matching_version_votes: usize,
}
impl MetadataQuorumAccumulator {
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
Self {
total_disks,
default_parity_count,
allow_early_stop,
valid_responses: 0,
not_found_responses: 0,
version_not_found_responses: 0,
ignored_errors: 0,
hard_errors: 0,
candidate: None,
candidate_votes: 0,
candidate_shard_mask: 0,
conflicting_metadata: false,
delete_marker_seen: false,
delete_marker_candidates: Vec::new(),
delete_marker_votes: 0,
requested_version_id: String::new(),
matching_version_votes: 0,
}
}
pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
self.requested_version_id = version_id.to_string();
self
}
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
self.observe_file_info_with_index(None, file_info);
}
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
self.observe_file_info_with_index(Some(disk_index), file_info);
}
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
if !file_info_is_valid_for_metadata(file_info) {
self.hard_errors = self.hard_errors.saturating_add(1);
return;
}
self.valid_responses = self.valid_responses.saturating_add(1);
// Track version match for versioned requests
if !self.requested_version_id.is_empty()
&& let Some(ref vid) = file_info.version_id
&& vid.to_string() == self.requested_version_id
{
self.matching_version_votes = self.matching_version_votes.saturating_add(1);
}
if file_info.is_canonical_delete_marker() {
self.delete_marker_seen = true;
if let Some((_, votes)) = self
.delete_marker_candidates
.iter_mut()
.find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
{
*votes = votes.saturating_add(1);
} else {
self.delete_marker_candidates.push((file_info.clone(), 1));
}
self.delete_marker_votes = self
.delete_marker_candidates
.iter()
.map(|(_, votes)| *votes)
.max()
.unwrap_or_default();
self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
return;
}
match &self.candidate {
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
self.candidate_votes = self.candidate_votes.saturating_add(1);
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
Some(_) => {
self.conflicting_metadata = true;
}
None => {
self.candidate = Some(file_info.clone());
self.candidate_votes = 1;
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
}
}
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
return None;
}
Some(1u16 << (erasure_index - 1))
}
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
self.candidate_read_reserve_target()
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
}
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
let candidate = self.candidate.as_ref()?;
Some(
candidate
.erasure
.data_blocks
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
)
}
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
match err {
DiskError::FileNotFound | DiskError::VolumeNotFound => {
self.not_found_responses = self.not_found_responses.saturating_add(1);
}
DiskError::FileVersionNotFound => {
self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
}
_ if is_metadata_fanout_ignored_error(err) => {
self.ignored_errors = self.ignored_errors.saturating_add(1);
}
_ => {
self.hard_errors = self.hard_errors.saturating_add(1);
}
}
}
pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.delete_marker_votes >= self.default_write_quorum() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
});
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self
.candidate
.as_ref()
.and_then(|candidate| self.candidate_latest_quorum(candidate))
.is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
{
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
});
}
None
}
/// Check if a versioned request can early-stop because the requested
/// version_id has reached quorum across disks.
pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.requested_version_id.is_empty() {
return None;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self.matching_version_votes >= self.read_quorum_for_version() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
});
}
None
}
pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
if !self.allow_early_stop {
return false;
}
if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
return true;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return false;
}
if !self.requested_version_id.is_empty()
&& self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
{
return true;
}
match &self.candidate {
Some(candidate) => self
.candidate_latest_quorum(candidate)
.is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
None => pending >= self.default_write_quorum(),
}
}
/// Compute the read quorum threshold for version-aware early-stop.
/// Uses `total_disks / 2` (like `missing_response_quorum`) when
/// `default_parity_count` is set, otherwise requires all disks.
pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
self.missing_response_quorum()
}
pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
if !self.allow_early_stop {
return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
}
if self.conflicting_metadata {
return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
}
if self.delete_marker_seen {
return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
}
let missing_response_quorum = self.missing_response_quorum();
if self.version_not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
}
if self.not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
}
if self.hard_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_ERROR;
}
if self.ignored_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
}
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
}
pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
if self.default_parity_count == 0 {
return Some(self.total_disks);
}
if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
return None;
}
let data_blocks = candidate.erasure.data_blocks;
Some(if data_blocks == candidate.erasure.parity_blocks {
data_blocks.saturating_add(1)
} else {
data_blocks
})
}
pub(crate) fn default_write_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
return self.total_disks;
}
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
if data_blocks == self.default_parity_count {
data_blocks.saturating_add(1)
} else {
data_blocks
}
}
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
self.total_disks
} else {
self.total_disks / 2
}
}
}
pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
left.volume == right.volume
&& left.name == right.name
&& left.version_id == right.version_id
&& left.is_latest == right.is_latest
&& left.deleted == right.deleted
&& left.mark_deleted == right.mark_deleted
&& left.transition_status == right.transition_status
&& left.transitioned_objname == right.transitioned_objname
&& left.transition_tier == right.transition_tier
&& left.transition_version_id == right.transition_version_id
&& left.transition_version == right.transition_version
&& left.transition_version_state == right.transition_version_state
&& left.expire_restored == right.expire_restored
&& left.size == right.size
&& left.mod_time == right.mod_time
&& left.mode == right.mode
&& left.written_by_version == right.written_by_version
&& left.metadata == right.metadata
&& left.replication_state_internal == right.replication_state_internal
&& left.parts == right.parts
&& left.checksum == right.checksum
&& left.versioned == right.versioned
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.data_dir == right.data_dir
&& left.erasure.algorithm == right.erasure.algorithm
&& left.erasure.data_blocks == right.erasure.data_blocks
&& left.erasure.parity_blocks == right.erasure.parity_blocks
&& left.erasure.block_size == right.erasure.block_size
&& left.erasure.distribution == right.erasure.distribution
}
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
}
crates/ecstore/src/set_disk/core/mod.rs
+1
View File
@@ -18,3 +18,4 @@
//! duplicating read/write/erasure logic.
pub(crate) mod io_primitives;
mod metadata_quorum;
crates/heal/src/heal/manager.rs
+60 -12
View File
@@ -45,6 +45,11 @@ use tracing::{debug, error, info, warn};
use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
// Each cache includes alias tokens in its count and byte budget. Eviction
// removes every token sharing a snapshot; neither cache retains repair state.
const MAX_COMPLETED_HEAL_TOKENS: usize = 1024;
const MAX_COMPLETED_HEAL_BYTES: usize = 64 * 1024 * 1024;
const MAX_COMPLETED_HEAL_RESULT_BYTES: usize = 1024 * 1024;
const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again";
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
@@ -180,6 +185,8 @@ fn record_displaced_terminal(
request: &HealRequest,
) -> Arc<CompletedHealStatus> {
let terminal = Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(),
status: HealTaskStatus::Failed {
error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"),
@@ -193,6 +200,7 @@ fn record_displaced_terminal(
let mut terminals = lock_displaced_terminals(registry);
prune_completed_heal_statuses(&mut terminals);
terminals.insert(request.id.clone(), Arc::clone(&terminal));
prune_completed_heal_statuses(&mut terminals);
terminal
}
@@ -209,9 +217,15 @@ async fn remove_displaced_task_aliases(
.collect::<Vec<_>>();
let mut displaced_terminals = lock_displaced_terminals(terminals);
prune_completed_heal_statuses(&mut displaced_terminals);
for alias_id in alias_ids {
displaced_terminals.insert(alias_id, Arc::clone(terminal));
if displaced_terminals
.get(task_id)
.is_some_and(|current| Arc::ptr_eq(current, terminal))
{
for alias_id in alias_ids {
displaced_terminals.insert(alias_id, Arc::clone(terminal));
}
}
prune_completed_heal_statuses(&mut displaced_terminals);
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
}
@@ -222,6 +236,36 @@ async fn remove_task_aliases_for_task(registry: &Arc<Mutex<HashMap<String, HealT
.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
}
// Callers hold active ownership until publication. Lock order is active ->
// retrying (when needed) -> aliases -> completed; queries release aliases
// before looking up active state. Publishing aliases before removing their
// mapping keeps both an already-resolved token and a new lookup valid.
async fn publish_completed_heal(
completed_heals: &Mutex<HashMap<String, Arc<CompletedHealStatus>>>,
task_aliases: &Mutex<HashMap<String, HealTaskAlias>>,
task_id: &str,
completed: CompletedHealStatus,
terminal: bool,
) {
let completed = Arc::new(completed);
completed.retained_bytes();
let mut aliases = task_aliases.lock().await;
let mut retained = completed_heals.lock().await;
if let Some(previous) = retained.get(task_id).cloned() {
for entry in retained.values_mut().filter(|entry| Arc::ptr_eq(entry, &previous)) {
*entry = Arc::clone(&completed);
}
}
retained.insert(task_id.to_owned(), Arc::clone(&completed));
if terminal {
for (alias_id, _) in aliases.iter().filter(|(_, alias)| alias.task_id == task_id) {
retained.insert(alias_id.clone(), Arc::clone(&completed));
}
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
}
prune_completed_heal_statuses(&mut retained);
}
#[derive(Debug, Clone)]
pub struct HealTaskReport {
pub status: HealTaskStatus,
@@ -268,7 +312,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
let result_items = match since {
None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(),
Some(cursor) => {
if cursor + 1 < completed.min_seq {
if cursor.saturating_add(1) < completed.min_seq {
lagged = true;
}
completed
@@ -283,7 +327,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
status: completed.status.clone(),
result_items,
result_items_truncated: completed.result_items_truncated || lagged,
progress: None,
progress: completed.progress.clone(),
next_seq: completed.next_seq,
min_seq: completed.min_seq,
}
@@ -1847,14 +1891,14 @@ impl HealManager {
pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
let canonical_task_id = self.canonical_task_id(task_id).await;
let active_heals = self.active_heals.lock().await;
if let Some(task) = active_heals.get(&canonical_task_id) {
Ok(task.get_progress().await)
} else {
Err(Error::TaskNotFound {
task_id: task_id.to_string(),
})
}
let progress = match self.lookup_task_state(&canonical_task_id, None).await {
TaskStateLookup::Active(task) => Some(task.get_progress().await),
TaskStateLookup::Completed(completed) => completed.progress.clone(),
_ => None,
};
progress.ok_or_else(|| Error::TaskNotFound {
task_id: task_id.to_string(),
})
}
/// Cancel task
@@ -1864,6 +1908,8 @@ impl HealManager {
let mut active_heals = self.active_heals.lock().await;
if let Some(task) = active_heals.get(&canonical_task_id) {
task.cancel().await?;
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await;
active_heals.remove(&canonical_task_id);
publish_active_heal_count(&active_heals);
info!(
@@ -1940,6 +1986,8 @@ impl HealManager {
for task_id in &task_ids {
if let Some(task) = active_heals.get(task_id) {
task.cancel().await?;
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await;
}
active_heals.remove(task_id);
cancelled += 1;
crates/heal/src/heal/manager/queue.rs
+129
View File
@@ -82,6 +82,8 @@ pub(super) enum QueuePushOutcome {
pub(super) struct CompletedHealStatus {
pub(super) heal_type: HealType,
pub(super) status: HealTaskStatus,
pub(super) progress: Option<HealProgress>,
pub(super) retained_bytes: std::sync::OnceLock<usize>,
pub(super) result_items_truncated: bool,
pub(super) completed_at: SystemTime,
/// Sequence-stamped retained window, archived with the completion so
@@ -92,6 +94,133 @@ pub(super) struct CompletedHealStatus {
pub(super) min_seq: u64,
}
impl CompletedHealStatus {
// Account for owned capacities, including nested drive arrays. Aliases
// conservatively charge the shared allocation again, keeping both token
// count and retained payload bounded without a second ownership index.
pub(super) fn retained_bytes(&self) -> usize {
*self.retained_bytes.get_or_init(|| self.measure_retained_bytes())
}
fn measure_retained_bytes(&self) -> usize {
let mut bytes = size_of::<Self>();
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
match &self.heal_type {
HealType::Cluster => {}
HealType::Bucket { bucket } => add(bucket.capacity()),
HealType::Object {
bucket,
object,
version_id,
}
| HealType::ECDecode {
bucket,
object,
version_id,
} => {
add(bucket.capacity());
add(object.capacity());
add(version_id.as_ref().map_or(0, String::capacity));
}
HealType::Prefix { bucket, prefix } => {
add(bucket.capacity());
add(prefix.capacity());
}
HealType::Metadata { bucket, object } => {
add(bucket.capacity());
add(object.capacity());
}
HealType::ErasureSet { buckets, set_disk_id } => {
add(buckets.capacity().saturating_mul(size_of::<String>()));
for bucket in buckets {
add(bucket.capacity());
}
add(set_disk_id.capacity());
}
}
if let HealTaskStatus::Failed { error } | HealTaskStatus::Retrying { error, .. } = &self.status {
add(error.capacity());
}
add(self
.progress
.as_ref()
.and_then(|progress| progress.current_object.as_ref())
.map_or(0, String::capacity));
add(self.seqed_items.capacity().saturating_mul(size_of::<(u64, HealResultItem)>()));
for (_, item) in &self.seqed_items {
add(Self::result_item_heap_bytes(item));
}
bytes
}
fn result_item_heap_bytes(item: &HealResultItem) -> usize {
let mut bytes = 0usize;
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
for value in [
&item.heal_item_type,
&item.bucket,
&item.object,
&item.version_id,
&item.detail,
] {
add(value.capacity());
}
for infos in [&item.before, &item.after] {
add(infos
.drives
.capacity()
.saturating_mul(size_of::<rustfs_madmin::heal_commands::HealDriveInfo>()));
for drive in &infos.drives {
add(drive.uuid.capacity());
add(drive.endpoint.capacity());
add(drive.state.capacity());
}
}
bytes
}
pub(super) fn bound_result_window(&mut self) {
let mut bytes = 0usize;
let retained = self
.seqed_items
.iter()
.rev()
.take_while(|(_, item)| {
bytes = bytes
.saturating_add(size_of::<(u64, HealResultItem)>())
.saturating_add(Self::result_item_heap_bytes(item));
bytes <= MAX_COMPLETED_HEAL_RESULT_BYTES
})
.count();
let truncated = retained < self.seqed_items.len();
if truncated {
self.seqed_items.drain(..self.seqed_items.len() - retained);
self.seqed_items.shrink_to_fit();
self.min_seq = self.seqed_items.first().map_or(self.next_seq, |(seq, _)| *seq);
self.result_items_truncated = true;
self.retained_bytes.take();
}
}
pub(super) async fn snapshot(task: &HealTask, status: HealTaskStatus) -> Self {
let seqed_items = task.get_seqed_result_items().await;
let (next_seq, min_seq) = task.result_seq_cursors();
let mut snapshot = Self {
heal_type: task.heal_type.clone(),
status,
progress: Some(task.get_progress().await),
retained_bytes: std::sync::OnceLock::new(),
result_items_truncated: task.result_items_truncated(),
completed_at: SystemTime::now(),
seqed_items,
next_seq,
min_seq,
};
snapshot.bound_result_window();
snapshot
}
}
#[derive(Debug, Clone)]
pub(super) struct HealTaskAlias {
pub(super) task_id: String,
crates/heal/src/heal/manager/scheduler.rs
+75 -39
View File
@@ -264,7 +264,7 @@ impl HealManager {
error: error.clone(),
retry_attempt: request.retry_attempts,
});
let retry_request_for_queue = retry_request;
let mut retry_request_for_queue = retry_request;
let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new());
if retry_request_for_queue.is_none() {
replacement_recovery_anchors_clone
@@ -272,7 +272,35 @@ impl HealManager {
.unwrap_or_else(|poisoned| poisoned.into_inner())
.remove(&task_id);
}
let mut completed_status = match retry_request_for_status {
Some(status) => status,
None => task.get_status().await,
};
let mut completed_status_entry = CompletedHealStatus::snapshot(&task, completed_status.clone()).await;
let completed_progress = task.get_progress().await;
#[cfg(test)]
tests::pause_completed_retention_before_publish(&task_id, &completed_status).await;
let mut active_heals_guard = active_heals_clone.lock().await;
let owns_completion = active_heals_guard.contains_key(&task_id);
let cancelled_completion = if owns_completion {
false
} else {
// Cancellation can win while a finished worker waits
// for active ownership. It must not resurrect a retry
// or replace an acknowledged cancellation with success.
retry_request_for_queue = None;
completed_heals_clone
.lock()
.await
.get(&task_id)
.is_some_and(|completed| completed.status == HealTaskStatus::Cancelled)
};
if cancelled_completion {
completed_status = HealTaskStatus::Cancelled;
completed_status_entry.status = HealTaskStatus::Cancelled;
}
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
// Keep retry ownership continuous: status snapshots acquire
// these locks in the same active -> retrying order.
let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
@@ -295,6 +323,16 @@ impl HealManager {
} else {
None
};
if owns_completion || cancelled_completion {
publish_completed_heal(
&completed_heals_clone,
&task_aliases_clone,
&task_id,
completed_status_entry,
terminal_completion,
)
.await;
}
let completed_task = active_heals_guard.remove(&task_id);
if let Some(completed_task) = completed_task.as_ref() {
publish_active_heal_count(&active_heals_guard);
@@ -304,33 +342,10 @@ impl HealManager {
drop(retrying_heals_guard.take());
drop(active_heals_guard);
if let Some(completed_task) = completed_task {
let completed_status = if let Some(status) = retry_request_for_status {
status
} else {
completed_task.get_status().await
};
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
let completed_progress = completed_task.get_progress().await;
// Single snapshot of the retained window: the task is
// finished and already off the active map, so there is
// no concurrent writer to race with.
let seqed_items = completed_task.get_seqed_result_items().await;
let (next_seq, min_seq) = completed_task.result_seq_cursors();
let completed_status_entry = CompletedHealStatus {
heal_type: completed_task.heal_type.clone(),
status: completed_status.clone(),
result_items_truncated: completed_task.result_items_truncated(),
completed_at: SystemTime::now(),
seqed_items,
next_seq,
min_seq,
};
let mut completed_heals_guard = completed_heals_clone.lock().await;
prune_completed_heal_statuses(&mut completed_heals_guard);
completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry));
drop(completed_heals_guard);
#[cfg(test)]
tests::pause_completed_retention_handoff(&task_id).await;
if completed_task.is_some() {
// update statistics
let mut stats = statistics_clone.write().await;
match completed_status {
@@ -352,10 +367,6 @@ impl HealManager {
} else {
release_mrf_repair_notice_targets(notice_targets);
}
task_aliases_clone
.lock()
.await
.retain(|alias_id, alias| alias_id != &task_id && alias.task_id != task_id);
}
}
@@ -718,17 +729,42 @@ pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
}
pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>) {
let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else {
return;
};
prune_completed_heal_statuses_at(completed_heals, SystemTime::now());
}
pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>, now: SystemTime) {
completed_heals.retain(|_, completed| {
completed
.completed_at
.duration_since(SystemTime::UNIX_EPOCH)
.map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION)
now.duration_since(completed.completed_at)
.map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
.unwrap_or(false)
});
let entry_bytes = |key: &String, value: &Arc<CompletedHealStatus>| {
key.capacity()
.saturating_add(size_of::<(String, Arc<CompletedHealStatus>)>())
.saturating_add(value.retained_bytes())
};
let mut bytes = completed_heals
.iter()
.fold(0usize, |total, (key, value)| total.saturating_add(entry_bytes(key, value)));
while completed_heals.len() > MAX_COMPLETED_HEAL_TOKENS || bytes > MAX_COMPLETED_HEAL_BYTES {
let Some(oldest) = completed_heals
.iter()
.min_by(|(left_id, left), (right_id, right)| {
left.completed_at.cmp(&right.completed_at).then_with(|| left_id.cmp(right_id))
})
.map(|(_, value)| Arc::clone(value))
else {
break;
};
completed_heals.retain(|key, value| {
if Arc::ptr_eq(value, &oldest) {
bytes = bytes.saturating_sub(entry_bytes(key, value));
false
} else {
true
}
});
}
}
pub(super) fn can_schedule_request(
crates/heal/src/heal/manager/tests.rs
+348 -3
View File
@@ -101,6 +101,326 @@ async fn process_manager_queue_once(manager: &HealManager) {
struct MockStorage;
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
CompletedHealStatus {
heal_type: HealType::Cluster,
status: HealTaskStatus::Completed,
progress: Some(HealProgress {
objects_scanned: 9,
objects_healed: 8,
objects_failed: 1,
..Default::default()
}),
retained_bytes: std::sync::OnceLock::new(),
result_items_truncated: false,
completed_at,
seqed_items: vec![(3, HealResultItem::default()), (4, HealResultItem::default())],
next_seq: 5,
min_seq: 3,
}
}
#[test]
fn completed_retention_cursor_boundaries_preserve_progress() {
let completed = completed_retention_fixture(SystemTime::now());
for (cursor, count, lagged) in [
(0, 2, true),
(1, 2, true),
(2, 2, false),
(3, 1, false),
(4, 0, false),
(5, 0, false),
(u64::MAX, 0, false),
] {
let report = completed_task_report(&completed, Some(cursor));
assert_eq!(report.result_items.len(), count, "cursor={cursor}");
assert_eq!(report.result_items_truncated, lagged, "cursor={cursor}");
assert_eq!(report.progress, completed.progress);
assert_eq!((report.next_seq, report.min_seq), (5, 3));
}
assert_eq!(completed_task_report(&completed, None).result_items.len(), 2);
}
#[tokio::test]
async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() {
let manager = HealManager::new(Arc::new(MockStorage), None);
let request = HealRequest::bucket("bucket".to_string());
manager.insert_task_alias("alias", &request.id).await;
let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
lock_displaced_terminals(&manager.displaced_terminals).remove(&request.id);
remove_displaced_task_aliases(&manager.task_aliases, &manager.displaced_terminals, &request.id, &terminal).await;
for token in [&request.id, &"alias".to_string()] {
assert!(matches!(manager.get_task_report(token).await, Err(Error::TaskNotFound { .. })));
}
assert!(manager.task_aliases.lock().await.is_empty());
assert!(lock_displaced_terminals(&manager.displaced_terminals).is_empty());
}
#[test]
fn completed_retention_count_ttl_and_alias_eviction_are_bounded() {
let now = SystemTime::now();
let mut entries = HashMap::new();
let oldest = Arc::new(completed_retention_fixture(now - KEEP_HEAL_TASK_STATUS_DURATION));
entries.insert("oldest".to_string(), Arc::clone(&oldest));
entries.insert("oldest-alias".to_string(), Arc::clone(&oldest));
for index in 2..MAX_COMPLETED_HEAL_TOKENS {
entries.insert(format!("task-{index}"), Arc::new(completed_retention_fixture(now)));
}
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS);
entries.insert("cap-plus-one".to_string(), Arc::new(completed_retention_fixture(now)));
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1);
assert!(!entries.contains_key("oldest"));
assert!(!entries.contains_key("oldest-alias"));
entries.clear();
entries.insert("ttl-boundary".to_string(), oldest);
entries.insert(
"expired".to_string(),
Arc::new(completed_retention_fixture(
now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1),
)),
);
entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1))));
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), 1);
assert!(entries.contains_key("ttl-boundary"));
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1));
assert!(entries.is_empty());
}
#[test]
fn completed_retention_total_byte_cap_and_cap_plus_one() {
let now = SystemTime::now();
let key = "large".to_string();
let mut entry = completed_retention_fixture(now);
let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
entry.retained_bytes.take();
entry.status = HealTaskStatus::Failed {
error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
};
assert_eq!(
entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
MAX_COMPLETED_HEAL_BYTES
);
let mut entries = HashMap::from([(key, Arc::new(entry))]);
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), 1, "exact byte cap remains retained");
let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
over.retained_bytes.take();
if let HealTaskStatus::Failed { error } = &mut over.status {
*error = "x".repeat(error.len() + 1);
}
entries.insert("large".to_string(), Arc::new(over));
prune_completed_heal_statuses_at(&mut entries, now);
assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
}
#[tokio::test]
async fn completed_retention_large_window_keeps_cursors_and_progress() {
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), Arc::new(MockStorage));
let mut snapshot = completed_retention_fixture(SystemTime::now());
snapshot.seqed_items[0].1.detail = "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES);
snapshot.bound_result_window();
assert_eq!(snapshot.seqed_items.len(), 1);
assert_eq!((snapshot.min_seq, snapshot.next_seq), (4, 5));
assert!(snapshot.result_items_truncated);
assert!(snapshot.retained_bytes() < MAX_COMPLETED_HEAL_RESULT_BYTES);
let report = completed_task_report(&snapshot, Some(0));
assert_eq!(report.progress.expect("progress retained").objects_scanned, 9);
assert!(report.result_items_truncated);
let active_max = task.get_result_items_since(Some(u64::MAX)).await;
assert!(active_max.items.is_empty());
assert!(!active_max.lagged);
}
#[test]
fn completed_retention_result_byte_cap_and_cap_plus_one() {
for extra in [0, 1] {
let mut snapshot = completed_retention_fixture(SystemTime::now());
snapshot.seqed_items = vec![(
4,
HealResultItem {
detail: "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES - size_of::<(u64, HealResultItem)>() + extra),
..Default::default()
},
)];
snapshot.min_seq = 4;
snapshot.bound_result_window();
assert_eq!(snapshot.seqed_items.len(), 1 - extra);
assert_eq!(snapshot.result_items_truncated, extra == 1);
assert_eq!(snapshot.min_seq, if extra == 0 { 4 } else { 5 });
assert_eq!(snapshot.next_seq, 5);
assert_eq!(snapshot.progress.as_ref().expect("progress retained").objects_scanned, 9);
}
}
#[derive(Default)]
struct CompletedRetentionHook {
started: Notify,
execute: Notify,
handoff: Notify,
finish: Notify,
pause_before_publish: bool,
before_publish: Notify,
publish: Notify,
prepared_status: Mutex<Option<HealTaskStatus>>,
}
static COMPLETED_RETENTION_HOOKS: LazyLock<Mutex<HashMap<String, Arc<CompletedRetentionHook>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
pub(super) async fn pause_completed_retention_handoff(task_id: &str) {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
if let Some(hook) = hook {
hook.handoff.notify_one();
hook.finish.notified().await;
}
}
pub(super) async fn pause_completed_retention_before_publish(task_id: &str, status: &HealTaskStatus) {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
if let Some(hook) = hook.filter(|hook| hook.pause_before_publish) {
*hook.prepared_status.lock().await = Some(status.clone());
hook.before_publish.notify_one();
hook.publish.notified().await;
}
}
#[tokio::test]
async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
let bucket = "completed-retention-retry-cancel";
let manager = HealManager::new(Arc::new(MockStorage), None);
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
let task_id = request.id.clone();
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
let alias = duplicate.id.clone();
let hook = Arc::new(CompletedRetentionHook {
pause_before_publish: true,
..Default::default()
});
{
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
hooks.insert(bucket.to_string(), Arc::clone(&hook));
hooks.insert(task_id.clone(), Arc::clone(&hook));
}
manager.submit_heal_request(request).await.expect("admit original");
manager.submit_heal_request(duplicate).await.expect("admit alias");
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
.await
.expect("scheduler starts");
let task = manager.active_heals.lock().await.get(&task_id).cloned().expect("active task");
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
hook.execute.notify_one();
tokio::time::timeout(Duration::from_secs(5), hook.before_publish.notified())
.await
.expect("retry snapshot prepared");
manager.cancel_task(&alias).await.expect("cancel wins active ownership");
assert!(matches!(*hook.prepared_status.lock().await, Some(HealTaskStatus::Retrying { .. })));
hook.publish.notify_one();
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
.await
.expect("scheduler finishes handoff");
for token in [&task_id, &alias] {
let report = manager.get_task_report(token).await.expect("cancelled token retained");
assert_eq!(report.status, HealTaskStatus::Cancelled);
assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
}
assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id));
hook.finish.notify_one();
COMPLETED_RETENTION_HOOKS
.lock()
.await
.retain(|key, _| key != bucket && key != &task_id);
}
#[tokio::test]
async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_handoff() {
for outcome in ["success", "failed", "cancelled"] {
let bucket = format!("completed-retention-{outcome}");
let hook = Arc::new(CompletedRetentionHook::default());
let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None));
let request = HealRequest::object(bucket.clone(), "object".to_string(), None);
let task_id = request.id.clone();
let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None);
let alias = duplicate.id.clone();
{
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
hooks.insert(bucket.clone(), Arc::clone(&hook));
hooks.insert(task_id.clone(), Arc::clone(&hook));
}
manager.submit_heal_request(request).await.expect("admit original");
manager.submit_heal_request(duplicate).await.expect("admit alias");
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
.await
.expect("scheduler reaches storage");
let task = manager
.active_heals
.lock()
.await
.get(&task_id)
.cloned()
.expect("task is active");
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
let before = manager.get_task_report(&alias).await.expect("alias resolves active progress");
assert_eq!(before.progress.as_ref().expect("active progress").objects_scanned, 1);
let poll_manager = Arc::clone(&manager);
let poll_alias = alias.clone();
let stop = CancellationToken::new();
let poll_stop = stop.clone();
let polling = tokio::spawn(async move {
while !poll_stop.is_cancelled() {
let report = poll_manager
.get_task_report(&poll_alias)
.await
.expect("handoff must never return NotFound");
assert!(report.progress.expect("progress never disappears").objects_scanned >= 1);
tokio::task::yield_now().await;
}
});
if outcome == "cancelled" {
manager.cancel_task(&alias).await.expect("cancel active task by alias");
} else {
hook.execute.notify_one();
}
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
.await
.expect("scheduler archives terminal");
assert!(!manager.active_heals.lock().await.contains_key(&task_id));
let expected = task.get_progress().await;
for token in [&task_id, &alias] {
assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
let report = manager
.get_task_report_for_path_since(&format!("{bucket}/object"), token, Some(u64::MAX))
.await
.expect("terminal token remains queryable at handoff");
assert_eq!(report.progress.as_ref(), Some(&expected));
assert!(report.result_items.is_empty());
match outcome {
"success" => assert_eq!(report.status, HealTaskStatus::Completed),
"failed" => assert!(matches!(report.status, HealTaskStatus::Failed { .. })),
_ => assert_eq!(report.status, HealTaskStatus::Cancelled),
}
}
let retained = manager.completed_heals.lock().await;
assert!(Arc::ptr_eq(&retained[&task_id], &retained[&alias]));
drop(retained);
stop.cancel();
polling.await.expect("concurrent polling succeeds");
// Archived progress must not alias a mutable live progress object.
task.progress.write().await.objects_scanned = 999;
assert_eq!(manager.get_task_report(&alias).await.expect("frozen report").progress, Some(expected));
hook.finish.notify_one();
COMPLETED_RETENTION_HOOKS
.lock()
.await
.retain(|key, _| key != &bucket && key != &task_id);
}
}
#[async_trait::async_trait]
impl HealStorageAPI for MockStorage {
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> {
@@ -123,6 +443,12 @@ impl HealStorageAPI for MockStorage {
}
async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(bucket).cloned();
if let Some(hook) = hook {
hook.started.notify_one();
hook.execute.notified().await;
return Ok(true);
}
Ok(bucket == "retry-transition")
}
@@ -133,13 +459,18 @@ impl HealStorageAPI for MockStorage {
_version_id: Option<&str>,
_opts: &HealOpts,
) -> Result<(HealResultItem, Option<Error>)> {
if bucket == "completed-retention-failed" {
return Err(Error::TaskExecutionFailed {
message: "retention fixture failure".to_string(),
});
}
if let Some(hook) = manager_recovery_test_hook() {
*hook
.heal_object_calls
.lock()
.expect("manager recovery object call lock should not poison") += 1;
}
if bucket == "retry-transition" {
if matches!(bucket, "retry-transition" | "completed-retention-retry-cancel") {
return Ok((
HealResultItem::default(),
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
@@ -1145,7 +1476,13 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
.expect("duplicate token should cancel merged active task");
assert!(manager.active_heals.lock().await.get(&active_task_id).is_none());
assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. })));
assert_eq!(
manager
.get_task_status(&active_task_id)
.await
.expect("cancelled task remains queryable"),
HealTaskStatus::Cancelled
);
}
#[tokio::test]
@@ -1638,6 +1975,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
manager.completed_heals.lock().await.insert(
task_id,
Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type,
status: HealTaskStatus::Retrying {
error: "Lock acquisition timeout".to_string(),
@@ -2053,7 +2392,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() {
"the overlapping admin task must be cancelled (removed from the active table) before the new one starts"
);
assert!(
matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })),
matches!(manager.get_task_status(&old_id).await, Ok(HealTaskStatus::Cancelled)),
"a cancelled task must no longer resolve as an active heal"
);
}
@@ -2360,6 +2699,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
manager.completed_heals.lock().await.insert(
task_id.clone(),
Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(),
status: HealTaskStatus::Retrying {
error: "transient disk failure".to_string(),
@@ -2395,6 +2736,8 @@ async fn test_get_task_status_reads_recent_completed_status() {
manager.completed_heals.lock().await.insert(
"completed-token".to_string(),
Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Bucket {
bucket: "bucket".to_string(),
},
@@ -2424,6 +2767,8 @@ async fn test_get_task_report_for_path_reads_completed_items() {
manager.completed_heals.lock().await.insert(
"completed-token".to_string(),
Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Object {
bucket: "bucket".to_string(),
object: "object".to_string(),
crates/heal/src/heal/task.rs
+1 -1
View File
@@ -999,7 +999,7 @@ impl HealTask {
let items = match since {
None => result_items.iter().map(|(_, item)| item.clone()).collect::<Vec<_>>(),
Some(cursor) => {
if cursor + 1 < min_seq {
if cursor.saturating_add(1) < min_seq {
lagged = true;
}
result_items
crates/madmin/src/client.rs
+72
View File
@@ -182,6 +182,9 @@ pub struct BackgroundHealStatus {
pub heal_active_tasks: u64,
#[serde(default)]
pub cluster_status_complete: bool,
/// Missing on older servers; absent coverage or counts mean unknown.
#[serde(default)]
pub coverage: Option<BackgroundHealCoverage>,
#[serde(default)]
pub progress: Option<serde_json::Value>,
/// Remaining wire fields (flattened `BackgroundHealInfo` plus the
@@ -190,6 +193,22 @@ pub struct BackgroundHealStatus {
pub extra: serde_json::Map<String, serde_json::Value>,
}
/// Node coverage of a background heal status snapshot. Counters describe only
/// nodes with usable snapshots; unknown peers may still be running heal work.
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BackgroundHealCoverage {
#[serde(default)]
pub expected: Option<usize>,
#[serde(default)]
pub responded: Option<usize>,
#[serde(default)]
pub unknown: Option<usize>,
/// Stable reason codes; unknown future codes are preserved verbatim.
#[serde(default)]
pub reasons: Vec<String>,
}
/// `GET /v3/scanner/status` response, typed at the fields operators branch
/// on; everything else passes through verbatim.
#[derive(Debug, Clone, Deserialize)]
@@ -630,9 +649,39 @@ mod tests {
assert_eq!(status.state, "active");
assert_eq!(status.heal_queue_length, 3);
assert!(status.cluster_status_complete);
assert!(status.coverage.is_none(), "legacy payloads have unknown coverage");
assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through");
}
#[test]
fn background_heal_status_missing_coverage_fields_remain_unknown() {
for raw in [json!({"state": "degraded"}), json!({"state": "degraded", "coverage": {}})] {
let status: BackgroundHealStatus = serde_json::from_value(raw).expect("partial legacy payload decodes");
assert!(!status.cluster_status_complete);
if let Some(coverage) = status.coverage {
assert_eq!(coverage.expected, None);
assert_eq!(coverage.responded, None);
assert_eq!(coverage.unknown, None);
}
}
}
#[test]
fn background_heal_status_preserves_future_fields_and_reasons() {
let raw = json!({
"state": "degraded", "clusterStatusComplete": false,
"coverage": {"expected": 3, "responded": 1, "unknown": 2, "reasons": ["future_reason"], "futureCoverage": true},
"futureStatus": {"value": 7}
});
let status: BackgroundHealStatus = serde_json::from_value(raw).expect("future additive fields decode");
assert_eq!(status.extra["futureStatus"]["value"], 7);
let coverage = status.coverage.expect("coverage supplied");
assert_eq!(coverage.expected, Some(3));
assert_eq!(coverage.responded, Some(1));
assert_eq!(coverage.unknown, Some(2));
assert_eq!(coverage.reasons, ["future_reason"]);
}
#[test]
fn scanner_status_defaults_freshness_to_unknown() {
let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}});
@@ -721,6 +770,7 @@ mod tests {
let status = client.background_heal_status().await.expect("status decodes");
assert_eq!(status.state, "idle");
assert!(status.coverage.is_none(), "older HTTP responses retain unknown coverage");
let request = server.recorded();
// The server registers this route POST-only; a GET here answers 405.
assert_eq!(request.method, "POST");
@@ -728,6 +778,28 @@ mod tests {
assert_eq!(request.query, "");
}
#[tokio::test]
async fn background_heal_status_decodes_partial_coverage_over_http() {
let body = r#"{"state":"degraded","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":false,"coverage":{"expected":3,"responded":1,"unknown":2,"reasons":["notification_system_unavailable"]},"futureStatus":true}"#;
let server = TestServer::spawn(body, 200).await;
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").expect("client builds");
let status = client
.background_heal_status()
.await
.expect("partial status is a successful response");
assert_eq!(status.state, "degraded");
assert!(!status.cluster_status_complete);
assert_eq!(status.extra["futureStatus"], true);
let coverage = status.coverage.expect("partial coverage supplied");
assert_eq!(coverage.expected, Some(3));
assert_eq!(coverage.responded, Some(1));
assert_eq!(coverage.unknown, Some(2));
assert_eq!(coverage.reasons, ["notification_system_unavailable"]);
let request = server.recorded();
assert_eq!(request.method, "POST");
assert_eq!(request.query, "", "reading status must not send heal control parameters");
}
#[tokio::test]
async fn http_error_status_maps_to_a_typed_error_with_body() {
let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await;
crates/protos/src/generated/proto_gen/node_service.rs
+286
View File
@@ -1283,6 +1283,54 @@ pub struct ScannerDirtyUsageSnapshotResponse {
#[prost(bytes = "bytes", tag = "7")]
pub response_proof: ::prost::bytes::Bytes,
}
/// Receiver-only protocol. Producers must retain whole-cycle ACK until they
/// have a durable per-bucket publication proof.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerScopedDirtyUsageEntry {
#[prost(string, tag = "1")]
pub bucket: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "2")]
pub bucket_incarnation: ::prost::bytes::Bytes,
#[prost(uint64, tag = "3")]
pub generation: u64,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ScannerScopedDirtyUsageAckRequest {
#[prost(bytes = "bytes", tag = "1")]
pub challenge: ::prost::bytes::Bytes,
#[prost(uint32, tag = "2")]
pub protocol_version: u32,
#[prost(string, tag = "3")]
pub owner_id: ::prost::alloc::string::String,
#[prost(string, tag = "4")]
pub instance_id: ::prost::alloc::string::String,
/// Only scope 1 (a complete bucket) is supported; zero is invalid.
#[prost(uint32, tag = "5")]
pub scope: u32,
#[prost(bool, tag = "6")]
pub probe_only: bool,
#[prost(message, repeated, tag = "7")]
pub entries: ::prost::alloc::vec::Vec<ScannerScopedDirtyUsageEntry>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerScopedDirtyUsageAckResponse {
#[prost(uint32, tag = "1")]
pub protocol_version: u32,
#[prost(string, tag = "2")]
pub owner_id: ::prost::alloc::string::String,
#[prost(string, tag = "3")]
pub instance_id: ::prost::alloc::string::String,
#[prost(bool, tag = "4")]
pub supported: bool,
#[prost(uint32, tag = "5")]
pub max_entries: u32,
#[prost(uint32, tag = "6")]
pub max_request_bytes: u32,
#[prost(uint64, tag = "7")]
pub cleared: u64,
#[prost(bytes = "bytes", tag = "8")]
pub response_proof: ::prost::bytes::Bytes,
}
/// A short-lived storage-owned read admission used only around a final
/// scanner metadata publication. It is intentionally separate from the
/// ScannerActivity observation wire so v6/v7 rolling compatibility remains
@@ -6282,6 +6330,244 @@ pub mod node_service_server {
}
}
/// Generated client implementations.
pub mod scanner_control_service_client {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::http::Uri;
use tonic::codegen::*;
#[derive(Debug, Clone)]
pub struct ScannerControlServiceClient<T> {
inner: tonic::client::Grpc<T>,
}
impl ScannerControlServiceClient<tonic::transport::Channel> {
/// Attempt to create a new client by connecting to a given endpoint.
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
where
D: TryInto<tonic::transport::Endpoint>,
D::Error: Into<StdError>,
{
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
Ok(Self::new(conn))
}
}
impl<T> ScannerControlServiceClient<T>
where
T: tonic::client::GrpcService<tonic::body::Body>,
T::Error: Into<StdError>,
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
{
pub fn new(inner: T) -> Self {
let inner = tonic::client::Grpc::new(inner);
Self { inner }
}
pub fn with_origin(inner: T, origin: Uri) -> Self {
let inner = tonic::client::Grpc::with_origin(inner, origin);
Self { inner }
}
pub fn with_interceptor<F>(inner: T, interceptor: F) -> ScannerControlServiceClient<InterceptedService<T, F>>
where
F: tonic::service::Interceptor,
T::ResponseBody: Default,
T: tonic::codegen::Service<
http::Request<tonic::body::Body>,
Response = http::Response<<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody>,
>,
<T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
Into<StdError> + std::marker::Send + std::marker::Sync,
{
ScannerControlServiceClient::new(InterceptedService::new(inner, interceptor))
}
/// Compress requests with the given encoding.
///
/// This requires the server to support it otherwise it might respond with an
/// error.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.send_compressed(encoding);
self
}
/// Enable decompressing responses.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.accept_compressed(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_decoding_message_size(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_encoding_message_size(limit);
self
}
pub async fn scanner_scoped_dirty_usage_ack(
&mut self,
request: impl tonic::IntoRequest<super::ScannerScopedDirtyUsageAckRequest>,
) -> std::result::Result<tonic::Response<super::ScannerScopedDirtyUsageAckResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.ScannerControlService/ScannerScopedDirtyUsageAck");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.ScannerControlService", "ScannerScopedDirtyUsageAck"));
self.inner.unary(req, path, codec).await
}
}
}
/// Generated server implementations.
pub mod scanner_control_service_server {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::*;
/// Generated trait containing gRPC methods that should be implemented for use with ScannerControlServiceServer.
#[async_trait]
pub trait ScannerControlService: std::marker::Send + std::marker::Sync + 'static {
async fn scanner_scoped_dirty_usage_ack(
&self,
request: tonic::Request<super::ScannerScopedDirtyUsageAckRequest>,
) -> std::result::Result<tonic::Response<super::ScannerScopedDirtyUsageAckResponse>, tonic::Status>;
}
#[derive(Debug)]
pub struct ScannerControlServiceServer<T> {
inner: Arc<T>,
accept_compression_encodings: EnabledCompressionEncodings,
send_compression_encodings: EnabledCompressionEncodings,
max_decoding_message_size: Option<usize>,
max_encoding_message_size: Option<usize>,
}
impl<T> ScannerControlServiceServer<T> {
pub fn new(inner: T) -> Self {
Self::from_arc(Arc::new(inner))
}
pub fn from_arc(inner: Arc<T>) -> Self {
Self {
inner,
accept_compression_encodings: Default::default(),
send_compression_encodings: Default::default(),
max_decoding_message_size: None,
max_encoding_message_size: None,
}
}
pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
where
F: tonic::service::Interceptor,
{
InterceptedService::new(Self::new(inner), interceptor)
}
/// Enable decompressing requests with the given encoding.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.accept_compression_encodings.enable(encoding);
self
}
/// Compress responses with the given encoding, if the client supports it.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.send_compression_encodings.enable(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.max_decoding_message_size = Some(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.max_encoding_message_size = Some(limit);
self
}
}
impl<T, B> tonic::codegen::Service<http::Request<B>> for ScannerControlServiceServer<T>
where
T: ScannerControlService,
B: Body + std::marker::Send + 'static,
B::Error: Into<StdError> + std::marker::Send + 'static,
{
type Response = http::Response<tonic::body::Body>;
type Error = std::convert::Infallible;
type Future = BoxFuture<Self::Response, Self::Error>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, req: http::Request<B>) -> Self::Future {
match req.uri().path() {
"/node_service.ScannerControlService/ScannerScopedDirtyUsageAck" => {
#[allow(non_camel_case_types)]
struct ScannerScopedDirtyUsageAckSvc<T: ScannerControlService>(pub Arc<T>);
impl<T: ScannerControlService> tonic::server::UnaryService<super::ScannerScopedDirtyUsageAckRequest>
for ScannerScopedDirtyUsageAckSvc<T>
{
type Response = super::ScannerScopedDirtyUsageAckResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::ScannerScopedDirtyUsageAckRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as ScannerControlService>::scanner_scoped_dirty_usage_ack(&inner, request).await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = ScannerScopedDirtyUsageAckSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
_ => Box::pin(async move {
let mut response = http::Response::new(tonic::body::Body::default());
let headers = response.headers_mut();
headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
Ok(response)
}),
}
}
}
impl<T> Clone for ScannerControlServiceServer<T> {
fn clone(&self) -> Self {
let inner = self.inner.clone();
Self {
inner,
accept_compression_encodings: self.accept_compression_encodings,
send_compression_encodings: self.send_compression_encodings,
max_decoding_message_size: self.max_decoding_message_size,
max_encoding_message_size: self.max_encoding_message_size,
}
}
}
/// Generated gRPC service name
pub const SERVICE_NAME: &str = "node_service.ScannerControlService";
impl<T> tonic::server::NamedService for ScannerControlServiceServer<T> {
const NAME: &'static str = SERVICE_NAME;
}
}
/// Generated client implementations.
pub mod heal_control_service_client {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::http::Uri;
crates/protos/src/lib.rs
+2
View File
@@ -541,6 +541,8 @@ pub fn canonical_scanner_activity_v7_response_body(
Ok(body)
}
pub mod scoped_dirty_usage;
pub fn canonical_scanner_dirty_usage_snapshot_request_body(
request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
crates/protos/src/node.proto
+34
View File
@@ -903,6 +903,36 @@ message ScannerDirtyUsageSnapshotResponse {
bytes response_proof = 7;
}
// Receiver-only protocol. Producers must retain whole-cycle ACK until they
// have a durable per-bucket publication proof.
message ScannerScopedDirtyUsageEntry {
string bucket = 1;
bytes bucket_incarnation = 2;
uint64 generation = 3;
}
message ScannerScopedDirtyUsageAckRequest {
bytes challenge = 1;
uint32 protocol_version = 2;
string owner_id = 3;
string instance_id = 4;
// Only scope 1 (a complete bucket) is supported; zero is invalid.
uint32 scope = 5;
bool probe_only = 6;
repeated ScannerScopedDirtyUsageEntry entries = 7;
}
message ScannerScopedDirtyUsageAckResponse {
uint32 protocol_version = 1;
string owner_id = 2;
string instance_id = 3;
bool supported = 4;
uint32 max_entries = 5;
uint32 max_request_bytes = 6;
uint64 cleared = 7;
bytes response_proof = 8;
}
// A short-lived storage-owned read admission used only around a final
// scanner metadata publication. It is intentionally separate from the
// ScannerActivity observation wire so v6/v7 rolling compatibility remains
@@ -1245,6 +1275,10 @@ service NodeService {
rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only
}
service ScannerControlService {
rpc ScannerScopedDirtyUsageAck(ScannerScopedDirtyUsageAckRequest) returns (ScannerScopedDirtyUsageAckResponse) {}; // auth-policy: body-bound
}
service HealControlService {
rpc HealControl(HealControlRequest) returns (HealControlResponse) {};
}
crates/protos/src/scoped_dirty_usage.rs
+213
View File
@@ -0,0 +1,213 @@
// Copyright 2024 RustFS Team
// Licensed under the Apache License, Version 2.0.
//! Bounded, authenticated receiver contract for per-bucket dirty acknowledgements.
use crate::CanonicalBodyBuilder;
use crate::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
use prost::Message;
pub const SCOPED_DIRTY_USAGE_PROTOCOL_VERSION: u32 = 1;
pub const SCOPED_DIRTY_USAGE_BUCKET_SCOPE: u32 = 1;
pub const SCOPED_DIRTY_USAGE_MAX_ENTRIES: u32 = 32;
pub const SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES: u32 = 8192;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScopedDirtyUsageRequestError {
UnsupportedProtocol,
UnsupportedScope,
InvalidIdentity,
InvalidGeneration,
InvalidEntries,
TooLarge,
}
impl std::fmt::Display for ScopedDirtyUsageRequestError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(match self {
Self::UnsupportedProtocol => "unsupported scoped dirty usage protocol",
Self::UnsupportedScope => "unsupported scoped dirty usage scope",
Self::InvalidIdentity => "invalid scoped dirty usage identity",
Self::InvalidGeneration => "invalid scoped dirty usage generation",
Self::InvalidEntries => "scoped dirty usage entries must be nonempty and strictly ordered",
Self::TooLarge => "scoped dirty usage request exceeds its budget",
})
}
}
impl std::error::Error for ScopedDirtyUsageRequestError {}
pub fn validate_scoped_dirty_usage_request(
request: &ScannerScopedDirtyUsageAckRequest,
) -> Result<(), ScopedDirtyUsageRequestError> {
use ScopedDirtyUsageRequestError as E;
if request.entries.len() > SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize
|| request.encoded_len() > SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize
{
return Err(E::TooLarge);
}
if request.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION {
return Err(E::UnsupportedProtocol);
}
if request.scope != SCOPED_DIRTY_USAGE_BUCKET_SCOPE {
return Err(E::UnsupportedScope);
}
if request.challenge.len() != 16 || request.owner_id.len() != 36 || request.instance_id.len() != 32 {
return Err(E::InvalidIdentity);
}
if request.entries.is_empty() || request.entries.windows(2).any(|pair| pair[0].bucket >= pair[1].bucket) {
return Err(E::InvalidEntries);
}
for entry in &request.entries {
if entry.bucket.is_empty()
|| entry.bucket.len() > 63
|| entry.bucket_incarnation.len() != 16
|| entry.bucket_incarnation.iter().all(|byte| *byte == 0)
{
return Err(E::InvalidIdentity);
}
if entry.generation == 0 || entry.generation == u64::MAX {
return Err(E::InvalidGeneration);
}
}
Ok(())
}
pub fn canonical_scoped_dirty_usage_request(
request: &ScannerScopedDirtyUsageAckRequest,
) -> Result<Vec<u8>, ScopedDirtyUsageRequestError> {
validate_scoped_dirty_usage_request(request)?;
let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-request-v1\0");
let encode = |_: std::num::TryFromIntError| ScopedDirtyUsageRequestError::TooLarge;
body.push_bytes(request.challenge.as_ref()).map_err(encode)?;
body.push_u32(request.protocol_version);
body.push_str(&request.owner_id).map_err(encode)?;
body.push_str(&request.instance_id).map_err(encode)?;
body.push_u32(request.scope);
body.push_bool(request.probe_only);
body.push_count(request.entries.len()).map_err(encode)?;
for entry in &request.entries {
body.push_str(&entry.bucket).map_err(encode)?;
body.push_bytes(entry.bucket_incarnation.as_ref()).map_err(encode)?;
body.push_u64(entry.generation);
}
Ok(body.finish())
}
pub fn canonical_scoped_dirty_usage_response(
request_body: &[u8],
response: &ScannerScopedDirtyUsageAckResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-response-v1\0");
body.push_bytes(request_body)?;
body.push_u32(response.protocol_version);
body.push_str(&response.owner_id)?;
body.push_str(&response.instance_id)?;
body.push_bool(response.supported);
body.push_u32(response.max_entries);
body.push_u32(response.max_request_bytes);
body.push_u64(response.cleared);
Ok(body.finish())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
fn request() -> ScannerScopedDirtyUsageAckRequest {
ScannerScopedDirtyUsageAckRequest {
challenge: vec![1; 16].into(),
protocol_version: 1,
owner_id: "11111111-1111-1111-1111-111111111111".into(),
instance_id: "a".repeat(32),
scope: 1,
probe_only: false,
entries: vec![ScannerScopedDirtyUsageEntry {
bucket: "photos".into(),
bucket_incarnation: vec![2; 16].into(),
generation: 8,
}],
}
}
#[test]
fn scoped_dirty_usage_binds_every_request_field() {
let base = request();
let baseline = canonical_scoped_dirty_usage_request(&base).expect("valid request");
for field in 0..9 {
let mut changed = base.clone();
match field {
0 => changed.challenge = vec![3; 16].into(),
1 => changed.protocol_version += 1,
2 => changed.owner_id = "22222222-2222-2222-2222-222222222222".into(),
3 => changed.instance_id = "b".repeat(32),
4 => changed.scope += 1,
5 => changed.probe_only = true,
6 => changed.entries[0].bucket = "videos".into(),
7 => changed.entries[0].bucket_incarnation = vec![3; 16].into(),
_ => changed.entries[0].generation += 1,
}
assert!(canonical_scoped_dirty_usage_request(&changed).map_or(true, |body| body != baseline));
}
}
#[test]
fn scoped_dirty_usage_binds_capability_and_ack_to_exact_request() {
let request = canonical_scoped_dirty_usage_request(&request()).expect("valid request");
let response = ScannerScopedDirtyUsageAckResponse {
protocol_version: 1,
owner_id: "owner".into(),
instance_id: "process".into(),
supported: true,
max_entries: 32,
max_request_bytes: 8192,
cleared: 1,
response_proof: vec![1; 32].into(),
};
let baseline = canonical_scoped_dirty_usage_response(&request, &response).expect("valid response");
for field in 0..7 {
let mut changed = response.clone();
match field {
0 => changed.protocol_version += 1,
1 => changed.owner_id.push('x'),
2 => changed.instance_id.push('x'),
3 => changed.supported = false,
4 => changed.max_entries += 1,
5 => changed.max_request_bytes += 1,
_ => changed.cleared += 1,
}
assert_ne!(
canonical_scoped_dirty_usage_response(&request, &changed).expect("response variant"),
baseline
);
}
assert_ne!(
canonical_scoped_dirty_usage_response(b"another request", &response).expect("request variant"),
baseline
);
}
#[test]
fn scoped_dirty_usage_rejects_overflow_unknown_and_duplicate_entries() {
let base = request();
let mut invalid = base.clone();
invalid.entries = vec![base.entries[0].clone(); SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize + 1];
assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge));
invalid = base.clone();
invalid.entries[0].bucket = "x".repeat(SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize);
assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge));
invalid = base.clone();
invalid.entries.push(base.entries[0].clone());
assert_eq!(
validate_scoped_dirty_usage_request(&invalid),
Err(ScopedDirtyUsageRequestError::InvalidEntries)
);
invalid = base;
invalid.entries[0].bucket_incarnation = vec![0; 16].into();
assert_eq!(
validate_scoped_dirty_usage_request(&invalid),
Err(ScopedDirtyUsageRequestError::InvalidIdentity)
);
}
}
crates/scanner/src/lib.rs
+3 -2
View File
@@ -90,8 +90,9 @@ pub use scanner::{
};
pub use scanner_io::{
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
scanner_maintenance_generation,
};
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
use std::sync::atomic::{AtomicU64, Ordering};
crates/scanner/src/scanner.rs
+15 -8
View File
@@ -1703,14 +1703,18 @@ where
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
let done_cycle = Metrics::time(Metric::ScanCycle);
let scan_result = crate::scanner_io::nsscanner_with_storage_status(
let scan_result = crate::scanner_io::nsscanner_with_storage_status_scoped(
storeapi.as_ref(),
cycle_budget.token(),
cycle_budget.clone(),
sender,
cycle_info.current,
leader_epoch,
scan_mode,
crate::scanner_io::ScannerCycleRequest {
ctx: cycle_budget.token(),
budget: cycle_budget.clone(),
updates: sender,
want_cycle: cycle_info.current,
leader_epoch,
scan_mode,
scan_scope: crate::scanner_io::ScannerBucketScanScope::default(),
persisted_usage_baseline: usage_persist_baseline.data.clone(),
},
)
.await;
let publication_defer_reason = match &scan_result {
@@ -3424,10 +3428,13 @@ use cycle_state::*;
use leadership::*;
use usage_store::*;
#[cfg(test)]
pub(crate) use activity::scanner_activity_snapshot_digest;
pub use activity::scanner_topology_digest;
pub(crate) use activity::{
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_structural_digest,
scanner_dirty_usage_acknowledgements,
};
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
pub use backlog::{
crates/scanner/src/scanner/activity.rs
+41
View File
@@ -902,6 +902,7 @@ where
observation
}
#[cfg(test)]
pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
let mut hasher = Sha256::new();
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
@@ -925,6 +926,30 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
hasher.finalize().into()
}
/// Hash the activity inputs that make an existing scanner cache unsafe to
/// reuse. Regular namespace writes and dirty-usage generations are omitted:
/// their affected buckets are tracked separately and may be refreshed from a
/// complete authoritative cache baseline.
pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
let mut hasher = Sha256::new();
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
for (host, activity) in snapshot {
let host = host.as_bytes();
let instance_id = activity.instance_id.as_bytes();
hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes());
hasher.update(host);
hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes());
hasher.update(instance_id);
hasher.update(activity.maintenance_generation.to_be_bytes());
hasher.update(activity.protocol_version.to_be_bytes());
hasher.update(activity.topology_digest);
hasher.update([u8::from(activity.data_movement_active)]);
hasher.update(activity.movement_generation.to_be_bytes());
hasher.update([u8::from(activity.publication_blocked)]);
}
hasher.finalize().into()
}
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
!snapshot.is_empty()
&& snapshot.values().all(|activity| {
@@ -955,6 +980,22 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna
.collect()
}
pub(crate) fn scanner_activity_dirty_usage_state_for_host<'a>(
snapshot: &'a ScannerActivitySnapshot,
host: &str,
) -> Option<(&'a str, u64, bool)> {
snapshot
.get(host)
.filter(|_| host != LOCAL_SCANNER_ACTIVITY_NODE)
.map(|activity| {
(
activity.instance_id.as_str(),
activity.dirty_usage_generation,
activity.dirty_usage_pending,
)
})
}
pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] {
let endpoint_pools = storeapi.endpoints();
let mut hasher = Sha256::new();
crates/scanner/src/scanner/cycle_state.rs
+280 -51
View File
@@ -379,6 +379,12 @@ pub(super) fn decode_recovery_marker_for_reset(
if !matches!(marker_revision, DataUsageCacheRevision::Etag(_)) {
return Err(ScannerError::Other("cycle recovery marker has no object revision".to_string()));
}
if let Ok(value) = serde_json::from_slice::<serde_json::Value>(data)
&& let Some(state) = value.get("state")
&& !matches!(state.as_str(), Some("blocked" | "cleanup-pending"))
{
return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string()));
}
let compat = serde_json::from_slice::<ScannerCycleRecoveryMarkerCompat>(data).ok();
let _schema_version = compat.as_ref().and_then(|marker| marker.schema_version);
let primary_revision = compat
@@ -406,7 +412,10 @@ pub(super) fn decode_recovery_marker_for_reset(
};
let state = match compat.as_ref().and_then(|marker| marker.state.as_deref()) {
Some("cleanup-pending") => "cleanup-pending",
_ => "blocked",
Some("blocked") | None => "blocked",
Some(_) => {
return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string()));
}
};
let now = unix_now_secs();
Ok(ScannerCycleRecoveryMarker {
@@ -721,17 +730,19 @@ async fn mark_cycle_recovery_cleanup_pending(
mut marker: ScannerCycleRecoveryMarker,
marker_revision: &DataUsageCacheRevision,
expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> {
marker.state = "cleanup-pending".to_string();
marker.last_attempt_at_unix_secs = unix_now_secs();
let bytes = serde_json::to_vec(&marker)
.map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?;
let info = save_config_with_publication_admission_for_epoch(
let info = save_reset_config(
storeapi.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
bytes,
marker_revision.preconditions(),
expected_epoch,
owns_reset,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?;
@@ -933,6 +944,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
.get_write_lock_quiet(Duration::from_secs(5))
.await
.map_err(|err| ScannerError::Other(format!("scanner leader lock is busy: {err}")))?;
let owns_reset = || !guard.is_lock_lost() && !ctx.is_cancelled();
if guard.is_lock_lost() {
return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string()));
@@ -952,7 +964,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
}
Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))),
};
let marker_data = marker_data.ok_or_else(|| ScannerError::Other("scanner cycle recovery marker is absent".to_string()))?;
let Some(marker_data) = marker_data else {
// A delete may commit before its reply is lost. Confirm both durable
// fences before treating a retry without its marker as completed.
let (cycle, epoch, revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
let floor = persisted_usage_floor(storeapi.clone()).await?;
if !matches!(revision, DataUsageCacheRevision::Etag(_))
|| epoch < floor.leader_epoch
|| cycle.next < floor.next_cycle
|| !owns_reset()
|| scanner_publication_admission_for_epoch(storeapi.clone(), reset_epoch)
.await
.is_none()
{
return Err(ScannerError::Other(
"scanner cycle recovery marker is absent without a completed reset fence".to_string(),
));
}
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
super::notify_scanner_cycle_recovery_wake();
return Ok(());
};
let (marker, force_full_rescan) = match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&marker_data) {
Ok(marker) if validate_recovery_marker(&marker).is_ok() => (marker, false),
_ => (decode_recovery_marker_for_reset(&marker_data, &marker_revision)?, true),
@@ -1026,8 +1058,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
}
};
if let Some((primary_cycle, primary_epoch)) = primary_state {
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
let (cleanup_marker, cleanup_marker_revision) =
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?;
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch, &owns_reset)
.await?;
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
let fence_epoch = primary_epoch
@@ -1047,12 +1081,14 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"preserved scanner cycle state exceeds the bounded object size".to_string(),
));
}
let preserved_info = save_config_with_publication_admission_for_epoch(
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
let preserved_info = save_reset_config(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
preserved_data,
primary_revision.preconditions(),
reset_epoch,
&owns_reset,
)
.await
.map_err(|err| {
@@ -1072,9 +1108,17 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after fencing newer cycle state".to_string(),
));
}
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch), false)
.await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
fence_scanner_usage_epoch_with_expected_epoch(
&ctx,
storeapi.clone(),
fence_epoch,
Some(reset_epoch),
false,
&owns_reset,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
if guard.is_lock_lost() {
return Err(ScannerError::Other(
"scanner leader lock was lost after fencing newer cycle state".to_string(),
@@ -1088,7 +1132,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner cycle state changed before recovery marker cleanup".to_string(),
));
}
delete_config_with_publication_admission_for_epoch(
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
delete_reset_config(
storeapi.clone(),
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
@@ -1100,6 +1145,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
..Default::default()
},
reset_epoch,
&owns_reset,
)
.await
.map_err(|err| {
@@ -1149,17 +1195,20 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
// Persist the cleanup-pending phase before rewriting the primary. If the
// process dies after the rewrite, startup still sees a durable fence and
// cannot mistake the partially completed reset for a healthy state.
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
let (marker, marker_revision) = if marker.state == "cleanup-pending" {
(marker, marker_revision)
} else {
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await?
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch, &owns_reset).await?
};
let rebuilt_info = save_config_with_publication_admission_for_epoch(
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
let rebuilt_info = save_reset_config(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
data,
primary_revision.preconditions(),
reset_epoch,
&owns_reset,
)
.await
.map_err(|err| {
@@ -1178,8 +1227,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after rebuilding cycle state".to_string(),
));
}
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
if let Err(err) =
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false).await
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false, &owns_reset)
.await
{
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
@@ -1249,7 +1300,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
));
}
if let Err(err) = delete_config_with_publication_admission_for_epoch(
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
if let Err(err) = delete_reset_config(
storeapi.clone(),
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
@@ -1261,6 +1313,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
..Default::default()
},
reset_epoch,
&owns_reset,
)
.await
{
@@ -1310,6 +1363,57 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
Ok(())
}
async fn verify_cycle_reset_intent(
storeapi: Arc<impl ScannerObjectIO>,
expected_revision: &DataUsageCacheRevision,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<(), ScannerError> {
let revision = read_config_revision(storeapi, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to verify scanner cycle reset intent: {err}")))?;
if &revision != expected_revision {
return Err(ScannerError::Other("scanner cycle reset intent changed".to_string()));
}
if !owns_reset() {
return Err(ScannerError::Other("scanner cycle reset ownership was lost".to_string()));
}
Ok(())
}
async fn save_reset_config(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
path: &str,
data: Vec<u8>,
preconditions: crate::HTTPPreconditions,
expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<crate::ScannerObjectInfo, EcstoreError> {
let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
if !owns_reset() {
return Err(EcstoreError::other("scanner reset ownership was lost before write"));
}
save_config_with_preconditions(storeapi, path, data, preconditions).await
}
async fn delete_reset_config(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
bucket: &str,
path: &str,
options: ScannerObjectOptions,
expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<crate::ScannerObjectInfo, EcstoreError> {
let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
if !owns_reset() {
return Err(EcstoreError::other("scanner reset ownership was lost before delete"));
}
storeapi.delete_config_object(bucket, path, options).await
}
fn scanner_usage_state_reset_paths() -> Vec<String> {
vec![
DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
@@ -1333,8 +1437,14 @@ pub(super) async fn read_usage_state_reset_slots(
Ok(slots)
}
fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<PersistedUsageFloor, ScannerError> {
let mut floor = PersistedUsageFloor::default();
enum ScannerUsageResetFloor {
Missing,
Trusted(PersistedUsageFloor),
Corrupt,
}
fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<ScannerUsageResetFloor, ScannerError> {
let mut floor = None;
for slot in slots {
let Some(data) = slot.data.as_deref() else {
continue;
@@ -1342,9 +1452,21 @@ fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<Persi
let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(data) else {
continue;
};
update_persisted_usage_floor(&mut floor, &usage, &slot.path)?;
if !data_usage_info_has_persisted_baseline_identity(&usage)
&& !(slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str() && data_usage_info_is_bootstrap_pending(&usage))
&& legacy_incomplete_usage_fence(data, &usage)
.and_then(|fence| fence.claimable_epoch())
.is_none()
{
continue;
}
update_persisted_usage_floor(floor.get_or_insert_with(PersistedUsageFloor::default), &usage, &slot.path)?;
}
Ok(floor)
Ok(match floor {
Some(floor) => ScannerUsageResetFloor::Trusted(floor),
None if slots.iter().any(|slot| slot.data.is_some()) => ScannerUsageResetFloor::Corrupt,
None => ScannerUsageResetFloor::Missing,
})
}
async fn read_cycle_state_for_usage_reset(
@@ -1401,11 +1523,12 @@ async fn delete_usage_state_reset_slot(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
slot: &ScannerUsageStateResetSlot,
expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<bool, ScannerError> {
if matches!(slot.revision, DataUsageCacheRevision::Missing) {
return Ok(false);
}
let delete_result = delete_config_with_publication_admission_for_epoch(
let delete_result = delete_reset_config(
storeapi.clone(),
RUSTFS_META_BUCKET,
&slot.path,
@@ -1415,6 +1538,7 @@ async fn delete_usage_state_reset_slot(
..Default::default()
},
expected_epoch,
owns_reset,
)
.await;
match delete_result {
@@ -1486,21 +1610,24 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary(
expected_publication_epoch: u64,
leader_epoch: Option<u64>,
context: ScannerUsageBootstrapPublishContext,
owns_publication: impl Fn() -> bool + Sync,
) -> Result<(), ScannerError> {
async fn inner(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
expected_revision: &DataUsageCacheRevision,
expected_publication_epoch: u64,
leader_epoch: Option<u64>,
owns_publication: &(impl Fn() -> bool + Sync),
) -> Result<(), ScannerUsageBootstrapPublishError> {
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::now(), leader_epoch);
let data = serde_json::to_vec(&marker).map_err(ScannerUsageBootstrapPublishError::Encode)?;
let save_result = save_config_with_publication_admission_for_epoch(
let save_result = save_reset_config(
storeapi.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
data.clone(),
expected_revision.preconditions(),
expected_publication_epoch,
owns_publication,
)
.await;
if save_result
@@ -1524,7 +1651,7 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary(
})
}
inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch)
inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch, &owns_publication)
.await
.map_err(|err| err.into_scanner_error(context))
}
@@ -1534,32 +1661,108 @@ pub(super) async fn reset_scanner_usage_state_slots_for_full_rebuild(
slots: &[ScannerUsageStateResetSlot],
expected_epoch: u64,
leader_epoch: u64,
owns_reset: impl Fn() -> bool + Sync,
) -> Result<Vec<String>, ScannerError> {
let mut reset_paths = Vec::new();
let primary = slots
.iter()
.find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str())
.ok_or_else(|| ScannerError::Other("scanner usage reset primary slot was not inspected".to_string()))?;
publish_scanner_usage_bootstrap_primary(
storeapi.clone(),
&primary.revision,
expected_epoch,
Some(leader_epoch),
ScannerUsageBootstrapPublishContext::Reset,
)
.await?;
if !owns_reset() {
return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string()));
}
let resume_epoch = usage_state_reset_resume_epoch(slots)?;
match resume_epoch {
Some(epoch) if epoch == leader_epoch => {}
Some(_) => return Err(ScannerError::Other("scanner usage reset bootstrap epoch changed".to_string())),
None => {
publish_scanner_usage_bootstrap_primary(
storeapi.clone(),
&primary.revision,
expected_epoch,
Some(leader_epoch),
ScannerUsageBootstrapPublishContext::Reset,
&owns_reset,
)
.await?;
}
}
let (data, intent_revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to inspect scanner usage reset intent: {err}")))?;
data.as_deref()
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok())
.filter(|usage| data_usage_info_is_bootstrap_pending(usage) && usage.scanner_epoch == Some(leader_epoch))
.ok_or_else(|| ScannerError::Other("scanner usage reset intent changed before cleanup".to_string()))?;
if !matches!(intent_revision, DataUsageCacheRevision::Etag(_))
|| (resume_epoch.is_some() && intent_revision != primary.revision)
{
return Err(ScannerError::Other("scanner usage reset intent revision changed".to_string()));
}
reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string());
for slot in slots.iter().filter(|slot| slot.path != DATA_USAGE_OBJ_NAME_PATH.as_str()) {
if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch).await? {
if let Some(usage) = slot
.data
.as_deref()
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok())
&& usage_epoch(&usage) >= leader_epoch
{
return Err(ScannerError::Other(format!(
"scanner usage reset slot is not older than its intent: {}",
slot.path
)));
}
let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to verify scanner usage reset intent: {err}")))?;
if revision != intent_revision {
return Err(ScannerError::Other("scanner usage reset intent changed during cleanup".to_string()));
}
if !owns_reset() {
return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string()));
}
if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch, &owns_reset).await? {
reset_paths.push(slot.path.clone());
}
}
let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to confirm scanner usage reset intent: {err}")))?;
if revision != intent_revision || !owns_reset() {
return Err(ScannerError::Other(
"scanner usage reset intent or ownership changed before completion".to_string(),
));
}
invalidate_admin_data_usage_snapshot_cache().await;
invalidate_data_usage_snapshot_cache().await;
Ok(reset_paths)
}
fn usage_state_reset_resume_epoch(slots: &[ScannerUsageStateResetSlot]) -> Result<Option<u64>, ScannerError> {
let primary = slots.iter().find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str());
let usage = primary
.and_then(|slot| slot.data.as_deref())
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok());
match usage {
Some(usage) if usage.usage_snapshot_bootstrap_pending => {
if !data_usage_info_is_bootstrap_pending(&usage) {
return Err(ScannerError::Other("scanner usage reset bootstrap is invalid".to_string()));
}
if usage.scanner_epoch.is_none() {
// Initial bootstrap has no reset owner yet.
return Ok(None);
}
usage
.scanner_epoch
.filter(|epoch| *epoch > 0 && *epoch < u64::MAX)
.map(Some)
.ok_or_else(|| ScannerError::Other("scanner usage reset bootstrap has no valid epoch".to_string()))
}
_ => Ok(None),
}
}
pub async fn reset_scanner_usage_state_for_full_rebuild(
ctx: CancellationToken,
storeapi: Arc<ECStore>,
@@ -1584,12 +1787,31 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
};
let (cycle, cycle_epoch, cycle_revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
let slots = read_usage_state_reset_slots(storeapi.clone()).await?;
let usage_floor = usage_state_reset_floor(&slots)?;
let leader_epoch = cycle_epoch
.max(usage_floor.leader_epoch)
.checked_add(1)
.filter(|epoch| *epoch < u64::MAX)
.ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?;
let usage_floor = match usage_state_reset_floor(&slots)? {
ScannerUsageResetFloor::Trusted(floor) => floor,
ScannerUsageResetFloor::Corrupt if matches!(cycle_revision, DataUsageCacheRevision::Missing) => {
return Err(ScannerError::Other("scanner usage reset has no trusted cycle or usage floor".to_string()));
}
ScannerUsageResetFloor::Missing | ScannerUsageResetFloor::Corrupt => PersistedUsageFloor {
next_cycle: cycle.next,
leader_epoch: cycle_epoch,
},
};
let resume_epoch = usage_state_reset_resume_epoch(&slots)?;
let leader_epoch = if let Some(epoch) = resume_epoch {
if epoch != cycle_epoch || usage_floor.leader_epoch > epoch || usage_floor.next_cycle > cycle.next {
return Err(ScannerError::Other(
"scanner usage reset bootstrap conflicts with the persisted cycle fence".to_string(),
));
}
epoch
} else {
cycle_epoch
.max(usage_floor.leader_epoch)
.checked_add(1)
.filter(|epoch| *epoch < u64::MAX)
.ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?
};
let rebuilt_cycle = CurrentCycle {
next: cycle.next.max(usage_floor.next_cycle),
..Default::default()
@@ -1602,21 +1824,24 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
"scanner leader lock was lost before fencing usage reset cycle state".to_string(),
));
}
save_config_with_publication_admission_for_epoch(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
cycle_data,
cycle_revision.preconditions(),
reset_epoch,
)
.await
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}"))
}
})?;
if resume_epoch.is_none() {
save_reset_config(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
cycle_data,
cycle_revision.preconditions(),
reset_epoch,
&|| !guard.is_lock_lost() && !ctx.is_cancelled(),
)
.await
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}"))
}
})?;
}
if guard.is_lock_lost() {
return Err(ScannerError::Other(
@@ -1624,7 +1849,10 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
));
}
let reset_paths =
reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch).await?;
reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch, || {
!guard.is_lock_lost() && !ctx.is_cancelled()
})
.await?;
if guard.is_lock_lost() {
return Err(ScannerError::Other(
"scanner leader lock was lost after publishing usage reset marker".to_string(),
@@ -2135,6 +2363,7 @@ async fn recover_legacy_incomplete_usage_floor(
expected_publication_epoch,
Some(primary.epoch),
ScannerUsageBootstrapPublishContext::Recovery,
|| true,
)
.await?;
warn!(
crates/scanner/src/scanner/leadership.rs
+7 -1
View File
@@ -191,6 +191,7 @@ pub(super) async fn initialize_usage_baseline_bootstrap(
expected_epoch,
None,
ScannerUsageBootstrapPublishContext::Initial,
|| true,
)
.await
}
@@ -201,9 +202,10 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
claimed_epoch: u64,
expected_publication_epoch: Option<u64>,
allow_bootstrap_pending: bool,
owns_fence: impl Fn() -> bool,
) -> Result<(), ScannerError> {
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() {
if ctx.is_cancelled() || !owns_fence() {
return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string()));
}
@@ -264,6 +266,9 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
"scanner usage epoch fence changed while preparing its conditional write".to_string(),
));
};
if ctx.is_cancelled() || !owns_fence() {
return Err(ScannerError::Other("scanner leadership was lost before usage fencing".to_string()));
}
save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions())
.await
};
@@ -319,6 +324,7 @@ pub(super) async fn complete_scanner_leadership_claim(
claimed_epoch,
expected_publication_epoch,
allow_bootstrap_pending,
|| true,
)
.await
{
crates/scanner/src/scanner/tests.rs
+415 -3
View File
@@ -634,6 +634,8 @@ struct MemoryConfigStore {
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
error_after_commit_deletes: Mutex<HashSet<String>>,
cancel_after_deletes: Mutex<HashMap<String, CancellationToken>>,
pause_next_publication_admission: Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>,
put_counts: Mutex<HashMap<String, usize>>,
publication_admission_blocked: AtomicBool,
block_publication_after_admissions: AtomicUsize,
@@ -4081,6 +4083,9 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
revisions.remove(&key);
drop(revisions);
drop(objects);
if let Some(token) = self.cancel_after_deletes.lock().await.remove(&key) {
token.cancel();
}
if self.error_after_commit_deletes.lock().await.remove(&key) {
return Err(EcstoreError::other("injected delete error after commit"));
}
@@ -4088,6 +4093,11 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
let pause = self.pause_next_publication_admission.lock().await.take();
if let Some((entered, resume)) = pause {
entered.notify_one();
resume.notified().await;
}
if self.publication_admission_blocked.load(Ordering::Acquire) {
return None;
}
@@ -4589,7 +4599,7 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met
.expect("publication must also read the intact backup");
assert_eq!(baseline.data.as_deref(), Some(data.as_slice()));
assert_eq!(baseline.revision, DataUsageCacheRevision::Missing);
fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false)
fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false, || true)
.await
.expect("legacy backup must be fenced into v2");
let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
@@ -4818,6 +4828,28 @@ async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() {
);
}
let cycle_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("cycle should remain before retry");
let marker_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("bootstrap should remain before retry");
let retry = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
.await
.expect("completed cleanup should be reentrant");
assert_eq!(retry.leader_epoch, result.leader_epoch);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("cycle should remain"),
cycle_before_retry
);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("bootstrap should remain"),
marker_before_retry
);
let (floor, state) = persisted_usage_floor_for_startup(store, false)
.await
.expect("reset marker should be resumable");
@@ -4973,7 +5005,7 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() {
store.objects.lock().await.insert(key.clone(), b"newer-json".to_vec());
store.revisions.lock().await.insert(key, 2);
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3)
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true)
.await
.expect_err("stale primary revision must not be overwritten");
assert!(
@@ -4983,6 +5015,348 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() {
);
}
#[tokio::test]
async fn scanner_usage_state_reset_resumes_every_cleanup_boundary_without_rewriting_intent() {
for completed in 0..=4 {
let store = Arc::new(MemoryConfigStore::default());
let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str();
let cleanup_paths = [
format!("{primary_path}.bkp"),
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str().to_string(),
];
for path in std::iter::once(primary_path).chain(cleanup_paths.iter().map(String::as_str)) {
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(1);
save_config(store.clone(), path, serde_json::to_vec(&usage).expect("fixture should encode"))
.await
.expect("fixture should persist");
}
// These objects belong to other owners, even when reset cleanup resumes.
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
save_config(store.clone(), path, b"retain".to_vec())
.await
.expect("unrelated state should persist");
}
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
let cancelled = CancellationToken::new();
if completed == 0 {
store
.cancel_after_successful_puts
.lock()
.await
.insert(memory_config_key(RUSTFS_META_BUCKET, primary_path), (2, cancelled.clone()));
} else {
store
.cancel_after_deletes
.lock()
.await
.insert(memory_config_key(RUSTFS_META_BUCKET, &cleanup_paths[completed - 1]), cancelled.clone());
}
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled())
.await
.expect_err("interruption should stop cleanup");
assert!(err.to_string().contains("ownership"), "boundary {completed}: {err}");
for (index, path) in cleanup_paths.iter().enumerate() {
assert_eq!(
store
.objects
.lock()
.await
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, path)),
index >= completed,
"boundary {completed}, slot {index}"
);
}
let intent = read_config_with_revision(store.clone(), primary_path)
.await
.expect("intent should persist");
let slots = read_usage_state_reset_slots(store.clone())
.await
.expect("restart should reload slots");
reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
.await
.expect("restart should complete the same intent");
assert_eq!(
read_config_with_revision(store.clone(), primary_path)
.await
.expect("intent should remain"),
intent
);
assert_eq!(store.put_counts.lock().await[&memory_config_key(RUSTFS_META_BUCKET, primary_path)], 2);
for path in cleanup_paths {
assert!(
!store
.objects
.lock()
.await
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, &path))
);
}
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
assert_eq!(read_config(store.clone(), path).await.expect("unrelated state should remain"), b"retain");
}
}
}
#[tokio::test]
async fn scanner_usage_state_reset_stops_usage_fence_after_owner_loss() {
let store = Arc::new(MemoryConfigStore::default());
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(1);
let bytes = serde_json::to_vec(&usage).expect("baseline should encode");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
.await
.expect("baseline should persist");
let checks = AtomicUsize::new(0);
let err = fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 3, Some(0), false, || {
checks.fetch_add(1, Ordering::SeqCst) == 0
})
.await
.expect_err("ownership lost during reads must prevent the write");
assert!(err.to_string().contains("leadership was lost"), "{err}");
assert_eq!(
read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("baseline should remain"),
bytes
);
}
#[tokio::test]
async fn scanner_usage_state_reset_cancels_during_publication_admission() {
for resuming in [false, true] {
let store = Arc::new(MemoryConfigStore::default());
let usage = if resuming {
scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3))
} else {
complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
};
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&usage).expect("primary should encode"),
)
.await
.expect("primary should persist");
save_config(store.clone(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), b"corrupt".to_vec())
.await
.expect("cleanup target should persist");
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
let before = store.objects.lock().await.clone();
let revisions_before = store.revisions.lock().await.clone();
let entered = Arc::new(tokio::sync::Notify::new());
let resume = Arc::new(tokio::sync::Notify::new());
*store.pause_next_publication_admission.lock().await = Some((entered.clone(), resume.clone()));
let cancelled = CancellationToken::new();
let (result, ()) = tokio::join!(
reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled()),
async {
entered.notified().await;
cancelled.cancel();
resume.notify_one();
}
);
let err = result.expect_err("losing ownership during admission must prevent mutation");
assert!(err.to_string().contains("ownership was lost"), "resuming={resuming}: {err}");
assert_eq!(*store.objects.lock().await, before);
assert_eq!(*store.revisions.lock().await, revisions_before);
}
}
#[tokio::test]
#[serial]
async fn scanner_usage_state_reset_rejects_corruption_without_a_trusted_floor() {
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), b"{corrupt".to_vec())
.await
.expect("corrupt primary should persist");
let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("evidence should load");
let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
.await
.expect_err("corruption must not become a zero floor");
assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("evidence should remain"),
before
);
assert!(matches!(
read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await,
Err(EcstoreError::ConfigNotFound)
));
let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
backup.scanner_epoch = Some(7);
backup.scanner_cycle = Some(40);
save_config(
store.clone(),
&format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
serde_json::to_vec(&backup).expect("backup should encode"),
)
.await
.expect("valid backup should persist");
let result = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store)
.await
.expect("valid backup should supply the recovery floor");
assert_eq!(result.leader_epoch, 8);
assert_eq!(result.next_cycle, 41);
}
#[tokio::test]
async fn scanner_usage_state_reset_rejects_replaced_intent_and_newer_cleanup_slot() {
let store = Arc::new(MemoryConfigStore::default());
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3));
let bytes = serde_json::to_vec(&marker).expect("marker should encode");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
.await
.expect("intent should persist");
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes)
.await
.expect("another intent should persist");
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
.await
.expect_err("same epoch cannot replace an intent revision");
assert!(err.to_string().contains("intent revision changed"), "{err}");
let mut newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
newer.scanner_epoch = Some(3);
let path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
let bytes = serde_json::to_vec(&newer).expect("newer snapshot should encode");
save_config(store.clone(), &path, bytes.clone())
.await
.expect("newer snapshot should persist");
let slots = read_usage_state_reset_slots(store.clone())
.await
.expect("slots should reload");
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
.await
.expect_err("cleanup cannot delete same-epoch progress");
assert!(err.to_string().contains("not older than its intent"), "{err}");
assert_eq!(read_config(store, &path).await.expect("newer snapshot should remain"), bytes);
}
#[tokio::test]
#[serial]
async fn scanner_usage_state_reset_rejects_decodable_untrusted_floor() {
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
let invalid_identity = DataUsageInfo {
usage_snapshot_complete: true,
buckets_count: 1,
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
..Default::default()
};
for usage in [DataUsageInfo::default(), invalid_identity] {
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&usage).expect("fixture should encode"),
)
.await
.expect("untrusted primary should persist");
let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("primary should load");
let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
.await
.expect_err("valid JSON alone cannot prove a usage floor");
assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("evidence should remain"),
before
);
assert!(matches!(
read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await,
Err(EcstoreError::ConfigNotFound)
));
}
}
#[test]
fn full_rescan_reset_rejects_unknown_marker_phase_even_with_invalid_compat_fields() {
for state in [serde_json::json!("rewrite-v2"), serde_json::json!(7), serde_json::Value::Null] {
let marker = serde_json::json!({"state": state, "retry_count": "future-type", "schema_version": 99});
let err = super::cycle_state::decode_recovery_marker_for_reset(
&serde_json::to_vec(&marker).expect("future marker should encode"),
&DataUsageCacheRevision::Etag("intent-1".to_string()),
)
.expect_err("unknown persistent phases must remain fenced");
assert!(err.to_string().contains("state is unsupported"), "{err}");
}
}
#[tokio::test]
#[serial]
async fn full_rescan_reset_preserves_unknown_phase_and_retries_completed_cleanup() {
let (_temp_dir, store) = setup_scanner_cycle_store().await;
save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt".to_vec())
.await
.expect("corrupt primary should persist");
save_config(
store.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
br#"{"state":"future-rewrite"}"#.to_vec(),
)
.await
.expect("future marker should persist");
let primary_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("primary should load");
let marker_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
.await
.expect("marker should load");
let err = reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
.await
.expect_err("unknown phase must block explicit reset");
assert!(err.to_string().contains("state is unsupported"), "{err}");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("primary should remain"),
primary_before
);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
.await
.expect("marker should remain"),
marker_before
);
save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{malformed".to_vec())
.await
.expect("recoverable marker should persist");
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
.await
.expect("reset should complete");
let primary = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("rebuilt primary should load");
let usage = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("fenced usage should load");
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
.await
.expect("retry after marker deletion should complete");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("rebuilt primary should remain"),
primary
);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("fenced usage should remain"),
usage
);
}
#[tokio::test]
async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
let store = Arc::new(MemoryConfigStore::default());
@@ -4993,7 +5367,7 @@ async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
.expect("usage reset slots should be inspected");
store.publication_admission_blocked.store(true, Ordering::Release);
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3)
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true)
.await
.expect_err("movement admission loss must defer reset");
assert!(
@@ -8169,6 +8543,44 @@ fn scanner_activity_snapshot_digest_fences_dirty_usage_state() {
assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending));
}
#[test]
fn scanner_activity_structural_digest_ignores_regular_bucket_writes() {
let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
let mut written = baseline.clone();
let activity = written.get_mut("node-2").expect("node should exist");
activity.namespace_generation = 8;
activity.dirty_usage_generation = 6;
activity.dirty_usage_pending = true;
assert_ne!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&written));
assert_eq!(
scanner_activity_structural_digest(&baseline),
scanner_activity_structural_digest(&written),
"bucket writes are refreshed through the dirty-bucket scope rather than invalidating every cache"
);
}
#[test]
fn scanner_activity_structural_digest_fences_restart_and_maintenance() {
let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
let mut restarted = baseline.clone();
restarted.get_mut("node-2").expect("node should exist").instance_id = "epoch-b".to_string();
let mut maintained = baseline.clone();
maintained
.get_mut("node-2")
.expect("node should exist")
.maintenance_generation = 4;
assert_ne!(
scanner_activity_structural_digest(&baseline),
scanner_activity_structural_digest(&restarted)
);
assert_ne!(
scanner_activity_structural_digest(&baseline),
scanner_activity_structural_digest(&maintained)
);
}
#[test]
fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() {
let snapshot = BTreeMap::from([
crates/scanner/src/scanner_io.rs
+121 -3
View File
@@ -21,6 +21,7 @@ use crate::{
DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState,
ScannerError, SizeSummary, TierStats,
};
use bytes::Bytes;
use futures::future::join_all;
use metrics::counter;
use rand::seq::SliceRandom as _;
@@ -54,6 +55,7 @@ use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, error, warn};
use crate::ScannerObjectInfo as ObjectInfo;
use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
use crate::storage_api::ScannerStorage;
use crate::storage_api::scan::NamespaceLocking as _;
use crate::storage_api::scanner_io::{BucketInfo, BucketOptions};
@@ -111,6 +113,121 @@ pub(crate) struct ScannerBucketScanScope {
baseline_scan_plan_digest: Option<DataUsageScanPlanDigest>,
}
impl ScannerBucketScanScope {
fn is_default(&self) -> bool {
self.selected_buckets.is_none() && self.baseline_scan_plan_digest.is_none()
}
fn from_dirty_buckets(selected_buckets: HashSet<String>, baseline_scan_plan_digest: DataUsageScanPlanDigest) -> Self {
Self {
selected_buckets: Some(Arc::new(selected_buckets)),
baseline_scan_plan_digest: Some(baseline_scan_plan_digest),
}
}
}
#[derive(Clone, Copy)]
pub(super) struct ScannerCacheBaselineProof<'a> {
pub(super) data: Option<&'a Bytes>,
pub(super) expected_sources: &'a HashSet<DataUsageCacheSource>,
pub(super) leader_epoch: u64,
pub(super) want_cycle: u64,
pub(super) scan_plan_digest: DataUsageScanPlanDigest,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct ScannerPeerDirtyUsageExpectation {
instance_id: String,
generation: u64,
pending: bool,
}
fn verified_remote_dirty_usage_buckets(
expected_peers: &HashMap<String, ScannerPeerDirtyUsageExpectation>,
peer_snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>,
) -> Option<HashSet<String>> {
if expected_peers.is_empty() || peer_snapshots.len() != expected_peers.len() {
return None;
}
let mut received_peers = HashSet::with_capacity(peer_snapshots.len());
let mut dirty_buckets = HashSet::new();
for (host, snapshot) in peer_snapshots {
let expected = expected_peers.get(&host)?;
if !received_peers.insert(host)
|| snapshot.instance_id != expected.instance_id
|| snapshot.generation != expected.generation
|| snapshot.generation == u64::MAX
|| snapshot.protocol_version != crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION
|| !snapshot.complete
|| snapshot.pending_bucket_count != u64::try_from(snapshot.buckets.len()).unwrap_or(u64::MAX)
|| (expected.pending && snapshot.pending_bucket_count == 0)
{
return None;
}
dirty_buckets.extend(snapshot.buckets.into_keys());
}
(received_peers.len() == expected_peers.len()).then_some(dirty_buckets)
}
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
let data = proof.data?;
let baseline = serde_json::from_slice::<DataUsageInfo>(data).ok()?;
if !baseline.is_complete_bucket_usage_snapshot()
|| baseline.usage_snapshot_partial
|| baseline.usage_snapshot_converged != Some(true)
|| baseline.scanner_epoch != Some(proof.leader_epoch)
|| baseline.usage_snapshot_set_states.len() != proof.expected_sources.len()
{
return None;
}
let mut states = HashSet::with_capacity(baseline.usage_snapshot_set_states.len());
for state in &baseline.usage_snapshot_set_states {
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
if !proof.expected_sources.contains(&source)
|| !states.insert(source)
|| !state.complete
|| state.tombstone
|| state.scanner_epoch != Some(proof.leader_epoch)
|| state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle)
|| state.scan_plan_digest != Some(proof.scan_plan_digest.0)
{
return None;
}
}
(states == *proof.expected_sources).then_some(proof.scan_plan_digest)
}
fn scoped_scan_scope_from_dirty_buckets(
requested_scope: ScannerBucketScanScope,
dirty_buckets: HashSet<String>,
dirty_snapshot_complete: bool,
all_buckets: &[BucketInfo],
baseline_proof: ScannerCacheBaselineProof<'_>,
) -> ScannerBucketScanScope {
if !requested_scope.is_default() || !dirty_snapshot_complete {
return requested_scope;
}
let current_buckets = all_buckets.iter().map(|bucket| bucket.name.as_str()).collect::<HashSet<_>>();
let selected_buckets = dirty_buckets
.into_iter()
.filter(|bucket| current_buckets.contains(bucket.as_str()))
.collect::<HashSet<_>>();
if selected_buckets.is_empty() {
return requested_scope;
}
let Some(baseline_scan_plan_digest) = complete_scanner_cache_baseline_plan_digest(baseline_proof) else {
return requested_scope;
};
ScannerBucketScanScope::from_dirty_buckets(selected_buckets, baseline_scan_plan_digest)
}
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
}
@@ -749,7 +866,7 @@ mod io_cache;
mod io_cycle;
#[cfg(test)]
use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan};
pub(crate) use io_cycle::nsscanner_with_storage_status;
pub(crate) use io_cycle::{ScannerCycleRequest, nsscanner_with_storage_status_scoped};
mod io_disk;
#[cfg(test)]
mod publish_gate_tests;
@@ -766,8 +883,9 @@ pub(crate) use cache::{
};
pub use dirty_usage::{
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
scanner_maintenance_generation,
};
#[cfg(test)]
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
crates/scanner/src/scanner_io/cache.rs
+18
View File
@@ -282,9 +282,26 @@ pub(super) fn completed_data_usage_info(
.iter()
.map(|(bucket, usage)| (bucket.clone(), usage.size))
.collect();
let mut usage_snapshot_set_states = results
.iter()
.map(|result| {
let source = result.info.source?;
Some(DataUsageSnapshotSetState {
pool_index: u64::try_from(source.pool_index).ok()?,
set_index: u64::try_from(source.set_index).ok()?,
scanner_cycle: Some(result.info.next_cycle),
scanner_epoch: Some(result.info.leader_epoch),
scan_plan_digest: Some(result.info.scan_plan_digest?.0),
complete: true,
tombstone: false,
})
})
.collect::<Option<Vec<_>>>()?;
usage_snapshot_set_states.sort_by_key(|state| (state.pool_index, state.set_index));
let data_usage_info = DataUsageInfo {
last_update: Some(merged_last_update),
scanner_cycle: Some(results.first()?.info.next_cycle),
scanner_epoch: Some(results.first()?.info.leader_epoch),
objects_total_count: u64::try_from(total.objects).ok()?,
versions_total_count: u64::try_from(total.versions).ok()?,
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
@@ -295,6 +312,7 @@ pub(super) fn completed_data_usage_info(
bucket_sizes,
buckets_usage,
usage_snapshot_complete: true,
usage_snapshot_set_states,
..Default::default()
};
Some((data_usage_info, merged_last_update))
crates/scanner/src/scanner_io/dirty_usage.rs
+106
View File
@@ -52,6 +52,112 @@ pub enum ScannerDirtyUsageAckError {
ProcessChanged,
#[error("scanner dirty usage generation cannot be acknowledged")]
InvalidGeneration,
#[error("scanner dirty usage bucket incarnation fence is unavailable")]
IncarnationUnavailable,
}
/// A scoped ACK requires storage-owned lifecycle and incarnation fences.
/// Callers must only send ACKs backed by durable per-bucket publication.
pub fn acknowledge_scoped_dirty_usage(
instance_id: &str,
entries: &[(&crate::storage_api::EcstoreBucketMetadataMutationGuard, u64)],
probe_only: bool,
) -> std::result::Result<u64, ScannerDirtyUsageAckError> {
// Lock order: sorted bucket lifecycle/metadata fences (caller), then dirty map.
// No await or storage operation occurs while the dirty map is locked.
let (cleared, pending) = {
let mut dirty = dirty_usage_buckets();
let checked = entries
.iter()
.map(|(guard, generation)| {
guard
.checked_bucket_incarnation()
.map(|(bucket, _)| (bucket, *generation))
.map_err(|_| ScannerDirtyUsageAckError::IncarnationUnavailable)
})
.collect::<std::result::Result<Vec<_>, _>>()?;
let cleared = apply_scoped_dirty_usage_ack(
instance_id,
scanner_activity_epoch(),
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire),
&mut dirty,
&checked,
probe_only,
)?;
if cleared > 0 {
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
}
(cleared, dirty.len())
};
if !probe_only {
global_metrics().record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared), usize_to_u64_saturated(pending));
}
Ok(usize_to_u64_saturated(cleared))
}
fn apply_scoped_dirty_usage_ack(
instance_id: &str,
current_instance: &str,
current_generation: u64,
dirty: &mut DirtyUsageBuckets,
entries: &[(&str, u64)],
probe_only: bool,
) -> std::result::Result<usize, ScannerDirtyUsageAckError> {
if instance_id != current_instance {
return Err(ScannerDirtyUsageAckError::ProcessChanged);
}
if current_generation == u64::MAX
|| entries
.iter()
.any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation)
{
return Err(ScannerDirtyUsageAckError::InvalidGeneration);
}
let mut cleared = 0;
if !probe_only {
for (bucket, generation) in entries {
if dirty.get(*bucket) == Some(generation) {
dirty.remove(*bucket);
cleared += 1;
}
}
}
Ok(cleared)
}
#[cfg(test)]
mod scoped_dirty_usage_tests {
use super::*;
#[test]
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
let mut dirty = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], true), Ok(0));
assert_eq!(dirty.len(), 2);
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(1));
assert_eq!(dirty.get("hot"), Some(&7));
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(0));
dirty.insert("cold".to_string(), 9);
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 9, &mut dirty, &[("cold", 8)], false), Ok(0));
assert_eq!(dirty.get("cold"), Some(&9));
}
#[test]
fn scoped_dirty_usage_rejects_restart_and_invalid_batch_before_clearing() {
let original = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
let mut dirty = original.clone();
assert_eq!(
apply_scoped_dirty_usage_ack("old", "new", 8, &mut dirty, &[("cold", 8)], false),
Err(ScannerDirtyUsageAckError::ProcessChanged)
);
for generation in [0, 9, u64::MAX] {
assert_eq!(
apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8), ("hot", generation)], false),
Err(ScannerDirtyUsageAckError::InvalidGeneration)
);
assert_eq!(dirty, original);
}
}
}
pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
crates/scanner/src/scanner_io/io_cycle.rs
+94 -1
View File
@@ -71,6 +71,7 @@ where
leader_epoch,
scan_mode,
scan_scope: ScannerBucketScanScope::default(),
persisted_usage_baseline: None,
};
nsscanner_with_storage_status_scoped(store, request).await
}
@@ -83,6 +84,79 @@ pub(crate) struct ScannerCycleRequest {
pub(crate) leader_epoch: u64,
pub(crate) scan_mode: HealScanMode,
pub(crate) scan_scope: ScannerBucketScanScope,
pub(crate) persisted_usage_baseline: Option<Bytes>,
}
struct ScannerBucketScopeResolution<'a> {
requested_scope: ScannerBucketScanScope,
baseline_proof: ScannerCacheBaselineProof<'a>,
activity_before: &'a crate::scanner::ScannerActivitySnapshot,
dirty_usage_snapshot: &'a DirtyUsageSnapshot,
all_buckets: &'a [BucketInfo],
}
async fn resolve_scanner_bucket_scan_scope<S>(
store: &S,
distributed: bool,
resolution: ScannerBucketScopeResolution<'_>,
) -> ScannerBucketScanScope
where
S: ScannerStorage,
{
if !resolution.requested_scope.is_default()
|| !resolution.dirty_usage_snapshot.covers_all_pending
|| resolution.dirty_usage_snapshot.generation == u64::MAX
|| resolution.dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES
{
return resolution.requested_scope;
}
let mut dirty_buckets = resolution
.dirty_usage_snapshot
.buckets
.keys()
.cloned()
.collect::<HashSet<_>>();
if distributed {
let Some(notification_system) = store.scanner_notification_system() else {
return resolution.requested_scope;
};
let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else {
return resolution.requested_scope;
};
let mut expected_peers = HashMap::new();
for (host, lease_instance_id, _) in crate::scanner::scanner_activity_publication_lease_targets(resolution.activity_before)
{
let Some((activity_instance_id, generation, pending)) =
crate::scanner::scanner_activity_dirty_usage_state_for_host(resolution.activity_before, &host)
else {
return resolution.requested_scope;
};
if activity_instance_id != lease_instance_id || expected_peers.contains_key(&host) {
return resolution.requested_scope;
}
expected_peers.insert(
host,
ScannerPeerDirtyUsageExpectation {
instance_id: activity_instance_id.to_string(),
generation,
pending,
},
);
}
let Some(remote_dirty_buckets) = verified_remote_dirty_usage_buckets(&expected_peers, peer_snapshots) else {
return resolution.requested_scope;
};
dirty_buckets.extend(remote_dirty_buckets);
}
scoped_scan_scope_from_dirty_buckets(
resolution.requested_scope,
dirty_buckets,
true,
resolution.all_buckets,
resolution.baseline_proof,
)
}
pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
@@ -97,6 +171,7 @@ where
leader_epoch,
scan_mode,
scan_scope,
persisted_usage_baseline,
} = request;
let child_token = ctx.child_token();
let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch);
@@ -186,8 +261,26 @@ where
}
bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
let scan_plan_digest =
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before));
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
let scan_scope = resolve_scanner_bucket_scan_scope(
store,
distributed,
ScannerBucketScopeResolution {
requested_scope: scan_scope,
baseline_proof: ScannerCacheBaselineProof {
data: persisted_usage_baseline.as_ref(),
expected_sources: &expected_sources,
leader_epoch,
want_cycle,
scan_plan_digest,
},
activity_before: &activity_before,
dirty_usage_snapshot: &dirty_usage_snapshot,
all_buckets: &all_buckets,
},
)
.await;
let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle));
let tier_registry = runtime_tier_registry_for_cycle(want_cycle, leader_epoch).await;
let tier_registry_generation = tier_registry.generation;
crates/scanner/src/scanner_io/publish_gate_tests.rs
+22
View File
@@ -655,9 +655,31 @@ fn completed_data_usage_info_requires_every_set_before_publish() {
.expect("all completed sets should produce a publishable data usage snapshot");
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
assert_eq!(data_usage_info.scanner_cycle, Some(0));
assert_eq!(data_usage_info.scanner_epoch, Some(0));
assert_eq!(data_usage_info.objects_total_count, 3);
assert_eq!(data_usage_info.buckets_usage.len(), 3);
assert!(data_usage_info.usage_snapshot_complete);
assert_eq!(
data_usage_info
.usage_snapshot_set_states
.iter()
.map(|state| {
(
state.pool_index,
state.set_index,
state.scanner_cycle,
state.scanner_epoch,
state.scan_plan_digest,
state.complete,
state.tombstone,
)
})
.collect::<Vec<_>>(),
vec![
(0, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false),
(1, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false),
]
);
assert_eq!(
data_usage_info
.buckets_usage
crates/scanner/src/scanner_io/tests.rs
+190
View File
@@ -17,6 +17,7 @@ use super::io_disk::tier_stats_template;
use super::*;
use crate::scanner_budget::ScannerCycleBudgetConfig;
use crate::scanner_folder::ScannerItem;
use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
use crate::storage_api::owner::{
EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
};
@@ -796,6 +797,195 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa
cache
}
fn complete_usage_baseline(
source: DataUsageCacheSource,
scan_plan_digest: DataUsageScanPlanDigest,
scanner_cycle: u64,
scanner_epoch: u64,
) -> bytes::Bytes {
let baseline = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
scanner_cycle: Some(scanner_cycle),
scanner_epoch: Some(scanner_epoch),
buckets_count: 1,
buckets_usage: HashMap::from([("photos".to_string(), Default::default())]),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(true),
usage_snapshot_set_states: vec![DataUsageSnapshotSetState {
pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"),
set_index: u64::try_from(source.set_index).expect("test set index should fit"),
scanner_cycle: Some(scanner_cycle),
scanner_epoch: Some(scanner_epoch),
scan_plan_digest: Some(scan_plan_digest.0),
complete: true,
tombstone: false,
}],
..Default::default()
};
bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode"))
}
#[test]
fn scoped_scan_requires_a_converged_complete_baseline_with_exact_set_provenance() {
let source = DataUsageCacheSource::new(1, 2);
let expected_sources = HashSet::from([source]);
let scan_plan_digest = DataUsageScanPlanDigest([9; 32]);
let baseline = complete_usage_baseline(source, scan_plan_digest, 7, 11);
assert_eq!(
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
data: Some(&baseline),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest,
}),
Some(scan_plan_digest)
);
let mut incomplete = serde_json::from_slice::<DataUsageInfo>(&baseline).expect("test baseline should decode");
incomplete.usage_snapshot_converged = Some(false);
let incomplete = bytes::Bytes::from(serde_json::to_vec(&incomplete).expect("test baseline should encode"));
assert_eq!(
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
data: Some(&incomplete),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest,
}),
None
);
let mut wrong_provenance = serde_json::from_slice::<DataUsageInfo>(&baseline).expect("test baseline should decode");
wrong_provenance.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32]);
let wrong_provenance = bytes::Bytes::from(serde_json::to_vec(&wrong_provenance).expect("test baseline should encode"));
assert_eq!(
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
data: Some(&wrong_provenance),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest,
}),
None
);
}
#[test]
fn scoped_scan_selects_only_current_dirty_buckets_after_baseline_validation() {
let source = DataUsageCacheSource::new(1, 2);
let expected_sources = HashSet::from([source]);
let baseline_scan_plan_digest = DataUsageScanPlanDigest([4; 32]);
let current_scan_plan_digest = DataUsageScanPlanDigest([5; 32]);
let baseline = complete_usage_baseline(source, current_scan_plan_digest, 7, 11);
let scope = scoped_scan_scope_from_dirty_buckets(
ScannerBucketScanScope::default(),
HashSet::from(["photos".to_string(), "deleted".to_string()]),
true,
&[bucket_info("photos")],
ScannerCacheBaselineProof {
data: Some(&baseline),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest: current_scan_plan_digest,
},
);
assert_eq!(scope.baseline_scan_plan_digest, Some(current_scan_plan_digest));
assert_eq!(
scope
.selected_buckets
.as_deref()
.expect("validated scope should select a bucket"),
&HashSet::from(["photos".to_string()])
);
assert_ne!(scope.baseline_scan_plan_digest, Some(baseline_scan_plan_digest));
}
fn peer_dirty_usage_snapshot(
instance_id: &str,
generation: u64,
complete: bool,
buckets: &[(&str, u64)],
) -> EcstoreScannerPeerDirtyUsageSnapshot {
EcstoreScannerPeerDirtyUsageSnapshot {
instance_id: instance_id.to_string(),
generation,
pending_bucket_count: u64::try_from(buckets.len()).expect("test bucket count should fit"),
protocol_version: crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
complete,
buckets: buckets
.iter()
.map(|(bucket, generation)| ((*bucket).to_string(), *generation))
.collect(),
}
}
#[test]
fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots() {
let expected_peers = HashMap::from([
(
"node-a:9000".to_string(),
ScannerPeerDirtyUsageExpectation {
instance_id: "instance-a".to_string(),
generation: 7,
pending: true,
},
),
(
"node-b:9000".to_string(),
ScannerPeerDirtyUsageExpectation {
instance_id: "instance-b".to_string(),
generation: 3,
pending: false,
},
),
]);
assert_eq!(
verified_remote_dirty_usage_buckets(
&expected_peers,
vec![
(
"node-a:9000".to_string(),
peer_dirty_usage_snapshot("instance-a", 7, true, &[("photos", 7)]),
),
(
"node-b:9000".to_string(),
peer_dirty_usage_snapshot("instance-b", 3, true, &[("archive", 3)]),
),
],
),
Some(HashSet::from(["photos".to_string(), "archive".to_string()]))
);
}
#[test]
fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state() {
let expected_peers = HashMap::from([(
"node-a:9000".to_string(),
ScannerPeerDirtyUsageExpectation {
instance_id: "instance-a".to_string(),
generation: 7,
pending: true,
},
)]);
for snapshot in [
peer_dirty_usage_snapshot("instance-a", 7, false, &[("photos", 7)]),
peer_dirty_usage_snapshot("instance-a", 6, true, &[("photos", 6)]),
peer_dirty_usage_snapshot("instance-b", 7, true, &[("photos", 7)]),
peer_dirty_usage_snapshot("instance-a", 7, true, &[]),
] {
assert!(
verified_remote_dirty_usage_buckets(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(),
"incomplete, stale, mismatched, or empty pending peer state must fall back to a full scan"
);
}
}
#[test]
fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
crates/scanner/src/storage_api.rs
+5 -3
View File
@@ -38,8 +38,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::object_opts_fr
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
get_replication_config as ecstore_get_replication_config,
BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard, get_lifecycle_config as ecstore_get_lifecycle_config,
get_object_lock_config as ecstore_get_object_lock_config, get_replication_config as ecstore_get_replication_config,
};
pub(crate) use rustfs_ecstore::api::bucket::replication::{
ReplicateObjectInfo, ReplicationConfig as EcstoreReplicationConfig,
@@ -103,7 +103,9 @@ pub(crate) use rustfs_ecstore::api::rebalance::{
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
RebalanceStats as EcstoreRebalanceStats,
};
pub(crate) use rustfs_ecstore::api::rpc::ScannerBucketListing as EcstoreScannerBucketListing;
pub(crate) use rustfs_ecstore::api::rpc::{
ScannerBucketListing as EcstoreScannerBucketListing, ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
pub(crate) use rustfs_ecstore::api::runtime::{
crates/zip/Cargo.toml
+9
View File
@@ -49,5 +49,14 @@ rustfs-rio.workspace = true
tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
thiserror = { workspace = true }
[dev-dependencies]
astral-tokio-tar = { workspace = true }
futures = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sha2 = { workspace = true }
tar-codec = { workspace = true }
tar-framing = { workspace = true }
[lints]
workspace = true
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md
Vendored
+24
View File
@@ -0,0 +1,24 @@
# minio-go Snowball fixtures
These request bodies are generated by
`github.com/minio/minio-go/v7.Client.PutObjectsSnowball` at the version pinned
in `generate/go.mod`. They cover the raw TAR and S2-compressed forms accepted by
RustFS Snowball extraction.
The decoded TAR intentionally ends immediately after the final padded member
body because minio-go flushes, rather than closes, its TAR writer. The
compatibility test permits that shape only when the authenticated request body
is complete at the exact member boundary; it does not make incomplete TAR
terminators generally valid.
Regenerate them from this directory with Go 1.25:
```console
cd generate
go mod download
go run . -out ..
```
`manifest.json` records the input objects and SHA-256 digest of each captured
request body. Review changes to the manifest and binary fixtures together when
updating minio-go.
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod
Vendored
+26
View File
@@ -0,0 +1,26 @@
module rustfs.local/snowball-fixture
go 1.25.0
require github.com/minio/minio-go/v7 v7.3.0
require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.19.2 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.41.0 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
)
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum
Vendored
+59
View File
@@ -0,0 +1,59 @@
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU=
github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go
Vendored
+193
View File
@@ -0,0 +1,193 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
const minioGoVersion = "v7.3.0"
type fixtureManifest struct {
Generator string `json:"generator"`
MinioGo string `json:"minio_go"`
GeneratedAt string `json:"generated_at"`
Objects []fixtureObject `json:"objects"`
Archives []fixtureArchive `json:"archives"`
}
type fixtureObject struct {
Key string `json:"key"`
Body string `json:"body"`
ModTime string `json:"mod_time"`
VersionID string `json:"version_id,omitempty"`
Headers map[string][]string `json:"headers,omitempty"`
}
type fixtureArchive struct {
File string `json:"file"`
Compressed bool `json:"compressed"`
Length int `json:"length"`
SHA256 string `json:"sha256"`
}
func objects() []fixtureObject {
return []fixtureObject{
{
Key: "alpha.txt",
Body: "alpha-body",
ModTime: "2024-01-02T03:04:05Z",
VersionID: "018cc251-f400-7c22-9e8d-8b1800000001",
Headers: map[string][]string{
"Content-Type": {"text/plain"},
"X-Amz-Meta-Owner": {"snowball-fixture"},
"X-Amz-Tagging": {"project=rustfs&source=minio-go"},
},
},
{
Key: "nested/世界.txt",
Body: "bravo-body",
ModTime: "2024-01-02T03:05:05Z",
Headers: map[string][]string{
"Content-Language": {"zh-CN"},
"X-Amz-Meta-Note": {"unicode-path"},
},
},
}
}
func captureSnowball(compressed bool, specs []fixtureObject) ([]byte, error) {
body := make(chan []byte, 1)
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
payload, err := io.ReadAll(request.Body)
if err != nil {
http.Error(writer, err.Error(), http.StatusInternalServerError)
return
}
body <- payload
writer.Header().Set("ETag", `"snowball-fixture"`)
writer.WriteHeader(http.StatusOK)
}))
defer server.Close()
client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{
// The S3 authentication layer removes AWS streaming-signature framing
// before Snowball extraction sees the request body. Anonymous signing
// captures those decoded archive bytes directly.
Creds: credentials.NewStatic("", "", "", credentials.SignatureAnonymous),
Secure: false,
Region: "us-east-1",
})
if err != nil {
return nil, fmt.Errorf("construct minio client: %w", err)
}
input := make(chan minio.SnowballObject, len(specs))
for _, spec := range specs {
modTime, err := time.Parse(time.RFC3339, spec.ModTime)
if err != nil {
return nil, fmt.Errorf("parse mod time for %q: %w", spec.Key, err)
}
headers := make(http.Header, len(spec.Headers))
for name, values := range spec.Headers {
headers[name] = append([]string(nil), values...)
}
input <- minio.SnowballObject{
Key: spec.Key,
Size: int64(len(spec.Body)),
ModTime: modTime,
Content: bytes.NewReader([]byte(spec.Body)),
VersionID: spec.VersionID,
Headers: headers,
}
}
close(input)
err = client.PutObjectsSnowball(context.Background(), "fixture-bucket", minio.SnowballOptions{
Opts: minio.PutObjectOptions{
ContentType: "application/octet-stream",
},
InMemory: true,
Compress: compressed,
}, input)
if err != nil {
return nil, fmt.Errorf("generate snowball request: %w", err)
}
return <-body, nil
}
func main() {
outDir := flag.String("out", "..", "fixture output directory")
flag.Parse()
specs := objects()
archives := make([]fixtureArchive, 0, 2)
for _, fixture := range []struct {
name string
compressed bool
}{
{name: "snowball.tar"},
{name: "snowball.tar.s2", compressed: true},
} {
payload, err := captureSnowball(fixture.compressed, specs)
if err != nil {
panic(err)
}
path := filepath.Join(*outDir, fixture.name)
if err := os.WriteFile(path, payload, 0o644); err != nil {
panic(fmt.Errorf("write %s: %w", path, err))
}
digest := sha256.Sum256(payload)
archives = append(archives, fixtureArchive{
File: fixture.name,
Compressed: fixture.compressed,
Length: len(payload),
SHA256: hex.EncodeToString(digest[:]),
})
}
manifest := fixtureManifest{
Generator: "github.com/minio/minio-go/v7.Client.PutObjectsSnowball",
MinioGo: minioGoVersion,
GeneratedAt: "2026-09-05T00:00:00Z",
Objects: specs,
Archives: archives,
}
payload, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
panic(err)
}
payload = append(payload, '\n')
path := filepath.Join(*outDir, "manifest.json")
if err := os.WriteFile(path, payload, 0o644); err != nil {
panic(fmt.Errorf("write %s: %w", path, err))
}
}
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json
Vendored
+51
View File
@@ -0,0 +1,51 @@
{
"generator": "github.com/minio/minio-go/v7.Client.PutObjectsSnowball",
"minio_go": "v7.3.0",
"generated_at": "2026-09-05T00:00:00Z",
"objects": [
{
"key": "alpha.txt",
"body": "alpha-body",
"mod_time": "2024-01-02T03:04:05Z",
"version_id": "018cc251-f400-7c22-9e8d-8b1800000001",
"headers": {
"Content-Type": [
"text/plain"
],
"X-Amz-Meta-Owner": [
"snowball-fixture"
],
"X-Amz-Tagging": [
"project=rustfs\u0026source=minio-go"
]
}
},
{
"key": "nested/世界.txt",
"body": "bravo-body",
"mod_time": "2024-01-02T03:05:05Z",
"headers": {
"Content-Language": [
"zh-CN"
],
"X-Amz-Meta-Note": [
"unicode-path"
]
}
}
],
"archives": [
{
"file": "snowball.tar",
"compressed": false,
"length": 4096,
"sha256": "f00f2789dcb65b567f722f49cfdac9705e7bdac6c0badae75194327c32193d2e"
},
{
"file": "snowball.tar.s2",
"compressed": true,
"length": 528,
"sha256": "f8a9d9aa9b9ccdfae24ded1bff3741aacb935f1457a252efc9266674ff13c992"
}
]
}
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar
Vendored
BIN
View File
Binary file not shown.
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2
Vendored
BIN
View File
Binary file not shown.
crates/zip/tests/snowball_tar_codec_compat.rs
+548
View File
@@ -0,0 +1,548 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::io::Cursor;
use futures::StreamExt;
use rustfs_zip::CompressionFormat;
use serde::Deserialize;
use sha2::{Digest, Sha256};
use tar_codec::{Archive as _, DecodePolicy, Member, MemberPayload as _, PaxDecodePolicy, PaxVendorExtensionPolicy, TarArchive};
use tar_framing::{
FrameError, FrameErrorInner, PaxKeyword, PaxRecord, PaxValue, StreamPolicy, UstarKind,
logical::{MemberExtensions, PaxState, TarReader},
};
use tokio::io::AsyncReadExt;
const FIXTURE_ROOT: &str = "fixtures/snowball/minio-go-v7.3.0";
const RAW_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar");
const S2_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2");
const MANIFEST: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/manifest.json");
#[derive(Debug, Deserialize)]
struct FixtureManifest {
generator: String,
minio_go: String,
generated_at: String,
objects: Vec<FixtureObject>,
archives: Vec<FixtureArchive>,
}
#[derive(Debug, Deserialize)]
struct FixtureObject {
key: String,
body: String,
mod_time: String,
#[serde(default)]
version_id: String,
#[serde(default)]
headers: BTreeMap<String, Vec<String>>,
}
#[derive(Debug, Deserialize)]
struct FixtureArchive {
file: String,
compressed: bool,
length: usize,
sha256: String,
}
#[derive(Debug, Eq, PartialEq)]
struct ParsedMember {
path: String,
size: u64,
mtime: Option<u64>,
body: Vec<u8>,
minio_pax: BTreeMap<String, Option<Vec<u8>>>,
}
fn sha256_hex(bytes: &[u8]) -> String {
let mut encoded = String::with_capacity(64);
for byte in Sha256::digest(bytes) {
write!(&mut encoded, "{byte:02x}").expect("writing to a String should not fail");
}
encoded
}
async fn decode_s2(bytes: &[u8]) -> Vec<u8> {
let mut decoder = CompressionFormat::S2
.get_decoder(Cursor::new(bytes.to_vec()))
.expect("S2 fixture decoder should be available");
let mut decoded = Vec::new();
decoder.read_to_end(&mut decoded).await.expect("S2 fixture should decode");
decoded
}
async fn parse_with_tokio_tar(bytes: &[u8]) -> Vec<ParsedMember> {
let mut archive = tokio_tar::Archive::new(Cursor::new(bytes.to_vec()));
let mut entries = archive.entries().expect("tokio-tar should create an entry stream");
let mut parsed = Vec::new();
while let Some(entry) = entries.next().await {
let mut entry = entry.expect("tokio-tar should parse the fixture member");
let kind = entry.header().entry_type();
if kind == tokio_tar::EntryType::XGlobalHeader {
continue;
}
let path_bytes = entry.path_bytes().expect("tokio-tar should resolve the fixture path");
let path = std::str::from_utf8(path_bytes.as_ref())
.expect("fixture paths should be UTF-8")
.to_owned();
let size = entry.effective_size();
let mtime = entry.header().mtime().ok();
let mut minio_pax = BTreeMap::new();
if let Some(extensions) = entry
.pax_extensions()
.await
.expect("tokio-tar should parse local PAX records")
{
for extension in extensions {
let extension = extension.expect("fixture PAX record should be valid");
let key = extension.key().expect("fixture PAX keys should be UTF-8");
if key.starts_with("minio.") {
minio_pax.insert(key.to_owned(), Some(extension.value_bytes().to_vec()));
}
}
}
let mut body = Vec::new();
entry
.read_to_end(&mut body)
.await
.expect("tokio-tar should read the fixture body");
parsed.push(ParsedMember {
path,
size,
mtime,
body,
minio_pax,
});
}
parsed
}
fn effective_minio_pax(state: &PaxState<'_>, known_keywords: &mut Vec<PaxKeyword>) -> BTreeMap<String, Option<Vec<u8>>> {
for extension in state.extensions() {
for record in extension.records() {
let keyword = record.keyword();
if matches!(&keyword, PaxKeyword::Vendor { vendor, .. } if vendor.as_ref() == "minio")
&& !known_keywords.contains(&keyword)
{
known_keywords.push(keyword);
}
}
}
known_keywords
.iter()
.filter_map(|keyword| {
let record = state.effective_record(keyword)?;
let PaxRecord::Vendor { vendor, name, value } = record else {
return None;
};
let key = format!("{vendor}.{name}");
let value = match value {
PaxValue::Value(value) => Some(value.to_vec()),
PaxValue::Deleted => None,
};
Some((key, value))
})
.collect()
}
fn effective_mtime(header_mtime: Option<u64>, extensions: &MemberExtensions<'_>) -> Option<u64> {
let MemberExtensions::Pax(state) = extensions else {
return header_mtime;
};
match state.effective_record(&PaxKeyword::Mtime) {
Some(PaxRecord::Mtime(PaxValue::Value(value))) => Some(*value),
Some(PaxRecord::Mtime(PaxValue::Deleted)) => None,
_ => header_mtime,
}
}
fn padded_member_end(position: u64, size: u64) -> u64 {
let padded_size = size.checked_add(511).expect("fixture member size should not overflow") / 512 * 512;
position
.checked_add(512)
.and_then(|position| position.checked_add(padded_size))
.expect("fixture member end should not overflow")
}
fn is_authenticated_footerless_end(error: &FrameError, last_member_end: Option<u64>, request_body_complete: bool) -> bool {
// The production gate must source `request_body_complete` from RustFS's
// length, checksum, and trailing-header validation state.
request_body_complete && matches!(&error.inner, FrameErrorInner::MissingEndMarker) && last_member_end == Some(error.position)
}
fn candidate_snowball_decode_policy() -> DecodePolicy {
DecodePolicy::default()
.allow_gnu(true)
.allow_all_nul_numeric_fields(true)
.max_gnu_extension_size(1_048_576)
.pax_policy(
PaxDecodePolicy::default()
.max_extension_size(1_048_576)
.max_global_extensions_size(67_108_864)
.allow_global_pax_extensions(false)
.allow_non_utf8_pax_vendor_values(false)
.allow_duplicate_pax_records(false)
.allow_global_pax_member_metadata(false)
.vendor_extension_policy(PaxVendorExtensionPolicy::ignore(["minio"])),
)
}
async fn parse_with_tar_framing(bytes: &[u8]) -> (Vec<ParsedMember>, Option<FrameError>, Option<u64>) {
let policy = StreamPolicy::default()
.max_pax_extension_size(1024 * 1024)
.max_global_pax_extensions_size(4 * 1024 * 1024)
.max_gnu_extension_size(128 * 1024);
let mut reader = TarReader::new(Cursor::new(bytes.to_vec())).with_policy(policy);
let mut parsed = Vec::new();
let mut known_minio_keywords = Vec::new();
let mut last_member_end = None;
loop {
let mut frame = match reader.next_frame().await {
Ok(Some(frame)) => frame,
Ok(None) => return (parsed, None, last_member_end),
Err(error) => return (parsed, Some(error), last_member_end),
};
assert_eq!(frame.header.kind, UstarKind::Regular);
let path = String::from_utf8(
frame
.effective_path()
.expect("tar-framing should resolve the fixture path")
.into_owned(),
)
.expect("fixture paths should be UTF-8");
let size = frame.header.effective_size;
let mtime = effective_mtime(frame.header.mtime, &frame.extensions);
let minio_pax = match &frame.extensions {
MemberExtensions::Pax(state) => effective_minio_pax(state, &mut known_minio_keywords),
MemberExtensions::Gnu { .. } => BTreeMap::new(),
};
let mut body = Vec::new();
let mut chunk = Vec::new();
while frame
.payload
.next_chunk(&mut chunk, 64 * 1024)
.await
.expect("tar-framing should read the fixture body")
{
body.extend_from_slice(&chunk);
}
last_member_end = Some(padded_member_end(frame.header.position, size));
parsed.push(ParsedMember {
path,
size,
mtime,
body,
minio_pax,
});
}
}
#[test]
fn checked_in_fixtures_match_the_minio_go_manifest() {
let manifest: FixtureManifest = serde_json::from_slice(MANIFEST).expect("fixture manifest should be valid JSON");
assert_eq!(manifest.generator, "github.com/minio/minio-go/v7.Client.PutObjectsSnowball");
assert_eq!(manifest.minio_go, "v7.3.0");
assert_eq!(manifest.generated_at, "2026-09-05T00:00:00Z");
assert_eq!(manifest.objects.len(), 2);
assert_eq!(manifest.objects[0].key, "alpha.txt");
assert_eq!(manifest.objects[0].body, "alpha-body");
assert_eq!(manifest.objects[0].mod_time, "2024-01-02T03:04:05Z");
assert_eq!(manifest.objects[0].version_id, "018cc251-f400-7c22-9e8d-8b1800000001");
assert_eq!(
manifest.objects[0].headers.get("X-Amz-Meta-Owner"),
Some(&vec!["snowball-fixture".to_owned()])
);
for archive in &manifest.archives {
let bytes = match archive.file.as_str() {
"snowball.tar" => RAW_FIXTURE,
"snowball.tar.s2" => S2_FIXTURE,
file => panic!("unexpected archive in {FIXTURE_ROOT}/manifest.json: {file}"),
};
assert_eq!(bytes.len(), archive.length);
assert_eq!(sha256_hex(bytes), archive.sha256);
assert_eq!(archive.compressed, archive.file.ends_with(".s2"));
}
}
#[tokio::test]
async fn minio_go_raw_and_s2_fixtures_have_identical_footerless_tar_data() {
assert_eq!(decode_s2(S2_FIXTURE).await, RAW_FIXTURE);
assert_eq!(RAW_FIXTURE.len() % 512, 0);
assert!(RAW_FIXTURE.len() >= 1024);
assert!(
!RAW_FIXTURE[RAW_FIXTURE.len() - 1024..].iter().all(|byte| *byte == 0),
"minio-go Flush output should not contain the standard two-block terminator"
);
}
#[tokio::test]
async fn tar_framing_matches_tokio_tar_before_rejecting_the_missing_terminator() {
let expected = parse_with_tokio_tar(RAW_FIXTURE).await;
let (actual, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await;
let error = error.expect("footerless minio-go fixture should fail strict termination");
assert_eq!(actual, expected);
assert_eq!(
actual,
[
ParsedMember {
path: "alpha.txt".to_owned(),
size: 10,
mtime: Some(1_704_164_645),
body: b"alpha-body".to_vec(),
minio_pax: BTreeMap::from([
("minio.metadata.Content-Type".to_owned(), Some(b"text/plain".to_vec()),),
("minio.metadata.X-Amz-Meta-Owner".to_owned(), Some(b"snowball-fixture".to_vec()),),
(
"minio.metadata.X-Amz-Tagging".to_owned(),
Some(b"project=rustfs&source=minio-go".to_vec()),
),
("minio.versionId".to_owned(), Some(b"018cc251-f400-7c22-9e8d-8b1800000001".to_vec()),),
]),
},
ParsedMember {
path: "nested/世界.txt".to_owned(),
size: 10,
mtime: Some(1_704_164_705),
body: b"bravo-body".to_vec(),
minio_pax: BTreeMap::from([
("minio.metadata.Content-Language".to_owned(), Some(b"zh-CN".to_vec()),),
("minio.metadata.X-Amz-Meta-Note".to_owned(), Some(b"unicode-path".to_vec()),),
]),
},
]
);
assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker));
assert_eq!(
error.position,
u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64")
);
assert_eq!(last_member_end, Some(error.position));
}
#[tokio::test]
async fn footerless_compatibility_requires_authenticated_eof_at_the_member_boundary() {
let (_, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await;
let error = error.expect("the real fixture should be footerless");
assert!(is_authenticated_footerless_end(&error, last_member_end, true));
assert!(!is_authenticated_footerless_end(&error, last_member_end, false));
let mut one_zero_block = RAW_FIXTURE.to_vec();
one_zero_block.extend([0; 512]);
let (_, error, last_member_end) = parse_with_tar_framing(&one_zero_block).await;
let error = error.expect("one zero block is not a valid TAR terminator");
assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker));
assert_eq!(
last_member_end,
Some(u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64"))
);
assert_eq!(
error.position,
u64::try_from(one_zero_block.len()).expect("fixture length should fit in u64")
);
assert!(!is_authenticated_footerless_end(&error, last_member_end, true));
}
#[tokio::test]
async fn tar_codec_policy_accepts_only_the_explicit_minio_vendor_namespace() {
let default_error = match TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec())).members().next().await {
Err(error) => error,
Ok(_) => panic!("the default policy should reject minio vendor records"),
};
assert!(default_error.to_string().contains("pax vendor extension minio."));
let mut members = TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec()))
.with_policy(candidate_snowball_decode_policy())
.members();
let mut bodies = Vec::new();
loop {
let member = match members.next().await {
Ok(Some(member)) => member,
Ok(None) => panic!("footerless minio-go fixture should not report a valid archive end"),
Err(error) => {
assert!(error.to_string().contains("missing two-block end-of-archive marker"));
break;
}
};
let Member::File { mut payload, .. } = member else {
panic!("fixture should contain only regular files");
};
let mut body = Vec::new();
let mut chunk = Vec::new();
while payload
.next_chunk(&mut chunk, 64 * 1024)
.await
.expect("tar-codec should read the fixture body")
{
body.extend_from_slice(&chunk);
}
bodies.push(body);
}
assert_eq!(bodies, [b"alpha-body".to_vec(), b"bravo-body".to_vec()]);
assert!(
members
.next()
.await
.expect("the member cursor should be fused after an error")
.is_none()
);
}
fn pax_record(key: &str, value: &str) -> Vec<u8> {
let payload = format!("{key}={value}\n");
let mut len = payload.len() + 3;
loop {
let record = format!("{len} {payload}");
if record.len() == len {
return record.into_bytes();
}
len = record.len();
}
}
async fn append_pax_header(
builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
entry_type: tokio_tar::EntryType,
records: &[(&str, &str)],
) {
let mut payload = Vec::new();
for (key, value) in records {
payload.extend(pax_record(key, value));
}
let mut header = tokio_tar::Header::new_ustar();
header.set_entry_type(entry_type);
header.set_size(u64::try_from(payload.len()).expect("PAX test payload should fit in u64"));
header.set_mode(0o644);
header.set_cksum();
builder
.append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload))
.await
.expect("PAX test header should be written");
}
async fn append_regular(builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>, path: &str) {
let body = path.as_bytes();
let mut header = tokio_tar::Header::new_ustar();
header.set_entry_type(tokio_tar::EntryType::Regular);
header.set_size(u64::try_from(body.len()).expect("test member body should fit in u64"));
header.set_mode(0o644);
header.set_mtime(1_704_164_645);
header.set_cksum();
builder
.append_data(&mut header, path, Cursor::new(body))
.await
.expect("ordinary test member should be written");
}
async fn archive_with_local_pax(records: &[(&str, &str)]) -> Vec<u8> {
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, records).await;
append_regular(&mut builder, "member.txt").await;
builder.into_inner().await.expect("policy archive should finish").into_inner()
}
#[tokio::test]
async fn candidate_policy_rejects_unknown_vendor_and_duplicate_pax_records() {
let unknown_vendor = archive_with_local_pax(&[("acme.metadata.owner", "mallory")]).await;
let error = match TarArchive::new(Cursor::new(unknown_vendor))
.with_policy(candidate_snowball_decode_policy())
.members()
.next()
.await
{
Err(error) => error,
Ok(_) => panic!("the candidate Snowball policy should reject unknown vendors"),
};
assert!(
error
.to_string()
.contains("pax vendor extension acme.metadata.owner is not allowed")
);
let duplicate = archive_with_local_pax(&[
("minio.metadata.x-amz-meta-owner", "first"),
("minio.metadata.x-amz-meta-owner", "second"),
])
.await;
let error = match TarArchive::new(Cursor::new(duplicate))
.with_policy(candidate_snowball_decode_policy())
.members()
.next()
.await
{
Err(error) => error,
Ok(_) => panic!("the candidate Snowball policy should reject duplicate PAX records"),
};
assert!(
error
.to_string()
.contains("pax extended header contains duplicate record minio.metadata.x-amz-meta-owner")
);
}
#[tokio::test]
async fn global_minio_pax_inheritance_is_an_explicit_migration_difference() {
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
append_pax_header(
&mut builder,
tokio_tar::EntryType::XGlobalHeader,
&[("minio.metadata.x-amz-meta-owner", "global")],
)
.await;
append_pax_header(
&mut builder,
tokio_tar::EntryType::XHeader,
&[("minio.metadata.x-amz-meta-owner", "local")],
)
.await;
append_regular(&mut builder, "local.txt").await;
append_regular(&mut builder, "inherited.txt").await;
let archive = builder
.into_inner()
.await
.expect("precedence archive should finish")
.into_inner();
let legacy = parse_with_tokio_tar(&archive).await;
let (framing, error, _) = parse_with_tar_framing(&archive).await;
assert!(error.is_none());
assert_eq!(legacy.len(), 2);
assert_eq!(framing.len(), 2);
let owner_key = "minio.metadata.x-amz-meta-owner";
assert_eq!(legacy[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec())));
assert!(!legacy[1].minio_pax.contains_key(owner_key));
assert_eq!(framing[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec())));
assert_eq!(framing[1].minio_pax.get(owner_key), Some(&Some(b"global".to_vec())));
let error = match TarArchive::new(Cursor::new(archive))
.with_policy(candidate_snowball_decode_policy())
.members()
.next()
.await
{
Err(error) => error,
Ok(_) => panic!("the candidate Snowball policy should reject global PAX state"),
};
assert!(error.to_string().contains("global pax extended headers are not allowed"));
}
deny.toml
+2 -2
View File
@@ -37,8 +37,8 @@ unknown-git = "deny"
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
allow-git = [
# Temporary tokio-tar fork pinned to the reviewed parser limits,
# cancellation safety, and error-fusing change while
# astral-sh/tokio-tar#118 awaits an upstream release.
# cancellation safety, and error-fusing change while Snowball is
# prototyped against tar-codec and Swift retains its current reader.
# owner: cxymds review: 2026-10
"https://github.com/cxymds/tokio-tar.git",
# Official s3s repository. Temporarily pinned to the merged generic REST
docs/architecture/compat-cleanup-register.md
+1 -1
View File
@@ -13,7 +13,7 @@
- `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation.
- `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains.
- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release.
- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on precedence-resolved MinIO PAX metadata; per-entry and cumulative extension limits; a physical-entry limit; cancellation-safe parsing and ownership of large streamed members; fused streams after errors; and compatibility with minio-go streams that omit the two-block terminator. Swift bulk extraction also uses the same fork. Keep the reviewed pin while the Snowball path is prototyped against tar-codec/tar-framing. Remove it only after a released API exposes the effective allowed vendor records, RustFS provides a cancellation-safe handoff for borrowed member payloads, footerless input is accepted solely when authenticated request framing proves EOF immediately after a complete member, the existing resource-limit, cancellation, error-fuse, and real minio-go fixtures pass against the replacement, and Swift no longer depends on the fork.
- `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources.
- `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources.
- `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive.
rustfs/src/admin/handlers/heal.rs
+177 -20
View File
@@ -39,7 +39,7 @@ use rustfs_utils::path::path_join;
use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashSet};
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::future::Future;
use std::path::PathBuf;
use std::sync::Arc;
@@ -261,6 +261,7 @@ struct BackgroundHealStatus<'a> {
heal_active_tasks: u64,
heal_operations: rustfs_heal::HealOperationsSnapshot,
cluster_status_complete: bool,
coverage: &'a BackgroundHealCoverage,
#[serde(skip_serializing_if = "Option::is_none")]
progress: Option<BackgroundHealProgress>,
}
@@ -300,6 +301,23 @@ fn background_heal_runtime_state(
type BackgroundHealProgress = rustfs_heal::HealProgress;
#[derive(Debug, Serialize)]
struct BackgroundHealCoverage {
expected: usize,
responded: usize,
unknown: usize,
reasons: BTreeSet<BackgroundHealCoverageReason>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
#[serde(rename_all = "snake_case")]
enum BackgroundHealCoverageReason {
NotificationSystemUnavailable,
PeerTopologyIncomplete,
PeerStatusUnsupported,
PeerStatusUnavailable,
}
#[derive(Debug)]
struct ClusterHealStatusSnapshot {
info: BackgroundHealInfo,
@@ -307,6 +325,7 @@ struct ClusterHealStatusSnapshot {
operations: rustfs_heal::HealOperationsSnapshot,
progress: Option<BackgroundHealProgress>,
complete: bool,
coverage: BackgroundHealCoverage,
}
fn add_priority_counts(total: &mut rustfs_heal::HealPriorityCounts, next: rustfs_heal::HealPriorityCounts) {
@@ -338,6 +357,7 @@ fn add_operations(total: &mut rustfs_heal::HealOperationsSnapshot, next: rustfs_
}
fn aggregate_cluster_heal_status(snapshots: Vec<NodeHealStatusSnapshot>) -> ClusterHealStatusSnapshot {
let responded = snapshots.len();
let mut info = BackgroundHealInfo::default();
let mut operations = rustfs_heal::HealOperationsSnapshot::default();
let mut progress = Vec::new();
@@ -379,6 +399,12 @@ fn aggregate_cluster_heal_status(snapshots: Vec<NodeHealStatusSnapshot>) -> Clus
operations,
progress,
complete: true,
coverage: BackgroundHealCoverage {
expected: responded,
responded,
unknown: 0,
reasons: BTreeSet::new(),
},
}
}
@@ -413,12 +439,14 @@ fn merge_peer_heal_statuses(
mut snapshots: Vec<NodeHealStatusSnapshot>,
peer_statuses: Vec<Result<Option<NodeHealStatusSnapshot>, String>>,
expected_nodes: usize,
topology_complete: bool,
coverage_reason: Option<BackgroundHealCoverageReason>,
) -> S3Result<ClusterHealStatusSnapshot> {
let mut reasons: BTreeSet<_> = coverage_reason.into_iter().collect();
for peer_status in peer_statuses {
match peer_status {
Ok(Some(snapshot)) => snapshots.push(snapshot),
Ok(None) => {
reasons.insert(BackgroundHealCoverageReason::PeerStatusUnsupported);
warn!(
event = EVENT_ADMIN_REQUEST_FAILED,
component = LOG_COMPONENT_ADMIN_API,
@@ -430,6 +458,7 @@ fn merge_peer_heal_statuses(
);
}
Err(err) => {
reasons.insert(BackgroundHealCoverageReason::PeerStatusUnavailable);
warn!(
event = EVENT_ADMIN_REQUEST_FAILED,
component = LOG_COMPONENT_ADMIN_API,
@@ -452,9 +481,12 @@ fn merge_peer_heal_statuses(
// so during a reconfiguration the count can equal `expected_nodes` while
// the topology is known-incomplete. Counting alone would report a
// definitive answer precisely when the membership itself is in doubt.
let complete = topology_complete && snapshots.len() == expected_nodes;
let complete = reasons.is_empty() && snapshots.len() == expected_nodes;
let mut status = aggregate_cluster_heal_status(snapshots);
status.complete = complete;
status.coverage.expected = expected_nodes;
status.coverage.unknown = expected_nodes.saturating_sub(status.coverage.responded);
status.coverage.reasons = reasons;
// A partial answer must never be mistakable for a definitive verdict: an
// unreachable peer might be mid-heal, so reporting the reachable nodes'
// "idle" (or disabled/uninitialized) as the cluster state would falsely
@@ -497,7 +529,12 @@ async fn read_cluster_heal_status(
return Ok(aggregate_cluster_heal_status(snapshots));
}
let Some(notification_system) = notification_system else {
return Err(cluster_heal_status_unavailable("notification_system_unavailable"));
return merge_peer_heal_statuses(
snapshots,
Vec::new(),
expected_nodes,
Some(BackgroundHealCoverageReason::NotificationSystemUnavailable),
);
};
// An incomplete peer topology (a down member's client slot, a rolling
// upgrade) previously failed the whole endpoint here, before any peer was
@@ -540,7 +577,12 @@ async fn read_cluster_heal_status(
}))
.await;
merge_peer_heal_statuses(snapshots, peer_statuses, expected_nodes, topology_complete)
merge_peer_heal_statuses(
snapshots,
peer_statuses,
expected_nodes,
(!topology_complete).then_some(BackgroundHealCoverageReason::PeerTopologyIncomplete),
)
}
async fn query_peer_replacement_recovery_status<E>(
@@ -1164,6 +1206,7 @@ fn encode_background_heal_status(
heal_operations: rustfs_heal::HealOperationsSnapshot,
progress: Option<BackgroundHealProgress>,
cluster_status_complete: bool,
coverage: &BackgroundHealCoverage,
) -> S3Result<Vec<u8>> {
let status = BackgroundHealStatus {
info,
@@ -1172,6 +1215,7 @@ fn encode_background_heal_status(
heal_active_tasks: heal_operations.active_tasks,
heal_operations,
cluster_status_complete,
coverage,
progress,
};
serde_json::to_vec(&status).map_err(|e| {
@@ -1461,6 +1505,7 @@ impl Operation for BackgroundHealStatusHandler {
cluster_status.operations,
cluster_status.progress,
cluster_status.complete,
&cluster_status.coverage,
)?;
info!(
event = EVENT_ADMIN_RESPONSE_EMITTED,
@@ -1515,13 +1560,14 @@ impl Operation for ReplacementRecoveryStatusHandler {
mod tests {
use super::extract_heal_init_params;
use super::{
BackgroundHealProgress, HealInitParams, HealResp, HealRuntimeState, aggregate_cluster_heal_status,
aggregate_replacement_recovery_cluster_status, background_heal_runtime_state, build_heal_channel_request,
build_replacement_recovery_status_response, encode_background_heal_status, encode_heal_control_path,
encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability, heal_channel_response_items,
heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id, json_response,
map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status,
query_peer_replacement_recovery_status, reject_heal_admission, validate_heal_request_mode, validate_heal_target,
BackgroundHealCoverage, BackgroundHealCoverageReason, BackgroundHealProgress, HealInitParams, HealResp, HealRuntimeState,
aggregate_cluster_heal_status, aggregate_replacement_recovery_cluster_status, background_heal_runtime_state,
build_heal_channel_request, build_replacement_recovery_status_response, encode_background_heal_status,
encode_heal_control_path, encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability,
heal_channel_response_items, heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id,
json_response, map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status,
query_peer_replacement_recovery_status, read_cluster_heal_status, reject_heal_admission, validate_heal_request_mode,
validate_heal_target,
};
use crate::storage::rpc::node_service::heal::{
NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status,
@@ -2175,7 +2221,13 @@ mod tests {
..Default::default()
};
let encoded = encode_background_heal_status(&info, HealRuntimeState::Active, operations, None, true)
let coverage = BackgroundHealCoverage {
expected: 1,
responded: 1,
unknown: 0,
reasons: Default::default(),
};
let encoded = encode_background_heal_status(&info, HealRuntimeState::Active, operations, None, true, &coverage)
.expect("background heal info should serialize");
let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize");
@@ -2225,6 +2277,12 @@ mod tests {
rustfs_heal::HealOperationsSnapshot::default(),
Some(progress),
true,
&BackgroundHealCoverage {
expected: 1,
responded: 1,
unknown: 0,
reasons: Default::default(),
},
)
.expect("background heal info should serialize");
let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize");
@@ -2398,6 +2456,84 @@ mod tests {
assert!(peer_topology_complete(1, 0, 0, 1, 0));
}
#[tokio::test]
async fn test_background_heal_status_without_notification_preserves_local_snapshot() {
let initialized = rustfs_heal::heal_runtime_initialized();
let info = BackgroundHealInfo {
bitrot_start_cycle: 37,
current_scan_mode: HealScanMode::Deep,
..Default::default()
};
for expected in [1, 3] {
let status = tokio::time::timeout(Duration::from_secs(1), read_cluster_heal_status(info.clone(), None, expected))
.await
.expect("local status must not wait for remote peers")
.expect("missing notification must retain the local snapshot");
assert_eq!(status.info.bitrot_start_cycle, 37);
assert_eq!(status.info.current_scan_mode, HealScanMode::Deep);
assert_eq!(status.complete, expected == 1);
assert_eq!(status.coverage.expected, expected);
assert_eq!(status.coverage.responded, 1);
assert_eq!(status.coverage.unknown, expected - 1);
if expected == 1 {
assert!(status.coverage.reasons.is_empty());
} else {
assert!(matches!(status.state, HealRuntimeState::Degraded | HealRuntimeState::Active));
assert_eq!(
status.coverage.reasons,
[BackgroundHealCoverageReason::NotificationSystemUnavailable].into()
);
}
let encoded = encode_background_heal_status(
&status.info,
status.state,
status.operations,
status.progress,
status.complete,
&status.coverage,
)
.expect("fallback status must encode");
let decoded: rustfs_madmin::client::BackgroundHealStatus =
serde_json::from_slice(&encoded).expect("the actual madmin client must decode the server response");
assert_eq!(decoded.cluster_status_complete, expected == 1);
let coverage = decoded.coverage.expect("new server supplies coverage");
assert_eq!(coverage.expected, Some(expected));
assert_eq!(coverage.responded, Some(1));
assert_eq!(coverage.unknown, Some(expected - 1));
if expected > 1 {
assert_eq!(coverage.reasons, ["notification_system_unavailable"]);
}
}
assert_eq!(
rustfs_heal::heal_runtime_initialized(),
initialized,
"reading status must not initialize heal"
);
}
#[test]
fn test_background_heal_status_coverage_reasons_are_bounded() {
let local = NodeHealStatusSnapshot::for_test(true, true, BackgroundHealInfo::default(), Default::default(), None);
let peers = (0..100)
.map(|index| {
if index % 2 == 0 {
Ok(None)
} else {
Err("peer unavailable".to_owned())
}
})
.collect();
let status = merge_peer_heal_statuses(vec![local], peers, 101, None).expect("local status remains available");
assert_eq!(status.coverage.responded, 1);
assert_eq!(status.coverage.unknown, 100);
assert_eq!(status.coverage.reasons.len(), 2);
let encoded = serde_json::to_vec(&status.coverage).expect("coverage encodes");
assert!(encoded.len() < 256, "coverage must not grow with peer failures");
let decoded: rustfs_madmin::client::BackgroundHealCoverage =
serde_json::from_slice(&encoded).expect("client coverage decodes");
assert_eq!(decoded.reasons, ["peer_status_unsupported", "peer_status_unavailable"]);
}
#[test]
fn test_peer_status_merge_degrades_explicitly_and_never_claims_idle() {
let local = || {
@@ -2413,15 +2549,20 @@ mod tests {
// but the safety property of the previous fail-closed behaviour is
// preserved: the partial answer is labelled Degraded, never Idle, so
// unknown peer work cannot be mistaken for "nothing is running".
let partial = merge_peer_heal_statuses(vec![local()], vec![Err("peer timeout".to_string())], 2, true)
let partial = merge_peer_heal_statuses(vec![local()], vec![Err("peer timeout".to_string())], 2, None)
.expect("an unreachable peer degrades the answer instead of destroying it");
assert!(!partial.complete);
assert_eq!(partial.state, HealRuntimeState::Degraded);
assert_eq!(partial.coverage.expected, 2);
assert_eq!(partial.coverage.responded, 1);
assert_eq!(partial.coverage.unknown, 1);
assert_eq!(partial.coverage.reasons, [BackgroundHealCoverageReason::PeerStatusUnavailable].into());
let older_peer = merge_peer_heal_statuses(vec![local()], vec![Ok(None)], 2, true)
let older_peer = merge_peer_heal_statuses(vec![local()], vec![Ok(None)], 2, None)
.expect("an older peer degrades the answer instead of destroying it");
assert!(!older_peer.complete);
assert_eq!(older_peer.state, HealRuntimeState::Degraded);
assert_eq!(older_peer.coverage.reasons, [BackgroundHealCoverageReason::PeerStatusUnsupported].into());
let known_active = NodeHealStatusSnapshot::for_test(
true,
@@ -2433,12 +2574,12 @@ mod tests {
},
None,
);
let partial_active = merge_peer_heal_statuses(vec![known_active], vec![Ok(None)], 2, true)
let partial_active = merge_peer_heal_statuses(vec![known_active], vec![Ok(None)], 2, None)
.expect("known active work may be reported as an explicit partial status");
assert!(!partial_active.complete);
assert_eq!(partial_active.state, HealRuntimeState::Active);
merge_peer_heal_statuses(Vec::new(), vec![Err("peer timeout".to_string())], 2, true)
merge_peer_heal_statuses(Vec::new(), vec![Err("peer timeout".to_string())], 2, None)
.expect_err("no snapshot at all still fails closed");
}
@@ -2459,12 +2600,22 @@ mod tests {
None,
)
};
let full_count_incomplete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, false)
.expect("incomplete topology degrades the answer instead of destroying it");
let full_count_incomplete_topology = merge_peer_heal_statuses(
vec![snapshot()],
vec![Ok(Some(snapshot()))],
2,
Some(BackgroundHealCoverageReason::PeerTopologyIncomplete),
)
.expect("incomplete topology degrades the answer instead of destroying it");
assert!(!full_count_incomplete_topology.complete);
assert_eq!(full_count_incomplete_topology.state, HealRuntimeState::Degraded);
assert_eq!(full_count_incomplete_topology.coverage.unknown, 0);
assert_eq!(
full_count_incomplete_topology.coverage.reasons,
[BackgroundHealCoverageReason::PeerTopologyIncomplete].into()
);
let full_count_complete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, true)
let full_count_complete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, None)
.expect("complete topology and full count is a definitive answer");
assert!(full_count_complete_topology.complete);
assert_eq!(full_count_complete_topology.state, HealRuntimeState::Idle);
@@ -2478,6 +2629,12 @@ mod tests {
rustfs_heal::HealOperationsSnapshot::default(),
None,
false,
&BackgroundHealCoverage {
expected: 2,
responded: 1,
unknown: 1,
reasons: [BackgroundHealCoverageReason::PeerStatusUnavailable].into(),
},
)
.expect("degraded status must serialize");
let json: serde_json::Value = serde_json::from_slice(&encoded).expect("valid json");
rustfs/src/admin/handlers/site_replication.rs
+418 -244
View File
@@ -190,7 +190,8 @@ fn site_replicator_service_account_policy() -> S3Result<Policy> {
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("parse site replicator policy failed: {e}")))
}
// Lock order: lifecycle -> bucket operation -> repair admission -> state -> per-bucket metadata.
// Lock order: lifecycle -> bucket-mutation admission -> per-bucket mutation
// -> bucket operation -> repair admission -> state -> per-bucket metadata.
// "state" is the distributed state-object lock in
// crate::site_replication::state_lock, entered through
// update_site_replication_state (P1-15). There is no process-local state
@@ -434,6 +435,7 @@ pub fn register_site_replication_route(r: &mut S3Router<AdminOperation>) -> std:
// into this module: startup sits below this layer and must not depend upwards. The admin
// router is built before startup reconciles, so the hook is always installed in time.
crate::site_replication_reconcile::register_site_replication_reconciler(reconcile_site_replication_wiring);
crate::site_replication_reconcile::register_site_replication_retry_drainer(reconcile_site_replication_retry_drain);
for (method, path, operation) in [
(Method::PUT, "/v3/site-replication/add", AdminOperation(&SiteReplicationAddHandler {})),
@@ -1803,28 +1805,61 @@ async fn reconcile_site_replication_buckets() -> S3Result<()> {
/// (`SiteReplicationEditHandler`), so a tick landing between them would rewrite the targets
/// from the stale endpoint. The pending marker in the persisted state closes that window.
/// Skipping costs nothing — the timer comes back.
async fn site_replication_reconcile_prerequisites_ready() -> bool {
if current_iam_handle().is_none() || current_object_store_handle().is_none() {
return false;
}
if let Err(err) = migrate_collapsed_retry_queue_paths().await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "retry_queue_migration_failed",
error = ?err,
"admin site replication state"
);
return false;
}
true
}
fn reconcile_site_replication_retry_drain() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
Box::pin(async {
let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
return;
};
if !site_replication_reconcile_prerequisites_ready().await {
return;
}
match load_site_replication_state().await {
Ok(state) => {
if state.pending_endpoint_refresh.is_some() || state.pending_rotation.is_some() || state.pending_remove.is_some()
{
return;
}
}
Err(_) => return,
}
// Admission above observes a lifecycle-stable state. The lightweight
// drain itself handles only idempotent bucket setup, reloads state
// under the distributed repair lock, and shares that lock with bucket
// deletion. Do not hold this process-local guard across peer I/O: an
// outage recovery must not make admin add/edit/remove time out.
drop(lifecycle);
drain_site_replication_retry_queue_lightweight().await;
})
}
fn reconcile_site_replication_wiring() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
Box::pin(async {
// The scheduler starts before IAM and the object store are guaranteed ready (IAM
// bootstrap may still be recovering), so an early tick returns quietly instead of
// logging a failure for every reconciler.
if current_iam_handle().is_none() || current_object_store_handle().is_none() {
return;
}
let Some(_lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
return;
};
if let Err(err) = migrate_collapsed_retry_queue_paths().await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "retry_queue_migration_failed",
error = ?err,
"admin site replication state"
);
if !site_replication_reconcile_prerequisites_ready().await {
return;
}
@@ -1878,8 +1913,9 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin<Box<dyn std::future::Fut
"admin site replication state"
);
}
// Failed peer deliveries recorded in the retry queue; runs behind the
// same lifecycle guard and pending_* gates as the reconcilers above.
// The retry path re-checks membership from distributed state before
// each request; release lifecycle before a potentially large replay.
drop(lifecycle);
drain_site_replication_retry_queue().await;
})
}
@@ -3046,6 +3082,7 @@ fn set_pending_endpoint_refresh(state: &mut SiteReplicationState, pending: Pendi
last_error: "endpoint target refresh pending".to_string(),
updated_at: Some(OffsetDateTime::now_utc()),
edit_generation: None,
peer_unreachable: false,
deletions_recorded: false,
});
state.pending_endpoint_refresh = Some(pending);
@@ -3592,16 +3629,15 @@ const PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS: u64 = 24 * 60 * 60 * 1_000_000_000
/// must be a site this state currently replicates with — the same membership
/// rule the load-time mark pruning applies, so every mark recorded behind
/// this check is one a reload would keep — and not this site itself, which
/// never delivers edits to itself. The caller IGNORES an inadmissible fence
/// rather than failing the request: the delivery applies exactly as an
/// unstamped (pre-fence) delivery would, no high-water mark is read or
/// written, and the worst a forged fence achieves is forfeiting an ordering
/// guarantee its sender was never owed. The generation itself is NOT
/// bounded here: a genuine origin whose hybrid clock persisted a wall-clock
/// excursion allocates arbitrarily far in the future, and refusing to
/// record its marks would strip the ordering fence from exactly the
/// deliveries that still race — the staleness window on the read side is
/// what defuses forged marks instead.
/// never delivers edits to itself. The caller acknowledges an inadmissible
/// fenced request without applying it: after a remove commits, an older
/// in-flight retry from the departed origin must not recreate topology. Old
/// peers remain compatible because their unstamped edits still follow the
/// pre-fence path. The generation itself is NOT bounded here: a genuine
/// origin whose hybrid clock persisted a wall-clock excursion allocates
/// arbitrarily far in the future, and refusing to record its marks would
/// strip the ordering fence from exactly the deliveries that still race —
/// the staleness window on the read side is what defuses forged marks instead.
fn peer_edit_fence_is_admissible(state: &SiteReplicationState, local_deployment_id: &str, fence: &(String, u64)) -> bool {
let (origin, generation) = fence;
if origin != local_deployment_id && state.peers.contains_key(origin) {
@@ -4791,105 +4827,135 @@ async fn backfill_existing_buckets_after_add(
let resync_id = Uuid::new_v4().to_string();
for bucket in &buckets {
let name = &bucket.name;
let operation_name = bucket.name.clone();
let lock_bucket = operation_name.clone();
let operation_state = state.clone();
let operation_local_peer = local_peer.clone();
let operation_resync_id = resync_id.clone();
let operation_bootstrap_token = bootstrap_token.map(str::to_owned);
let bucket_errors = with_site_replication_bucket_mutation_lock(store.clone(), &lock_bucket, move || async move {
let mut errors = SiteReplicationErrorSummary::default();
let name = &operation_name;
if let Err(err) = ensure_site_replication_bucket_versioning(name).await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_versioning_setup_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name}: versioning setup failed: {err}"));
continue;
}
match ensure_site_replication_bucket_setup(name).await {
Ok(true) => {}
Ok(false) => {
// Runtime targets unavailable: the setup silently no-ops, which would make the
// downstream make-bucket broadcast and resync fail. Record it and skip so the
// operator sees this bucket was not propagated instead of an unqualified success.
if let Err(err) = ensure_site_replication_bucket_versioning(name).await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_setup_skipped",
"admin site replication state"
);
errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)"));
continue;
}
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_setup_failed",
result = "backfill_versioning_setup_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name}: bucket setup failed: {err}"));
errors.push(format!("{name}: versioning setup failed: {err}"));
return errors;
}
}
// Broadcast the bucket to peers so they create it too (idempotent on the peer side).
// Read the real lock_enabled flag so peers recreate the bucket with the same object-lock
// setting — object lock cannot be added after bucket creation.
let lock_enabled = match metadata_sys::get(name).await {
Ok(bm) => bm.lock_enabled,
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_metadata_read_failed",
fallback = "lock_enabled=false",
error = ?err,
"admin site replication state"
);
false
match ensure_site_replication_bucket_setup(name).await {
Ok(true) => {}
Ok(false) => {
// Runtime targets unavailable: the setup silently no-ops, which would make the
// downstream make-bucket broadcast and resync fail. Record it and skip so the
// operator sees this bucket was not propagated instead of an unqualified success.
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_setup_skipped",
"admin site replication state"
);
errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)"));
return errors;
}
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_setup_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name}: bucket setup failed: {err}"));
}
}
};
if let Err(err) = broadcast_site_replication_make_bucket(name, lock_enabled, None, bootstrap_token).await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_make_bucket_broadcast_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name}: make-bucket broadcast failed: {err}"));
}
// Kick a resync toward every remote peer so existing objects travel across.
for peer in state.peers.values() {
if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) {
continue;
}
let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await;
let result = if manifest.target_arn.is_empty() {
manifest
} else {
start_site_bucket_resync(name, &manifest.target_arn, &resync_id).await
// Broadcast the bucket to peers so they create it too (idempotent on the peer side).
// Read the real lock_enabled flag so peers recreate the bucket with the same object-lock
// setting — object lock cannot be added after bucket creation.
let lock_enabled = match metadata_sys::get(name).await {
Ok(bm) => bm.lock_enabled,
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_metadata_read_failed",
fallback = "lock_enabled=false",
error = ?err,
"admin site replication state"
);
false
}
};
if result.status == "failed" {
if let Err(err) =
broadcast_site_replication_make_bucket(name, lock_enabled, None, operation_bootstrap_token.as_deref()).await
{
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
peer = %peer.endpoint,
result = "backfill_resync_kick_failed",
detail = %result.err_detail,
result = "backfill_make_bucket_broadcast_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail));
errors.push(format!("{name}: make-bucket broadcast failed: {err}"));
}
// Kick a resync toward every remote peer so existing objects travel across.
for peer in operation_state.peers.values() {
if peer.deployment_id == operation_local_peer.deployment_id
|| same_identity_endpoint(&peer.endpoint, &operation_local_peer.endpoint)
{
continue;
}
let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await;
let result = if manifest.target_arn.is_empty() {
manifest
} else {
start_site_bucket_resync(name, &manifest.target_arn, &operation_resync_id).await
};
if result.status == "failed" {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
peer = %peer.endpoint,
result = "backfill_resync_kick_failed",
detail = %result.err_detail,
"admin site replication state"
);
errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail));
}
}
errors
})
.await;
match bucket_errors {
Ok(bucket_errors) => errors.extend(bucket_errors),
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %lock_bucket,
result = "backfill_bucket_mutation_lock_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{lock_bucket}: bucket mutation lock failed: {err}"));
}
}
}
@@ -6072,146 +6138,204 @@ fn parse_peer_join_response(body: &[u8], fallback_peer: PeerInfo) -> Result<SRPe
serde_json::from_slice(body)
}
fn ensure_add_bucket_set_matches_preflight(expected: &HashSet<String>, present: &HashSet<String>) -> S3Result<()> {
let mut missing = expected.difference(present).cloned().collect::<Vec<_>>();
if !missing.is_empty() {
missing.sort_unstable();
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!(
"bucket `{}` disappeared while site replication was being added; peers may already be joined — re-run replicate add",
missing[0]
),
));
}
let mut unexpected = present.difference(expected).cloned().collect::<Vec<_>>();
if !unexpected.is_empty() {
unexpected.sort_unstable();
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!(
"bucket `{}` appeared while site replication was being added; peers may already be joined — re-run replicate add",
unexpected[0]
),
));
}
Ok(())
}
#[async_trait::async_trait]
impl Operation for SiteReplicationAddHandler {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?;
reject_site_replicator_on_public_admin(&cred)?;
let replicate_ilm_expiry = sr_add_replicate_ilm_expiry(&req.uri);
let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers);
let lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?;
// Everything up to the commit below is preflight: peer probes, IAM
// work and the join fan-out all talk to the network, so none of it may
// run inside the state transaction. The snapshot read here is what the
// `updated_at` CAS in the commit validates.
let current_state = load_site_replication_state().await?;
if pending_endpoint_refresh(&current_state).is_some() {
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
}
let local_peer = current_local_peer(&req, &current_state);
let mut sites: Vec<PeerSite> = read_site_replication_json(req, &cred.secret_key, true).await?;
// The web console's "Set Up Site Replication" omits the local deployment from the payload;
// inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`.
ensure_local_site_present(&mut sites, &local_peer);
validate_add_sites(&sites, &local_peer)?;
let preflight_infos = add_preflight_infos(&sites, &current_state, &local_peer).await?;
validate_add_preflight_topology(&preflight_infos, &local_peer)?;
let expected_updated_at = current_state.updated_at;
require_add_peer_tls_capability(&sites, &local_peer).await?;
// Early exit on a state that moved under the preflight probes, BEFORE
// the IAM write and the join fan-out change anything remote. Advisory
// only — the binding check is the CAS inside the commit — but it fences
// the common race off the side-effect path and refreshes the merge
// base so the CAS window is only the join round trips.
let latest_state = load_site_replication_state().await?;
ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?;
let current_state = latest_state;
let (service_account_access_key, service_account_secret_key) =
ensure_site_replicator_service_account(&cred.access_key, false).await?;
let bootstrap_buckets = preflight_infos
.iter()
.filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint))
.flat_map(|info| info.buckets.keys().cloned())
.collect();
let add_in_progress_guard = SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets)?;
let mut state = merge_add_sites(
current_state,
local_peer.clone(),
sites.clone(),
service_account_access_key.clone(),
cred.access_key.clone(),
replicate_ilm_expiry,
);
state.sync_state_initialized = true;
let join_req = SRPeerJoinEnvelope {
request: SRPeerJoinReq {
svc_acct_access_key: service_account_access_key,
svc_acct_secret_key: service_account_secret_key.clone(),
svc_acct_parent: String::new(),
peers: state.peers.clone(),
updated_at: state.updated_at,
},
defer_sync_state_enable: true,
};
let peer_join_path =
with_site_replication_bootstrap_token(SITE_REPLICATION_PEER_JOIN_PATH, &add_in_progress_guard.token.to_string());
let admin_access_key = cred.access_key.clone();
let admission_store = current_object_store_handle()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
let list_store = admission_store.clone();
let (state, edit_generation, local_peer, service_account_secret_key, mut initial_sync_errors, _add_guard) =
with_site_replication_bucket_mutation_admission_lock(admission_store, move || async move {
// The writer starts before the local bucket snapshot and stays
// held through every peer join and the topology commit. A
// delete followed by a same-name create therefore cannot hide
// behind an unchanged final name set. Peer bootstrap callbacks
// use their internal path and do not acquire this public-
// mutation admission lock.
let current_state = load_site_replication_state().await?;
if pending_endpoint_refresh(&current_state).is_some() {
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
}
let local_peer = local_peer_at_endpoint(local_endpoint, &current_state);
// The web console's "Set Up Site Replication" omits the local deployment from the payload;
// inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`.
ensure_local_site_present(&mut sites, &local_peer);
validate_add_sites(&sites, &local_peer)?;
let preflight_infos = add_preflight_infos(&sites, &current_state, &local_peer).await?;
validate_add_preflight_topology(&preflight_infos, &local_peer)?;
let expected_updated_at = current_state.updated_at;
require_add_peer_tls_capability(&sites, &local_peer).await?;
// Early exit on a state that moved under the preflight probes, BEFORE
// the IAM write and the join fan-out change anything remote. Advisory
// only — the binding check is the CAS inside the commit — but it fences
// the common race off the side-effect path and refreshes the merge
// base so the CAS window is only the join round trips.
let latest_state = load_site_replication_state().await?;
ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?;
let current_state = latest_state;
let (service_account_access_key, service_account_secret_key) =
ensure_site_replicator_service_account(&admin_access_key, false).await?;
let expected_buckets: HashSet<String> =
preflight_infos.iter().flat_map(|info| info.buckets.keys().cloned()).collect();
let bootstrap_buckets: HashSet<String> = preflight_infos
.iter()
.filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint))
.flat_map(|info| info.buckets.keys().cloned())
.collect();
let add_in_progress_guard =
SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets.clone())?;
let mut state = merge_add_sites(
current_state,
local_peer.clone(),
sites.clone(),
service_account_access_key.clone(),
admin_access_key,
replicate_ilm_expiry,
);
state.sync_state_initialized = true;
let join_req = SRPeerJoinEnvelope {
request: SRPeerJoinReq {
svc_acct_access_key: service_account_access_key,
svc_acct_secret_key: service_account_secret_key.clone(),
svc_acct_parent: String::new(),
peers: state.peers.clone(),
updated_at: state.updated_at,
},
defer_sync_state_enable: true,
};
let peer_join_path = with_site_replication_bootstrap_token(
SITE_REPLICATION_PEER_JOIN_PATH,
&add_in_progress_guard.token.to_string(),
);
let mut joined_endpoints = HashSet::new();
let mut initial_sync_errors = SiteReplicationErrorSummary::default();
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
{
continue;
}
let mut joined_endpoints = HashSet::new();
let mut initial_sync_errors = SiteReplicationErrorSummary::default();
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
{
continue;
}
let mut peer_join_req = join_req.clone();
peer_join_req.request.svc_acct_parent = site.access_key.clone();
let connection = PeerConnection::try_from(site)?;
let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key)
.send(&site.secret_key, &peer_join_req)
let mut peer_join_req = join_req.clone();
peer_join_req.request.svc_acct_parent = site.access_key.clone();
let connection = PeerConnection::try_from(site)?;
let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key)
.send(&site.secret_key, &peer_join_req)
.await?;
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
fallback_peer.deployment_id = preflight.deployment_id.clone();
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("parse peer join response from {} failed: {e}", site.endpoint),
)
})?;
if !join_response.initial_sync_error_message.is_empty() {
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
}
// An explicit no-op join. The peer answered 200 but wrote nothing —
// its persisted state is already newer than the snapshot it was
// sent — so the add is only PARTIALLY configured and saying
// "configured successfully" would be a lie (rustfs/rustfs#5963).
// `None` (a MinIO peer, or one older than the field) is not a
// no-op signal and is deliberately not reported.
if join_response.applied == Some(false) {
initial_sync_errors.push(format!(
"{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \
the site is not configured against this peer",
site.endpoint
));
}
state = reconcile_peer_with_actual_identity(state, join_response.peer);
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("peer join response from {} did not identify the requested site", site.endpoint),
)
})?;
validate_proposed_peer(&reconciled_peer).map_err(|err| {
S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!("invalid peer join response from {}: {err}", site.endpoint),
)
})?;
}
mark_unknown_peer_sync_enabled(&mut state.peers);
// Commit. The state transaction's CAS still fences topology
// writers that do not use bucket admission. By this point
// remote sites may already have accepted their joins, so a
// mismatch asks the operator to re-run add and reconverge.
let next_state = state;
let present = list_store
.list_bucket(&BucketOptions::default())
.await
.map_err(ApiError::from)?
.into_iter()
.map(|bucket| bucket.name)
.collect::<HashSet<_>>();
ensure_add_bucket_set_matches_preflight(&expected_buckets, &present)?;
let (state, edit_generation) = update_site_replication_state(move |state| {
if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() {
return Err(s3_error!(
InvalidRequest,
"site replication state changed during peer join; the peers may already be joined — re-run replicate add"
));
}
adopt_add_commit_state(state, next_state);
let edit_generation = next_peer_edit_generation(state);
Ok((state.clone(), edit_generation))
})
.await?;
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
fallback_peer.deployment_id = preflight.deployment_id.clone();
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("parse peer join response from {} failed: {e}", site.endpoint),
)
})?;
if !join_response.initial_sync_error_message.is_empty() {
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
}
// An explicit no-op join. The peer answered 200 but wrote nothing —
// its persisted state is already newer than the snapshot it was
// sent — so the add is only PARTIALLY configured and saying
// "configured successfully" would be a lie (rustfs/rustfs#5963).
// `None` (a MinIO peer, or one older than the field) is not a
// no-op signal and is deliberately not reported.
if join_response.applied == Some(false) {
initial_sync_errors.push(format!(
"{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \
the site is not configured against this peer",
site.endpoint
));
}
state = reconcile_peer_with_actual_identity(state, join_response.peer);
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("peer join response from {} did not identify the requested site", site.endpoint),
)
})?;
validate_proposed_peer(&reconciled_peer).map_err(|err| {
S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!("invalid peer join response from {}: {err}", site.endpoint),
)
})?;
}
mark_unknown_peer_sync_enabled(&mut state.peers);
// Commit. The CAS runs inside the transaction, against the state the
// transaction itself loaded — the peer round trips above took however
// long they took, and only this check can tell whether the topology
// this add was planned against is still the current one. The error
// says so: by this point the remote sites already accepted their
// joins, and re-running the add is what reconverges the local side.
let next_state = state;
let (state, edit_generation) = update_site_replication_state(move |state| {
if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() {
return Err(s3_error!(
InvalidRequest,
"site replication state changed during peer join; the peers may already be joined — re-run replicate add"
));
}
adopt_add_commit_state(state, next_state);
let edit_generation = next_peer_edit_generation(state);
Ok((state.clone(), edit_generation))
})
.await?;
Ok((
state,
edit_generation,
local_peer,
service_account_secret_key,
initial_sync_errors,
add_in_progress_guard,
))
})
.await?;
// The finalize fan-out delivers peer-edit payloads, so it carries the
// generation allocated in the commit above: the receiving site orders
@@ -7185,8 +7309,14 @@ impl Operation for SRPeerEditHandler {
// The fence is self-reported — the shared service account means
// the sender cannot be identified — so it is honoured only after
// the admissibility check, against the same state it will gate.
let commit_fence =
commit_fence.filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence));
let commit_fence = match commit_fence {
Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence),
// A fenced edit can only come from a current remote peer. If
// that origin left while the retry was in flight, applying
// its body here would resurrect the removed topology.
Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked)),
None => None,
};
// Ordering fence: the sending site allocates the generation under
// its state-object lock, so a delivery that lost the race carries
// a generation this site has already passed. Applying it would
@@ -8886,6 +9016,41 @@ mod tests {
);
}
#[test]
fn add_admission_starts_before_preflight_and_rejects_bucket_set_changes() {
let expected = HashSet::from(["remote-owned".to_string(), "shared".to_string()]);
let present = HashSet::from(["shared".to_string()]);
let err = ensure_add_bucket_set_matches_preflight(&expected, &present)
.expect_err("a missing bootstrap bucket must reject the topology commit");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
let present = HashSet::from([
"remote-owned".to_string(),
"shared".to_string(),
"created-during-add".to_string(),
]);
let err = ensure_add_bucket_set_matches_preflight(&expected, &present)
.expect_err("a bucket created during add must reject the topology commit");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
let src = include_str!("site_replication.rs");
let add = src
.split("impl Operation for SiteReplicationAddHandler")
.nth(1)
.and_then(|rest| rest.split("pub struct SiteReplicationRemoveHandler").next())
.expect("add handler block");
let admission = add
.find("with_site_replication_bucket_mutation_admission_lock")
.expect("distributed mutation admission");
let preflight = add.find("add_preflight_infos").expect("bucket preflight");
let validation = add
.find("ensure_add_bucket_set_matches_preflight")
.expect("bucket-set validation");
let commit = add.find("adopt_add_commit_state").expect("topology commit");
assert!(admission < preflight && preflight < validation && validation < commit);
}
#[test]
fn test_tls_capability_gates_run_before_add_or_edit_state_side_effects() {
let src = include_str!("site_replication.rs");
@@ -9182,13 +9347,19 @@ mod tests {
);
// Fence hardening: origin and generation are self-reported by a
// caller the shared service account cannot identify, so the handler
// must pass the fence through the admissibility check — against the
// same state the fence gates, i.e. inside the transaction — before
// reading or raising any high-water mark.
// must admit the fence against the same state it gates. An origin
// removed while a retry was in flight is acknowledged without
// applying the stale body; otherwise it could recreate topology.
assert!(
handler_block.contains(".filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence))"),
handler_block.contains(
"Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence)"
),
"SRPeerEditHandler must admit a fence only through peer_edit_fence_is_admissible inside the state transaction"
);
assert!(
handler_block.contains("Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked))"),
"SRPeerEditHandler must not apply a fenced edit after its origin leaves the current topology"
);
// P1-15 PR2: both halves of the fence and the edit they fence share
// ONE transaction. Checking the fence against a state read outside the
// lock would let the check pass on one snapshot and the write land on
@@ -10352,8 +10523,9 @@ mod tests {
/// A fence is self-reported: every site authenticates peer traffic with
/// the same site-replicator credential, so a compromised peer can stamp
/// ANY origin with ANY generation. An origin the receiver does not
/// replicate with — or the receiver itself — is ignored and plants no
/// mark; a mark a compromised peer plants for a CURRENT origin cannot
/// replicate with — or the receiver itself — is inadmissible and plants
/// no mark; the handler acknowledges such a request without applying its
/// body. A mark a compromised peer plants for a CURRENT origin cannot
/// silence that origin, because the staleness window refuses to fence on
/// a mark implausibly far above the genuine deliveries.
#[test]
@@ -12791,6 +12963,7 @@ mod tests {
last_error: "site replication is not enabled".to_string(),
updated_at: Some(OffsetDateTime::now_utc()),
edit_generation: None,
peer_unreachable: false,
deletions_recorded: false,
}],
..Default::default()
@@ -12989,6 +13162,7 @@ mod tests {
last_error: "peer offline".to_string(),
updated_at: Some(OffsetDateTime::now_utc()),
edit_generation: None,
peer_unreachable: false,
deletions_recorded: false,
}],
..Default::default()
rustfs/src/app/bucket_usecase.rs
+63 -37
View File
@@ -75,7 +75,8 @@ use crate::auth::get_condition_values_with_client_info;
use crate::error::ApiError;
use crate::shared_types::RemoteAddr;
use crate::site_replication::{
site_replication_bucket_meta_hook, site_replication_delete_bucket_hook, site_replication_make_bucket_hook,
cancel_site_replication_delete_bucket, commit_site_replication_delete_bucket, prepare_site_replication_delete_bucket,
site_replication_bucket_meta_hook, site_replication_make_bucket_hook, with_site_replication_bucket_mutation_lock,
};
use crate::storage::storage_api::lock_bucket_targets_metadata;
use http::StatusCode;
@@ -1331,23 +1332,34 @@ impl DefaultBucketUsecase {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let make_result = store
.make_bucket(
&bucket,
&MakeBucketOptions {
force_create: false,
lock_enabled,
..Default::default()
},
)
.await;
// Keep the local namespace mutation and its peer hook ordered across
// every node in this site. Otherwise a delete waiting for repair
// coordination can arrive after this create on remote sites.
let operation_bucket = bucket.clone();
let operation_store = store.clone();
let make_result = with_site_replication_bucket_mutation_lock(store, &bucket, move || async move {
let make_result = operation_store
.make_bucket(
&operation_bucket,
&MakeBucketOptions {
force_create: false,
lock_enabled,
..Default::default()
},
)
.await;
if make_result.is_ok() {
crate::storage::invalidate_bucket_validation_cache(&operation_bucket);
if let Err(err) = site_replication_make_bucket_hook(&operation_bucket, lock_enabled).await {
warn!(bucket = %operation_bucket, error = ?err, "site replication make bucket hook failed");
}
}
make_result
})
.await?;
match make_result {
Ok(()) => {
// Invalidate the bucket validation cache so subsequent GETs
// see the newly created bucket immediately.
crate::storage::invalidate_bucket_validation_cache(&bucket);
}
Ok(()) => {}
Err(StorageError::BucketExists(_)) => {
// Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK;
// non-owner gets 409 BucketAlreadyExists.
@@ -1358,10 +1370,6 @@ impl DefaultBucketUsecase {
Err(e) => return Err(ApiError::from(e).into()),
}
if let Err(err) = site_replication_make_bucket_hook(&bucket, lock_enabled).await {
warn!(bucket = %bucket, error = ?err, "site replication make bucket hook failed");
}
let output = CreateBucketOutput::default();
counter!("rustfs_create_bucket_total").increment(1);
let result = Ok(S3Response::new(output));
@@ -1397,16 +1405,41 @@ impl DefaultBucketUsecase {
authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?;
}
store
.delete_bucket(
&input.bucket,
&DeleteBucketOptions {
force,
..Default::default()
},
)
.await
.map_err(ApiError::from)?;
// Keep the local namespace mutation and its peer hook ordered across
// every node in this site so an older delete cannot overtake a new
// same-name make while it waits for repair coordination.
let operation_bucket = input.bucket.clone();
let operation_store = store.clone();
with_site_replication_bucket_mutation_lock(store, &input.bucket, move || async move {
let intent = prepare_site_replication_delete_bucket(&operation_bucket, force).await?;
let delete_result = operation_store
.delete_bucket(
&operation_bucket,
&DeleteBucketOptions {
force,
..Default::default()
},
)
.await;
match delete_result {
Ok(()) => {
crate::storage::invalidate_bucket_validation_cache(&operation_bucket);
if let Some(intent) = intent
&& let Err(err) = commit_site_replication_delete_bucket(&intent).await
{
warn!(bucket = %operation_bucket, error = ?err, "site replication delete bucket hook failed");
}
Ok::<(), S3Error>(())
}
Err(err) => {
if let Some(intent) = intent {
cancel_site_replication_delete_bucket(intent).await;
}
Err(S3Error::from(ApiError::from(err)))
}
}
})
.await??;
// Drop every cached object body for the now-deleted bucket so dead
// bytes do not sit resident until TTL. Covers both the normal and the
@@ -1415,16 +1448,9 @@ impl DefaultBucketUsecase {
let cache_adapter = current_object_data_cache_for_context(self.context.as_deref());
let _ = invalidate_object_data_cache_bucket_after_delete(&cache_adapter, &input.bucket).await;
// Invalidate bucket validation cache
crate::storage::invalidate_bucket_validation_cache(&input.bucket);
// Re-evaluate lifecycle and replication after bucket removal.
rustfs_scanner::record_scanner_maintenance_change(&input.bucket);
if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await {
warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed");
}
// Notify peers to drop their cached metadata for the now-deleted bucket.
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
notify_bucket_metadata_delete(input.bucket.clone(), request_context);
rustfs/src/server/http.rs
+47
View File
@@ -208,6 +208,7 @@ const EVENT_PEER_ADDR_UNAVAILABLE: &str = "peer_addr_unavailable";
const EVENT_RPC_SIGNATURE_VERIFICATION_FAILED: &str = "rpc_signature_verification_failed";
const EVENT_GRPC_TRACE_CONTEXT_PROPAGATION_FAILED: &str = "grpc_trace_context_propagation_failed";
const HEAL_CONTROL_TONIC_RPC_PATH: &str = "/node_service.HealControlService/HealControl";
const SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH: &str = "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck";
const TIER_MUTATION_PREPARE_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/PrepareTierMutation";
const TIER_MUTATION_COMMIT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/CommitTierMutation";
const TIER_MUTATION_ABORT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/AbortTierMutation";
@@ -1856,6 +1857,7 @@ fn process_connection(
);
let rpc_service = RpcRequestPathService::new(
Routes::new(node_service)
.add_service(InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth))
.add_service(heal_control_service)
.add_service(tier_mutation_control_service)
.prepare(),
@@ -2259,6 +2261,7 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
.strip_prefix(TONIC_RPC_PREFIX)
.and_then(|suffix| suffix.strip_prefix('/'))
.or_else(|| (target.uri.path() == HEAL_CONTROL_TONIC_RPC_PATH).then_some("HealControl"))
.or_else(|| (target.uri.path() == SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH).then_some("ScannerScopedDirtyUsageAck"))
.or_else(|| (target.uri.path() == TIER_MUTATION_PREPARE_TONIC_RPC_PATH).then_some("PrepareTierMutation"))
.or_else(|| (target.uri.path() == TIER_MUTATION_COMMIT_TONIC_RPC_PATH).then_some("CommitTierMutation"))
.or_else(|| (target.uri.path() == TIER_MUTATION_ABORT_TONIC_RPC_PATH).then_some("AbortTierMutation"))
@@ -3427,6 +3430,50 @@ mod tests {
rustfs_common::set_global_local_node_name(&previous_node_name).await;
}
#[tokio::test]
#[serial_test::serial]
async fn scoped_dirty_usage_peer_probe_reaches_handler_through_production_auth() {
let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string());
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind scoped ACK auth test");
let addr = listener.local_addr().expect("listener address");
let previous_node_name = rustfs_common::get_global_local_node_name().await;
rustfs_common::set_global_local_node_name(&addr.to_string()).await;
let node = InterceptedService::new(NodeServiceServer::new(make_server()), check_auth);
let scanner = InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth);
let service = RpcRequestPathService::new(Routes::new(node).add_service(scanner).prepare());
let server = tokio::spawn(async move {
let (socket, _) = listener.accept().await.expect("accept test connection");
ConnBuilder::new(TokioExecutor::new())
.serve_connection(TokioIo::new(socket), TowerToHyperService::new(service))
.await
.expect("serve scoped ACK auth test");
});
let host = rustfs_utils::XHost::try_from(addr.to_string()).expect("peer address");
let client = storage::PeerRestClient::new(host, format!("http://{addr}"));
let result = client
.scanner_scoped_dirty_usage_capability(
"11111111-1111-1111-1111-111111111111".to_string(),
"a".repeat(32),
vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
bucket: "photos".into(),
bucket_incarnation: vec![1; 16].into(),
generation: 8,
}],
)
.await;
client.evict_connection().await;
server.abort();
let _ = server.await;
rustfs_common::set_global_local_node_name(&previous_node_name).await;
let error = result
.expect_err("probe must fail closed without the requested storage owner")
.to_string();
assert!(
error.contains("storage layer is not initialized") || error.contains("scoped dirty usage peer or process changed"),
"signed probe must pass production path authentication and reach owner validation: {error}"
);
}
#[tokio::test]
#[serial_test::serial]
async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() {
rustfs/src/site_replication/hooks.rs
+427 -35
View File
@@ -22,6 +22,57 @@ pub(crate) const SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION: &str = "confi
pub(crate) static SITE_REPLICATION_BUCKET_OP_LOCK: LazyLock<RwLock<()>> = LazyLock::new(|| RwLock::new(()));
const SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX: &str = "config/site-replication/bucket-mutation";
pub(crate) const SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH: &str =
"config/site-replication/bucket-mutation-admission.lock";
pub(crate) fn site_replication_bucket_mutation_lock_path(bucket: &str) -> String {
format!("{SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX}/{bucket}.lock")
}
pub(crate) async fn with_site_replication_bucket_mutation_lock<F, Fut, T>(
store: Arc<ECStore>,
bucket: &str,
operation: F,
) -> S3Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = T> + Send + 'static,
T: Send + 'static,
{
let mutation_store = store.clone();
let mutation_path = site_replication_bucket_mutation_lock_path(bucket);
with_config_object_read_lock(
store,
SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(),
move || async move {
with_config_object_write_lock(mutation_store, mutation_path, operation)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))
},
)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))?
}
/// Exclude every local bucket namespace mutation from an add's local preflight
/// snapshot until its topology commit. Peer bootstrap callbacks do not enter
/// this public-mutation admission path, so they can finish while the writer is
/// held; post-commit fan-out and backfill must run after it is released.
pub(crate) async fn with_site_replication_bucket_mutation_admission_lock<F, Fut, T>(
store: Arc<ECStore>,
operation: F,
) -> S3Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
T: Send + 'static,
{
with_config_object_write_lock(store, SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), operation)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))?
}
#[derive(Debug, Default)]
pub(crate) struct SiteReplicationBootstrapPlan {
pub(crate) iam_items: Vec<SRIAMItem>,
@@ -329,6 +380,91 @@ pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteRep
Ok(plan)
}
/// Build only the two bucket operations needed by the lightweight retry
/// drain. The full bootstrap plan scans every bucket and IAM record; doing
/// that on a 30-second recovery cadence would make lifecycle admission scale
/// with the whole site instead of the one queued bucket.
pub(crate) fn site_replication_bucket_retry_plan_for(
bucket: &SRBucketInfo,
replicate_ilm_expiry: bool,
) -> S3Result<SiteReplicationBootstrapPlan> {
let mut plan = SiteReplicationBootstrapPlan {
bucket_make_ops: vec![bootstrap_bucket_make_op_path(bucket)],
bucket_configure_ops: vec![bootstrap_bucket_op_path(
&bucket.bucket,
SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION,
)],
..Default::default()
};
append_bootstrap_bucket_items(&mut plan, bucket, replicate_ilm_expiry)?;
Ok(plan)
}
pub(crate) fn site_replication_bucket_retry_plan_from_info(
bucket: &SRBucketInfo,
replicate_ilm_expiry: bool,
) -> S3Result<SiteReplicationBootstrapPlan> {
let mut plan = site_replication_bucket_retry_plan_for(bucket, replicate_ilm_expiry)?;
// Omit only metadata the make/configure operations can reproduce exactly.
// Non-default versioning fields and operator-authored replication rules
// remain in the plan; their extra request cost intentionally defers the
// event to the complete drain when the lightweight budget is too small.
plan.bucket_items.retain(|item| !retry_bucket_metadata_is_redundant(item));
Ok(plan)
}
fn retry_bucket_metadata_is_redundant(item: &SRBucketMeta) -> bool {
match item.r#type.as_str() {
"version-config" => item.versioning.as_deref().is_some_and(|raw| {
deserialize::<VersioningConfiguration>(&decode_bucket_meta_wire_value(raw)).is_ok_and(|config| {
config
== VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
}
})
}),
"replication-config" => item.replication_config.as_deref().is_some_and(|raw| {
deserialize::<ReplicationConfiguration>(&decode_bucket_meta_wire_value(raw))
.is_ok_and(|config| config.role.trim().is_empty() && config.rules.iter().all(is_derived_site_replication_rule))
}),
// `Some("")` is the in-memory sentinel used when the bucket is lock
// enabled but has no object-lock configuration body. The make query
// carries lockEnabled=true; sending an empty metadata body is neither
// useful nor parseable.
"object-lock-config" => item.object_lock_config.as_deref() == Some(""),
_ => false,
}
}
pub(crate) async fn site_replication_bucket_retry_plan(
bucket: &str,
replicate_ilm_expiry: bool,
) -> S3Result<SiteReplicationBootstrapPlan> {
let Some(store) = current_object_store_handle() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let bucket_info = match store.get_bucket_info(bucket, &BucketOptions::default()).await {
Ok(bucket_info) => bucket_info,
Err(err) if is_err_bucket_not_found(&err) => return Ok(SiteReplicationBootstrapPlan::default()),
Err(err) => return Err(ApiError::from(err).into()),
};
let lock_enabled = bucket_info.object_locking;
let metadata = metadata_sys::get(bucket).await.map_err(ApiError::from)?;
let mut bucket_info = SRBucketInfo {
bucket: bucket.to_string(),
created_at: bucket_info.created,
location: current_region().map(|region| region.to_string()).unwrap_or_default(),
api_version: Some(SITE_REPL_API_VERSION.to_string()),
..Default::default()
};
populate_sr_bucket_info_from_metadata(&mut bucket_info, &metadata).await;
if lock_enabled && bucket_info.object_lock_config.is_none() {
bucket_info.object_lock_config = Some(String::new());
}
site_replication_bucket_retry_plan_from_info(&bucket_info, replicate_ilm_expiry)
}
pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> {
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await;
let runtime = {
@@ -393,20 +529,273 @@ pub(crate) async fn broadcast_site_replication_make_bucket(
broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await
}
pub async fn site_replication_delete_bucket_hook(bucket: &str, force_delete: bool) -> S3Result<()> {
const SITE_REPLICATION_DELETE_INTENT_PENDING: &str =
"bucket deletion reserved; local completion and peer delivery are not yet known";
#[derive(Clone)]
struct SiteReplicationDeleteBucketReservation {
peer: PeerInfo,
previous: Option<SiteReplicationRetryEvent>,
observed: SiteReplicationRetryEvent,
}
pub(crate) struct SiteReplicationDeleteBucketIntent {
path: String,
reservations: Vec<SiteReplicationDeleteBucketReservation>,
displaced: Vec<SiteReplicationRetryEvent>,
}
fn site_replication_delete_bucket_path(bucket: &str, force_delete: bool) -> String {
let operation = if force_delete {
"force-delete-bucket"
} else {
"delete-bucket"
};
let path = format!(
format!(
"/rustfs/admin/v3/site-replication/peer/bucket-ops?{}",
form_urlencoded::Serializer::new(String::new())
.append_pair("bucket", bucket)
.append_pair("operation", operation)
.finish()
);
broadcast_site_replication_json(&path, &serde_json::json!({})).await
)
}
/// Reserve every destructive peer delivery before the local namespace is
/// changed. The state transaction either persists the complete set or writes
/// nothing, so a full/unreadable queue fails the S3 delete closed.
pub(crate) async fn prepare_site_replication_delete_bucket(
bucket: &str,
force_delete: bool,
) -> S3Result<Option<SiteReplicationDeleteBucketIntent>> {
let path = site_replication_delete_bucket_path(bucket, force_delete);
let reservation_path = path.clone();
update_site_replication_state_when_changed(move |state| {
if !state.enabled() {
return Ok(StateCommit::Unchanged(None));
}
let local_peer = current_local_runtime_peer(state);
let peers = state
.peers
.values()
.filter(|peer| {
peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
})
.cloned()
.collect::<Vec<_>>();
if peers.is_empty() {
return Ok(StateCommit::Unchanged(None));
}
let mut reservations = Vec::with_capacity(peers.len());
let mut displaced = Vec::new();
for peer in peers {
let previous = state
.retry_queue
.iter()
.find(|event| retry_event_matches(event, &peer, &reservation_path))
.cloned();
displaced.extend(upsert_site_replication_retry_event(
&mut state.retry_queue,
&peer,
&reservation_path,
SITE_REPLICATION_DELETE_INTENT_PENDING,
None,
)?);
let observed = state
.retry_queue
.iter()
.find(|event| retry_event_matches(event, &peer, &reservation_path))
.cloned()
.ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InternalError,
"site replication delete reservation disappeared before commit".to_string(),
)
})?;
reservations.push(SiteReplicationDeleteBucketReservation {
peer,
previous,
observed,
});
}
Ok(StateCommit::Changed(Some(SiteReplicationDeleteBucketIntent {
path: reservation_path,
reservations,
displaced,
})))
})
.await
}
/// Roll back a reservation when the local storage delete definitively failed.
/// A concurrently revised reservation is preserved; it belongs to a newer
/// observation and this operation has no authority to settle it.
pub(crate) async fn cancel_site_replication_delete_bucket(intent: SiteReplicationDeleteBucketIntent) {
let path = intent.path.clone();
let result = update_site_replication_state_when_changed(move |state| {
let mut changed = false;
for reservation in intent.reservations {
let Some(index) = state.retry_queue.iter().position(|event| {
retry_event_matches(event, &reservation.peer, &reservation.observed.path)
&& event.id == reservation.observed.id
&& event.updated_at == reservation.observed.updated_at
}) else {
continue;
};
if let Some(previous) = reservation.previous {
state.retry_queue[index] = previous;
} else {
state.retry_queue.remove(index);
}
changed = true;
}
let mut restored_all = true;
for displaced in intent.displaced {
let duplicate = state.retry_queue.iter().any(|event| {
event.id == displaced.id
|| (event.peer_deployment_id == displaced.peer_deployment_id && event.path == displaced.path)
});
if duplicate {
continue;
}
if state.retry_queue.len() >= SITE_REPLICATION_RETRY_QUEUE_LIMIT {
restored_all = false;
continue;
}
state.retry_queue.push(displaced);
changed = true;
}
Ok(if changed {
StateCommit::Changed(restored_all)
} else {
StateCommit::Unchanged(restored_all)
})
})
.await;
match result {
Ok(true) => {}
Ok(false) => warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
path,
result = "delete_intent_cancel_incomplete",
"admin site replication state"
),
Err(err) => warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
path,
result = "delete_intent_cancel_failed",
error = ?err,
"admin site replication state"
),
}
}
async fn broadcast_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> {
let sends = intent.reservations.iter().cloned().map(|reservation| {
let request_path = intent.path.clone();
async move {
let fallback_peer = reservation.peer.clone();
let observed = reservation.observed.clone();
let delivery_path = request_path.clone();
let delivery = with_site_replication_state_read_lock(move |state| async move {
let Some(current_peer) = state.peers.get(&fallback_peer.deployment_id).cloned() else {
return Ok(None);
};
let service_account_secret_key =
match site_replicator_service_account_secret(&state.service_account_access_key).await {
Ok(secret) => secret,
Err(err) => {
let Some(secret) = legacy_site_replicator_state_secret(&state) else {
return Err(err);
};
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "legacy_state_service_account_secret_fallback",
error = ?err,
"admin site replication state"
);
secret
}
};
let result = async {
let transport = PeerTransport::for_runtime_peer(&current_peer).await?;
PeerAdminRequest::put(&transport.connection, &delivery_path, &state.service_account_access_key)
.with_client(&transport.client)
.send(&service_account_secret_key, &serde_json::json!({}))
.await
}
.await;
Ok(Some((current_peer, result)))
})
.await;
match delivery {
Ok(Some((current_peer, Ok(_)))) => {
dequeue_observed_site_replication_retry_event(&current_peer, &observed).await;
None
}
Ok(Some((current_peer, Err(err)))) => {
// Keep the failed deletion operator-visible, but never
// replay it automatically: without a bucket-incarnation
// fence, a delayed delete could erase a recreated bucket.
enqueue_site_replication_retry_event(&current_peer, &request_path, &err).await;
Some(err)
}
Ok(None) => {
dequeue_observed_site_replication_retry_event(&reservation.peer, &observed).await;
None
}
Err(err) => {
enqueue_site_replication_retry_event(&reservation.peer, &request_path, &err).await;
Some(err)
}
}
}
});
futures::future::join_all(sends)
.await
.into_iter()
.flatten()
.next()
.map_or(Ok(()), Err)
}
pub(crate) async fn commit_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> {
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await;
let store =
current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
let retry_peers = intent
.reservations
.iter()
.map(|reservation| reservation.peer.clone())
.collect::<Vec<_>>();
let retry_path = intent.path.clone();
let delivery_intent = SiteReplicationDeleteBucketIntent {
path: intent.path.clone(),
reservations: intent.reservations.clone(),
displaced: Vec::new(),
};
match with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move {
broadcast_site_replication_delete_bucket(&delivery_intent).await
})
.await
{
Ok(result) => result,
Err(err) => {
let err: S3Error = ApiError::from(err).into();
for peer in &retry_peers {
enqueue_site_replication_retry_event(peer, &retry_path, &err).await;
}
Err(err)
}
}
}
pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Result<()> {
@@ -515,6 +904,39 @@ pub(crate) fn maybe_time(value: OffsetDateTime) -> Option<OffsetDateTime> {
(value != OffsetDateTime::UNIX_EPOCH).then_some(value)
}
async fn populate_sr_bucket_info_from_metadata(entry: &mut SRBucketInfo, metadata: &BucketMetadata) {
entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok());
entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml);
entry.tags = raw_config_to_base64(&metadata.tagging_config_xml);
entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml);
entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml);
entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml);
entry.quota_config = raw_config_to_base64(&metadata.quota_config_json);
// Expiry subset only: this entry feeds both the bootstrap/repair plan
// (peers must not receive transition rules) and cross-site consistency
// views (transition rules are site-local and would read as false
// mismatches). A deleted expiry state is a `None` value with the
// deletion's axis so repair can converge peers that missed the live
// delete.
let expiry_statement = lifecycle_expiry_statement(metadata);
entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone());
entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml);
entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at);
entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at);
entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at);
entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at);
entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at);
entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at);
entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at);
// The expiry axis, not the whole-config write time: local transition-only
// edits inflate the latter, and a repair item stamped with it could
// out-rank a newer real expiry edit on a third site.
entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis);
entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at);
entry.replication_targets_online =
Some(site_replication_targets_online(&entry.bucket, &metadata.replication_config_xml).await);
}
pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S3Result<SRInfo> {
let Some(store) = current_object_store_handle() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
@@ -546,37 +968,7 @@ pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &Pee
};
if let Some(metadata) = metadata {
entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok());
entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml);
entry.tags = raw_config_to_base64(&metadata.tagging_config_xml);
entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml);
entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml);
entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml);
entry.quota_config = raw_config_to_base64(&metadata.quota_config_json);
// Expiry subset only: this entry feeds both the bootstrap/repair
// plan (peers must not receive transition rules) and cross-site
// consistency views (transition rules are site-local and would
// read as false mismatches). A deleted expiry state is a `None`
// value with the deletion's axis so repair can converge peers
// that missed the live delete.
let expiry_statement = lifecycle_expiry_statement(&metadata);
entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone());
entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml);
entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at);
entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at);
entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at);
entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at);
entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at);
entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at);
entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at);
// The expiry axis, not the whole-config write time: local
// transition-only edits inflate the latter, and a repair item
// stamped with it could out-rank a newer real expiry edit on a
// third site.
entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis);
entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at);
entry.replication_targets_online =
Some(site_replication_targets_online(&bucket.name, &metadata.replication_config_xml).await);
populate_sr_bucket_info_from_metadata(&mut entry, &metadata).await;
}
info.buckets.insert(bucket.name, entry);
rustfs/src/site_replication/mod.rs
+7 -6
View File
@@ -47,6 +47,7 @@ use self::identity::{
canonical_endpoint, deployment_id_for_endpoint, mark_unknown_peer_sync_enabled, normalize_peer_map_by_identity_with,
same_identity_endpoint,
};
pub(crate) use self::state_lock::with_site_replication_state_read_lock;
use self::state_lock::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock};
use crate::auth::constant_time_eq;
use crate::config::get_config_snapshot;
@@ -64,12 +65,12 @@ use crate::storage_api::site_replication::s3::{
#[cfg(test)]
use crate::storage_api::site_replication::save_config as save_admin_config;
use crate::storage_api::site_replication::{
ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketTarget,
BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract, StorageError,
VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize, is_site_replication_role,
lock_bucket_targets_metadata, metadata_sys, read_config as read_admin_config, read_config_no_lock,
replication_target_arn_deployment_id, save_config_no_lock, serialize, site_replication_rule_deployment_id,
with_config_object_read_lock, with_config_object_write_lock,
ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketMetadata, BucketOperations,
BucketOptions, BucketTarget, BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract,
StorageError, VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize,
is_err_bucket_not_found, is_site_replication_role, lock_bucket_targets_metadata, metadata_sys,
read_config as read_admin_config, read_config_no_lock, replication_target_arn_deployment_id, save_config_no_lock, serialize,
site_replication_rule_deployment_id, with_config_object_read_lock, with_config_object_write_lock,
};
use base64_simd::STANDARD as BASE64_STANDARD;
use base64_simd::URL_SAFE_NO_PAD;
rustfs/src/site_replication/repair.rs
+3 -1
View File
@@ -649,7 +649,9 @@ pub(crate) async fn persist_site_replication_repair_task(
let path = path.to_string();
update_site_replication_state(move |state| {
match failure.as_deref() {
Some(error) => upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None),
Some(error) => {
upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None)?;
}
None => {
dequeue_site_replication_retry_events_including_escalated(&mut state.retry_queue, &peer, &path);
// A repair is the operator's accountability transfer for the
rustfs/src/site_replication/retry.rs
+701 -148
View File
File diff suppressed because it is too large Load Diff
rustfs/src/site_replication/state_lock.rs
+31 -6
View File
@@ -28,14 +28,18 @@
//! process-local lock must never be reintroduced in front of it as if it
//! added protection. All IO inside the closure must use the `*_no_lock`
//! config helpers — the locked variants would self-deadlock on the same
//! object lock. Do not perform peer network calls or take other config locks
//! inside the closure.
//! object lock. Write-lock closures must not perform peer network calls or
//! take other config locks. A read-lock closure may carry bounded peer
//! delivery only when the receiver cannot write this state. Peer-edit
//! delivery must run after the read lock is released.
//!
//! Lock order: lifecycle -> bucket operation -> repair admission
//! -> state object lock -> per-bucket metadata.
//! Lock order: lifecycle -> bucket-mutation admission -> per-bucket mutation
//! -> bucket operation -> repair admission -> state object lock ->
//! per-bucket metadata. A path may skip levels, but must not acquire an
//! earlier level while holding a later one.
use super::{S3Error, S3ErrorCode, S3Result};
use crate::storage_api::site_replication::{ECStore, with_config_object_write_lock};
use super::{S3Error, S3ErrorCode, S3Result, SiteReplicationState, load_site_replication_state_no_lock};
use crate::storage_api::site_replication::{ECStore, with_config_object_read_lock, with_config_object_write_lock};
use std::sync::Arc;
use crate::runtime_sources::current_object_store_handle;
@@ -57,6 +61,27 @@ where
with_site_replication_state_lock_on(store, operation).await
}
/// Hold the distributed state-object read lock while `operation` validates a
/// topology snapshot. The closure may carry a bounded peer delivery only when
/// its receiver cannot write site replication state; peer-edit delivery must
/// run after this lock is released. Topology writers use the matching write
/// lock through [`with_site_replication_state_lock`].
pub(crate) async fn with_site_replication_state_read_lock<T, F, Fut>(operation: F) -> S3Result<T>
where
T: Send + 'static,
F: FnOnce(SiteReplicationState) -> Fut + Send + 'static,
Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
{
let store = current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
let read_store = store.clone();
with_config_object_read_lock(store, SITE_REPLICATION_STATE_PATH.to_string(), move || async move {
let state = load_site_replication_state_no_lock(read_store).await?;
operation(state).await
})
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock site replication state failed: {e}")))?
}
/// Context-store variant for callers that resolve their store from an
/// explicit [`AppContext`] (the service-side reload driven over node RPC).
///
rustfs/src/site_replication/tests.rs
+568 -41
View File
@@ -33,6 +33,18 @@ use temp_env::with_var;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
#[test]
fn test_bucket_mutation_lock_path_is_bucket_scoped() {
assert_eq!(
site_replication_bucket_mutation_lock_path("photos"),
"config/site-replication/bucket-mutation/photos.lock"
);
assert_ne!(
site_replication_bucket_mutation_lock_path("photos"),
site_replication_bucket_mutation_lock_path("videos")
);
}
fn valid_test_ca_pem(name: &str) -> String {
rcgen::generate_simple_self_signed(vec![name.to_string()])
.expect("generate test CA")
@@ -381,6 +393,30 @@ async fn peer_clients_do_not_follow_redirects() {
assert!(tls_server.await.expect("custom redirect TLS server task"));
}
#[tokio::test]
async fn peer_http_error_body_cannot_spoof_an_unreachable_peer() {
let (endpoint, ca_pem, server) = spawn_test_tls_server_with_response(
b"HTTP/1.1 500 Internal Server Error\r\ncontent-length: 27\r\nconnection: close\r\n\r\ndownstream failed (connect)",
)
.await;
let connection = validate_peer_connection_inner(&endpoint, false, &ca_pem, true).expect("custom CA peer connection");
let client =
build_custom_site_replication_peer_client(&empty_outbound_tls_state(), &connection).expect("custom CA peer client");
let err = PeerAdminRequest::post(&connection, SITE_REPLICATION_PEER_DEVNULL_PATH, "access-key")
.with_client(&client)
.send("secret-key", &serde_json::json!({}))
.await
.expect_err("HTTP 500 must fail");
let detail = err.to_string();
assert!(detail.contains("downstream failed (connect)"));
assert!(
!retry_error_indicates_peer_unreachable(&detail),
"an untrusted response body must not enable the fast reachability probe"
);
assert!(server.await.expect("HTTP error TLS server task"));
}
fn peer(name: &str, endpoint: &str) -> PeerInfo {
PeerInfo {
name: name.to_string(),
@@ -419,6 +455,7 @@ fn drain_event(peer: &str, path: &str, retry_count: u32, updated_at: Option<Offs
last_error: "remote-operation-failed".to_string(),
updated_at,
edit_generation: None,
peer_unreachable: false,
deletions_recorded: false,
}
}
@@ -532,25 +569,25 @@ fn test_record_failed_iam_delivery_records_deletions_and_flags_entry() {
// Non-deletion failure: entry flagged, no record.
let mut user_update = user_delete_item("alice");
user_update.iam_user.as_mut().expect("iam user").is_delete_req = false;
record_failed_iam_delivery(&mut state, &target, &user_update, "peer offline");
record_failed_iam_delivery(&mut state, &target, &user_update, "peer offline").expect("record failure");
assert_eq!(state.retry_queue.len(), 1);
assert_eq!(state.retry_queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH);
assert!(state.retry_queue[0].deletions_recorded);
assert!(state.iam_deletion_replays.is_empty());
// Deletion failure: recorded for replay, entry stays flagged.
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline");
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
assert_eq!(state.iam_deletion_replays.len(), 1);
assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:alice");
assert!(state.retry_queue[0].deletions_recorded);
assert_eq!(state.retry_queue.len(), 1, "IAM failures stay collapsed per peer");
// Same entity again: newest body replaces the record.
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline");
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
assert_eq!(state.iam_deletion_replays.len(), 1);
// Different entity: second record.
record_failed_iam_delivery(&mut state, &target, &policy_delete_item("readonly"), "peer offline");
record_failed_iam_delivery(&mut state, &target, &policy_delete_item("readonly"), "peer offline").expect("record failure");
assert_eq!(state.iam_deletion_replays.len(), 2);
// A legacy entry (created without recording) is never stamped.
@@ -565,8 +602,9 @@ fn test_record_failed_iam_delivery_records_deletions_and_flags_entry() {
SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH,
"peer offline",
None,
);
record_failed_iam_delivery(&mut state, &legacy, &user_delete_item("bob"), "peer offline");
)
.expect("upsert retry event");
record_failed_iam_delivery(&mut state, &legacy, &user_delete_item("bob"), "peer offline").expect("record failure");
let legacy_event = state
.retry_queue
.iter()
@@ -589,12 +627,13 @@ fn test_record_failed_iam_delivery_overflow_degrades_to_escalation() {
};
let mut state = deletion_replay_state(&target);
for index in 0..SITE_REPLICATION_IAM_DELETION_REPLAY_LIMIT_PER_PEER {
record_failed_iam_delivery(&mut state, &target, &policy_delete_item(&format!("p{index}")), "peer offline");
record_failed_iam_delivery(&mut state, &target, &policy_delete_item(&format!("p{index}")), "peer offline")
.expect("record failure");
}
assert!(state.retry_queue[0].deletions_recorded);
assert_eq!(state.iam_deletion_replays.len(), SITE_REPLICATION_IAM_DELETION_REPLAY_LIMIT_PER_PEER);
record_failed_iam_delivery(&mut state, &target, &policy_delete_item("one-too-many"), "peer offline");
record_failed_iam_delivery(&mut state, &target, &policy_delete_item("one-too-many"), "peer offline").expect("record failure");
assert_eq!(
state.iam_deletion_replays.len(),
SITE_REPLICATION_IAM_DELETION_REPLAY_LIMIT_PER_PEER,
@@ -620,33 +659,23 @@ fn test_settle_replayed_iam_retry_events_settles_or_escalates() {
// Fully recorded: settles.
let mut state = deletion_replay_state(&target);
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline");
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
state.retry_queue[0].updated_at = Some(snapshot_at);
let observed = state.retry_queue[0].clone();
let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect();
assert!(settle_replayed_iam_retry_events(
&mut state,
&target,
SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH,
Some(snapshot_at),
&replayed,
));
assert!(settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed));
assert!(state.retry_queue.is_empty());
assert!(state.iam_deletion_replays.is_empty());
// Not fully recorded: replayed records are still removed, but the entry
// escalates instead of settling.
let mut state = deletion_replay_state(&target);
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline");
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
state.retry_queue[0].updated_at = Some(snapshot_at);
state.retry_queue[0].deletions_recorded = false;
let observed = state.retry_queue[0].clone();
let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect();
assert!(!settle_replayed_iam_retry_events(
&mut state,
&target,
SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH,
Some(snapshot_at),
&replayed,
));
assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed));
assert!(state.iam_deletion_replays.is_empty());
assert_eq!(state.retry_queue.len(), 1);
assert_eq!(state.retry_queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER);
@@ -654,17 +683,13 @@ fn test_settle_replayed_iam_retry_events_settles_or_escalates() {
// Newer failure since the snapshot: entry untouched and drain-eligible,
// residual (unreplayed) record kept for the next pass.
let mut state = deletion_replay_state(&target);
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline");
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
state.retry_queue[0].updated_at = Some(snapshot_at);
let observed = state.retry_queue[0].clone();
let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect();
record_failed_iam_delivery(&mut state, &target, &user_delete_item("bob"), "peer offline");
record_failed_iam_delivery(&mut state, &target, &user_delete_item("bob"), "peer offline").expect("record failure");
state.retry_queue[0].updated_at = Some(snapshot_at + time::Duration::seconds(5));
assert!(!settle_replayed_iam_retry_events(
&mut state,
&target,
SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH,
Some(snapshot_at),
&replayed,
));
assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed));
assert_eq!(state.retry_queue.len(), 1);
assert_ne!(state.retry_queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER);
assert!(
@@ -673,6 +698,24 @@ fn test_settle_replayed_iam_retry_events_settles_or_escalates() {
);
assert_eq!(state.iam_deletion_replays.len(), 1);
assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:bob");
// A newer deletion of the same entity gets a fresh replay-record id. An
// older settlement therefore removes neither its body nor its queue
// revision, even if the persisted timestamps happen to be equal.
let mut state = deletion_replay_state(&target);
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "first failure").expect("record failure");
state.retry_queue[0].updated_at = Some(snapshot_at);
let observed = state.retry_queue[0].clone();
let replayed = vec![state.iam_deletion_replays[0].id.clone()];
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "newer failure").expect("record newer failure");
state.retry_queue[0].updated_at = Some(snapshot_at);
assert_ne!(state.iam_deletion_replays[0].id, replayed[0]);
assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed));
assert_eq!(state.retry_queue.len(), 1);
assert_ne!(state.retry_queue[0].id, observed.id);
assert_eq!(state.iam_deletion_replays.len(), 1);
assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:alice");
}
/// Merging legacy wire-path rows into the collapsed entry must not launder an
@@ -748,6 +791,281 @@ fn test_classify_site_replication_retry_event_actions() {
assert_eq!(classify("/rustfs/admin/v3/site-replication/peer/unknown"), None);
}
#[test]
fn test_bucket_make_retry_replays_matching_configure_before_settlement() {
let make_photos =
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string();
let configure_photos =
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication".to_string();
let plan = SiteReplicationBootstrapPlan {
bucket_make_ops: vec![
make_photos.clone(),
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=videos&operation=make-with-versioning".to_string(),
],
bucket_configure_ops: vec![
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=videos&operation=configure-replication".to_string(),
configure_photos.clone(),
],
bucket_items: vec![
SRBucketMeta {
bucket: "videos".to_string(),
r#type: "tags".to_string(),
..Default::default()
},
SRBucketMeta {
bucket: "photos".to_string(),
r#type: "policy".to_string(),
..Default::default()
},
],
..Default::default()
};
let tasks = bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos")
.expect("make retry plan should include its configure follow-up");
assert_eq!(
tasks.iter().map(SiteReplicationRepairTask::path).collect::<Vec<_>>(),
vec![
make_photos.as_str(),
"/rustfs/admin/v3/site-replication/peer/bucket-meta",
configure_photos.as_str()
]
);
assert!(matches!(tasks[0], SiteReplicationRepairTask::BucketMake(_)));
assert!(matches!(&tasks[1], SiteReplicationRepairTask::BucketMetadata(item) if item.bucket == "photos"));
assert!(matches!(tasks[2], SiteReplicationRepairTask::Replication(_)));
}
#[test]
fn test_bucket_make_retry_without_matching_configure_fails_closed() {
let plan = SiteReplicationBootstrapPlan {
bucket_make_ops: vec![
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string(),
],
..Default::default()
};
let err = match bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos") {
Ok(_) => panic!("make retry must not settle without a matching configure operation"),
Err(err) => err,
};
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[test]
fn test_retry_drain_bounds_each_peer_round_to_one_small_request_chain() {
let plan = SiteReplicationBootstrapPlan {
iam_items: vec![SRIAMItem::default(); 3],
bucket_make_ops: vec![
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string(),
],
bucket_configure_ops: vec![
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication".to_string(),
],
bucket_items: vec![SRBucketMeta {
bucket: "photos".to_string(),
r#type: "tags".to_string(),
..Default::default()
}],
..Default::default()
};
let make = RetryDrainAction::BucketOpReplay {
operation: SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING.to_string(),
bucket: "photos".to_string(),
};
assert!(is_lightweight_retry_drain_action(&make));
assert!(!is_lightweight_retry_drain_action(&RetryDrainAction::IamSnapshot));
assert!(!is_lightweight_retry_drain_action(&RetryDrainAction::PeerEdit));
assert_eq!(retry_drain_request_count(&make, Some(&plan)), 3);
assert!(retry_drain_request_count(&make, Some(&plan)) <= SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER);
assert!(
retry_drain_request_count(&RetryDrainAction::IamSnapshot, Some(&plan))
> SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER
);
assert!(
retry_drain_request_count(&RetryDrainAction::PeerEdit, Some(&plan)) > SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER
);
}
#[test]
fn test_lightweight_retry_peer_rotation_covers_all_queued_peers() {
let limit = SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY;
for peer_count in 1..=(limit * 3 + 1) {
let rounds = peer_count.div_ceil(limit);
let mut seen = HashSet::new();
for round in 7..(7 + rounds as i64) {
let start = lightweight_retry_peer_rotation(peer_count, round);
for offset in 0..limit.min(peer_count) {
seen.insert((start + offset) % peer_count);
}
}
assert_eq!(
seen.len(),
peer_count,
"every peer must enter the bounded lightweight window within {rounds} rounds"
);
}
}
#[test]
fn test_lightweight_bucket_retry_plan_is_targeted_and_preserves_make_options() {
let created_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
let bucket = SRBucketInfo {
bucket: "photos".to_string(),
created_at: Some(created_at),
object_lock_config: Some(String::new()),
tags: Some("dGFncy14bWw=".to_string()),
tag_config_updated_at: Some(created_at),
..Default::default()
};
let plan = site_replication_bucket_retry_plan_for(&bucket, false).expect("targeted retry plan");
assert!(plan.iam_items.is_empty());
assert_eq!(plan.bucket_make_ops.len(), 1);
assert!(plan.bucket_make_ops[0].contains("bucket=photos"));
assert!(plan.bucket_make_ops[0].contains("lockEnabled=true"));
assert!(plan.bucket_make_ops[0].contains("createdAt="));
assert_eq!(plan.bucket_items.len(), 2);
assert_eq!(plan.bucket_items[0].r#type, "tags");
assert_eq!(plan.bucket_items[1].r#type, "object-lock-config");
assert_eq!(plan.bucket_configure_ops.len(), 1);
assert!(plan.bucket_configure_ops[0].contains("operation=configure-replication"));
let tasks =
bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos").expect("retry task chain");
assert!(matches!(tasks[0], SiteReplicationRepairTask::BucketMake(_)));
assert!(matches!(tasks[1], SiteReplicationRepairTask::BucketMetadata(_)));
assert!(matches!(tasks[2], SiteReplicationRepairTask::BucketMetadata(_)));
assert!(matches!(tasks[3], SiteReplicationRepairTask::Replication(_)));
}
#[test]
fn test_lightweight_bucket_retry_plan_orders_real_metadata_and_counts_it() {
let versioning = bucket_versioning_xml().expect("canonical versioning config");
let replication = serialize(&site_repl_config("remote-dep")).expect("derived replication config");
let bucket = SRBucketInfo {
bucket: "photos".to_string(),
policy: Some(serde_json::json!({"Version":"2012-10-17","Statement":[]})),
tags: Some(BASE64_STANDARD.encode_to_string("<Tagging/>")),
versioning: Some(BASE64_STANDARD.encode_to_string(&versioning)),
replication_config: Some(BASE64_STANDARD.encode_to_string(&replication)),
..Default::default()
};
let plan = site_replication_bucket_retry_plan_from_info(&bucket, false).expect("targeted retry plan");
let tasks = bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos")
.expect("bucket replay tasks");
assert!(matches!(tasks.first(), Some(SiteReplicationRepairTask::BucketMake(_))));
assert!(matches!(tasks.last(), Some(SiteReplicationRepairTask::Replication(_))));
assert!(
tasks[1..tasks.len() - 1]
.iter()
.all(|task| matches!(task, SiteReplicationRepairTask::BucketMetadata(_)))
);
assert_eq!(tasks.len(), 4, "make + policy + tags + configure must all count against the budget");
assert!(
tasks.len() <= SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER,
"the complete metadata chain must fit the bounded lightweight replay"
);
let mut operator_replication = site_repl_config("remote-dep");
operator_replication.rules.push(operator_rule("operator-backup"));
let mut bucket_with_operator_rule = bucket;
bucket_with_operator_rule.replication_config =
Some(BASE64_STANDARD.encode_to_string(&serialize(&operator_replication).expect("operator replication config")));
let plan = site_replication_bucket_retry_plan_from_info(&bucket_with_operator_rule, false).expect("targeted retry plan");
assert!(
plan.bucket_items.iter().any(|item| item.r#type == "replication-config"),
"operator-authored replication rules cannot be replaced by configure-replication"
);
}
#[test]
fn test_delete_bucket_broadcast_fences_target_membership_through_delivery() {
let hooks = include_str!("hooks.rs");
let delete_broadcast = hooks
.split("async fn broadcast_site_replication_delete_bucket")
.nth(1)
.and_then(|rest| rest.split("pub(crate) async fn commit_site_replication_delete_bucket").next())
.expect("delete-bucket broadcast should exist");
assert!(
delete_broadcast.contains("with_site_replication_state_read_lock(move |state| async move {")
&& delete_broadcast.contains("state.peers.get(&fallback_peer.deployment_id)")
&& delete_broadcast.contains("site_replicator_service_account_secret(&state.service_account_access_key)")
&& delete_broadcast
.contains("PeerAdminRequest::put(&transport.connection, &delivery_path, &state.service_account_access_key)"),
"a destructive bucket delivery must resolve current topology and credentials under the distributed state read lock"
);
assert!(
delete_broadcast.contains("enqueue_site_replication_retry_event(&current_peer, &request_path, &err).await"),
"a failed destructive delivery must remain visible for operator repair"
);
let usecase = include_str!("../app/bucket_usecase.rs");
let delete = usecase
.split("async fn execute_delete_bucket_inner")
.nth(1)
.and_then(|rest| rest.split("pub async fn execute_head_bucket").next())
.expect("delete bucket usecase");
assert!(
delete
.find("prepare_site_replication_delete_bucket")
.expect("durable reservation")
< delete.find(".delete_bucket(").expect("local delete"),
"destructive peer liabilities must be persisted before the local bucket is deleted"
);
}
#[test]
fn test_bucket_retry_settlement_preserves_a_newer_same_path_failure() {
let peer = peer("remote", "https://remote.example.com");
let path = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication";
let observed_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
let observed = drain_event("remote", path, 1, Some(observed_at));
let mut queue = vec![observed.clone()];
queue[0].id = "evt-remote-new-revision".to_string();
queue[0].retry_count += 1;
assert_eq!(settle_observed_site_replication_retry_event(&mut queue, &peer, &observed), 0);
assert_eq!(queue.len(), 1, "a newer same-timestamp failure must survive stale settlement");
let current = queue[0].clone();
assert_eq!(settle_observed_site_replication_retry_event(&mut queue, &peer, &current), 1);
assert!(queue.is_empty());
}
#[test]
fn test_reachable_probe_promotion_is_fenced_by_the_observed_event() {
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
let path = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning";
let mut event = drain_event("remote", path, 3, Some(now));
event.peer_unreachable = true;
let recovered = event.clone();
let mut state = SiteReplicationState {
retry_queue: vec![event],
..Default::default()
};
state
.peers
.insert("remote".to_string(), peer("remote", "https://remote.example.com"));
assert_eq!(mark_reachable_deferred_retry_events(&mut state, &[recovered.clone()]), 1);
assert_eq!(state.retry_queue[0].updated_at, None);
assert!(!state.retry_queue[0].peer_unreachable);
assert_eq!(
actionable_site_replication_retry_events(&state, now).len(),
1,
"a successful probe must make the event replayable in the same drain tick"
);
state.retry_queue[0].updated_at = Some(now + time::Duration::seconds(1));
state.retry_queue[0].peer_unreachable = true;
assert_eq!(mark_reachable_deferred_retry_events(&mut state, &[recovered]), 0);
assert_eq!(state.retry_queue[0].updated_at, Some(now + time::Duration::seconds(1)));
assert!(state.retry_queue[0].peer_unreachable);
}
#[test]
fn test_retry_snapshot_fingerprint_detects_concurrent_iam_change() {
let old = SRIAMItem {
@@ -828,6 +1146,100 @@ fn test_site_replication_retry_backoff_schedule() {
assert!(elapsed(30, 86_401));
}
#[test]
fn test_retry_error_marks_peer_unreachable_only_for_connection_failures() {
let mut queue = Vec::new();
let peer = peer("remote", "https://remote.example.com");
let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning";
upsert_site_replication_retry_event(
&mut queue,
&peer,
bucket_make,
"peer request to https://remote.example.com failed (connect): connection refused",
None,
)
.expect("upsert retry event");
assert!(queue[0].peer_unreachable);
upsert_site_replication_retry_event(
&mut queue,
&peer,
bucket_make,
"peer request to https://remote.example.com failed (timeout): request exceeded 10 seconds",
None,
)
.expect("upsert retry event");
assert!(
!queue[0].peer_unreachable,
"a whole-request timeout does not prove the peer is unreachable"
);
upsert_site_replication_retry_event(
&mut queue,
&peer,
bucket_make,
"peer request to https://remote.example.com failed with 500 Internal Server Error: downstream failed (connect)",
None,
)
.expect("upsert retry event");
assert!(
!queue[0].peer_unreachable,
"application failures and their untrusted bodies must keep the normal replay backoff"
);
upsert_site_replication_retry_event(
&mut queue,
&peer,
bucket_make,
"peer request to https://remote.example.com failed with 500 Internal Server Error: backend failed (connect): spoofed",
None,
)
.expect("upsert retry event");
assert!(!queue[0].peer_unreachable, "peer response bodies must not spoof transport failures");
}
#[test]
fn test_connect_timeout_is_classified_as_a_connection_failure() {
assert_eq!(classify_peer_transport_error(true, true, "tcp connect timed out"), "connect");
assert_eq!(classify_peer_transport_error(false, true, "request timed out"), "timeout");
assert_eq!(
classify_peer_transport_error(false, true, "request timed out for https://tls-gateway.example"),
"timeout"
);
assert_eq!(classify_peer_transport_error(true, false, "tls handshake failed"), "tls handshake");
}
#[test]
fn test_retry_event_peer_unreachable_is_legacy_serde_default() {
let json = r#"{
"id":"evt-legacy",
"peer_deployment_id":"remote",
"peer_endpoint":"https://remote.example.com",
"path":"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning",
"retry_count":1,
"failed":false,
"last_error":"peer request to https://remote.example.com failed (connect): connection refused"
}"#;
let mut event: SiteReplicationRetryEvent = serde_json::from_str(json).expect("legacy retry event decodes");
assert!(!event.peer_unreachable);
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
event.updated_at = Some(now - time::Duration::seconds(30));
let mut state = SiteReplicationState::default();
state
.peers
.insert("remote".to_string(), peer("remote", "https://remote.example.com"));
state.retry_queue.push(event);
assert_eq!(
deferred_site_replication_retry_events(&state, now).len(),
1,
"rolling-upgrade records must retain fast recovery from their trusted outer error shape"
);
}
/// The actionable subset respects classification, peer membership and
/// backoff; everything else stays untouched in the queue.
#[test]
@@ -915,6 +1327,51 @@ fn test_deferred_site_replication_retry_events_partition() {
assert_eq!(actionable[0].path, "/rustfs/admin/v3/site-replication/peer/bucket-meta");
}
#[test]
fn test_deferred_retry_events_probe_fresh_peer_transport_failures() {
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
let mut state = SiteReplicationState::default();
state
.peers
.insert("remote".to_string(), peer("remote", "https://remote.example.com"));
let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning";
let mut fresh_transport_failure = drain_event("remote", bucket_make, 1, Some(now - time::Duration::seconds(30)));
fresh_transport_failure.peer_unreachable = true;
state.retry_queue.push(fresh_transport_failure);
let deferred = deferred_site_replication_retry_events(&state, now);
assert_eq!(
deferred.len(),
1,
"fresh transport failures must be eligible for a cheap reachability probe"
);
assert_eq!(deferred[0].path, bucket_make);
let actionable = actionable_site_replication_retry_events(&state, now);
assert!(actionable.is_empty(), "the event is still protected from direct replay by normal backoff");
}
#[test]
fn test_deferred_retry_events_do_not_probe_fresh_application_failures() {
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
let mut state = SiteReplicationState::default();
state
.peers
.insert("remote".to_string(), peer("remote", "https://remote.example.com"));
let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning";
state
.retry_queue
.push(drain_event("remote", bucket_make, 1, Some(now - time::Duration::seconds(30))));
assert!(
deferred_site_replication_retry_events(&state, now).is_empty(),
"reachable peers that reject an operation must keep the base replay backoff"
);
assert!(actionable_site_replication_retry_events(&state, now).is_empty());
}
/// The drain settles a peer-edit success under a freshly allocated
/// generation; legacy queue entries carry `edit_generation: None` and
/// must be cleared by that generation-scoped settlement (`(Some, None)`
@@ -996,7 +1453,7 @@ fn test_escalate_up_to_marks_snapshot_replayed_and_keeps_newer_failures() {
// successful Bob update on the shared wire path cannot erase it even
// before the drain runs.
let mut queue = Vec::new();
upsert_site_replication_retry_event(&mut queue, &target, path, "alice delete failed", None);
upsert_site_replication_retry_event(&mut queue, &target, path, "alice delete failed", None).expect("upsert retry event");
assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH);
assert_eq!(dequeue_site_replication_retry_events(&mut queue, &target, path), 0);
assert_eq!(queue.len(), 1);
@@ -1005,7 +1462,7 @@ fn test_escalate_up_to_marks_snapshot_replayed_and_keeps_newer_failures() {
// A later hook failure overwrites the marker and re-arms the drain.
let mut queue = vec![drain_event("remote", path, 2, Some(snapshot_at))];
escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at));
upsert_site_replication_retry_event(&mut queue, &target, path, "peer offline", None);
upsert_site_replication_retry_event(&mut queue, &target, path, "peer offline", None).expect("upsert retry event");
assert!(classify_site_replication_retry_event(&queue[0]).is_some());
// Legacy entry without a timestamp: escalated.
@@ -1781,17 +2238,85 @@ fn test_retry_event_upsert_marks_repeated_failures() {
};
let mut queue = Vec::new();
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "first", None);
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "second", None);
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "third", None);
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "first", None)
.expect("upsert retry event");
let first_revision = queue[0].id.clone();
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "second", None)
.expect("upsert retry event");
let second_revision = queue[0].id.clone();
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "third", None)
.expect("upsert retry event");
assert_eq!(queue.len(), 1);
assert_ne!(first_revision, second_revision);
assert_ne!(second_revision, queue[0].id, "each failure must advance the settlement revision");
assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH);
assert_eq!(queue[0].retry_count, SITE_REPLICATION_RETRY_FAILED_AFTER);
assert!(queue[0].failed);
assert_eq!(queue[0].last_error, "third");
}
#[test]
fn retry_queue_capacity_never_evicts_destructive_bucket_liabilities() {
let target = PeerInfo {
deployment_id: "remote-dep".to_string(),
..peer("remote", "https://remote.example.com")
};
let destructive = |index: usize| SiteReplicationRetryEvent {
id: format!("delete-{index}"),
peer_deployment_id: target.deployment_id.clone(),
peer_endpoint: target.endpoint.clone(),
path: format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?bucket=bucket-{index}&operation=delete-bucket"),
..Default::default()
};
let mut queue = (0..SITE_REPLICATION_RETRY_QUEUE_LIMIT).map(destructive).collect::<Vec<_>>();
let original_ids = queue.iter().map(|event| event.id.clone()).collect::<HashSet<_>>();
let new_path = format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?bucket=overflow&operation=force-delete-bucket");
let err = upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None)
.expect_err("an all-destructive full queue must fail closed");
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(queue.len(), SITE_REPLICATION_RETRY_QUEUE_LIMIT);
assert_eq!(queue.iter().map(|event| event.id.clone()).collect::<HashSet<_>>(), original_ids);
queue[0] = SiteReplicationRetryEvent {
id: "iam-snapshot".to_string(),
peer_deployment_id: target.deployment_id.clone(),
peer_endpoint: target.endpoint.clone(),
path: SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH.to_string(),
deletions_recorded: true,
..Default::default()
};
upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None)
.expect_err("a collapsed IAM liability may contain a deletion and must not be evicted");
assert!(queue.iter().any(|event| event.id == "iam-snapshot"));
queue[0] = SiteReplicationRetryEvent {
id: "rebuildable".to_string(),
peer_deployment_id: target.deployment_id.clone(),
peer_endpoint: target.endpoint.clone(),
path: SITE_REPLICATION_PEER_EDIT_PATH.to_string(),
..Default::default()
};
let preserved_delete_ids = queue
.iter()
.filter(|event| is_destructive_bucket_retry_path(&event.path))
.map(|event| event.id.clone())
.collect::<HashSet<_>>();
let evicted = upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None)
.expect("a rebuildable row may make room for a destructive liability");
assert_eq!(evicted.len(), 1);
assert_eq!(evicted[0].id, "rebuildable");
assert_eq!(queue.len(), SITE_REPLICATION_RETRY_QUEUE_LIMIT);
assert!(
preserved_delete_ids
.iter()
.all(|id| queue.iter().any(|event| &event.id == id))
);
assert!(queue.iter().any(|event| event.path == new_path));
}
/// P1-15 review follow-up: a successful peer-edit delivery only proves the
/// peer reached the state THAT delivery carried. Settling it must not
/// erase a retry event a newer edit left behind, or the local site sits on
@@ -1807,7 +2332,8 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() {
// Edit A (generation 5) delivered successfully and is stalled before
// settling. Edit B (generation 6) commits meanwhile, fails delivery to
// the same peer, and enqueues.
upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "peer offline", Some(6));
upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "peer offline", Some(6))
.expect("upsert retry event");
// A resumes: its own settlement must leave B's retry alone.
assert_eq!(
@@ -1818,7 +2344,8 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() {
assert_eq!(queue[0].edit_generation, Some(6));
// An even older delivery failing afterwards must not lower the fence.
upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "still offline", Some(4));
upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "still offline", Some(4))
.expect("upsert retry event");
assert_eq!(queue[0].edit_generation, Some(6));
// B's own delivery succeeding is what clears it.
@@ -1831,7 +2358,7 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() {
// Collapsed broadcast failures live under an internal snapshot path;
// an unrelated success on their shared wire path cannot settle them.
let iam_path = "/rustfs/admin/v3/site-replication/peer/iam-item";
upsert_site_replication_retry_event(&mut queue, &peer, iam_path, "peer offline", None);
upsert_site_replication_retry_event(&mut queue, &peer, iam_path, "peer offline", None).expect("upsert retry event");
assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, iam_path), 0);
assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH);
}
rustfs/src/site_replication/transport.rs
+17 -13
View File
@@ -331,6 +331,7 @@ pub(crate) fn runtime_peer_connection(peer: &PeerInfo) -> S3Result<PeerConnectio
})
}
#[derive(Clone)]
pub(crate) struct PeerTransport {
pub(crate) connection: PeerConnection,
pub(crate) client: reqwest::Client,
@@ -697,19 +698,7 @@ impl<'a> PeerAdminRequest<'a> {
}
let response = req.send().await.map_err(|e| {
let classify = if e.is_timeout() {
"timeout"
} else if e.is_connect() && e.to_string().to_ascii_lowercase().contains("dns") {
"dns resolution"
} else if e.to_string().to_ascii_lowercase().contains("certificate")
|| e.to_string().to_ascii_lowercase().contains("tls")
{
"tls handshake"
} else if e.is_connect() {
"connect"
} else {
"request"
};
let classify = classify_peer_transport_error(e.is_connect(), e.is_timeout(), &e.to_string());
S3Error::with_message(S3ErrorCode::InternalError, format!("peer request to {url} failed ({classify}): {e}"))
})?;
@@ -825,6 +814,21 @@ impl<'a> PeerAdminRequest<'a> {
}
}
pub(crate) fn classify_peer_transport_error(is_connect: bool, is_timeout: bool, detail: &str) -> &'static str {
let detail = detail.to_ascii_lowercase();
if is_connect && detail.contains("dns") {
"dns resolution"
} else if is_connect && (detail.contains("certificate") || detail.contains("tls")) {
"tls handshake"
} else if is_connect {
"connect"
} else if is_timeout {
"timeout"
} else {
"request"
}
}
pub(crate) fn peer_error_may_be_secret_mismatch(detail: &str) -> bool {
let detail = detail.to_ascii_lowercase();
detail.contains("signaturedoesnotmatch")
rustfs/src/site_replication_reconcile.rs
+45 -3
View File
@@ -28,16 +28,19 @@ use std::pin::Pin;
use std::sync::OnceLock;
use std::time::Duration;
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tracing::warn;
const RECONCILE_INTERVAL: Duration = Duration::from_secs(600);
pub(crate) const RETRY_DRAIN_INTERVAL: Duration = Duration::from_secs(30);
/// A reconciler reports its own failures; the outcome carries no value because neither
/// caller can act on one — a site that cannot repair its replication wiring still serves S3.
type ReconcileHook = fn() -> Pin<Box<dyn Future<Output = ()> + Send>>;
static RECONCILER: OnceLock<ReconcileHook> = OnceLock::new();
static RETRY_DRAINER: OnceLock<ReconcileHook> = OnceLock::new();
/// Install the admin layer's reconciler. Idempotent: a second call is ignored, which keeps
/// repeated router construction (tests, the embedded server) from panicking.
@@ -45,6 +48,12 @@ pub(crate) fn register_site_replication_reconciler(reconcile: ReconcileHook) {
let _ = RECONCILER.set(reconcile);
}
/// Install the admin layer's lightweight retry drain. Idempotent for the same
/// reason as [`register_site_replication_reconciler`].
pub(crate) fn register_site_replication_retry_drainer(drain: ReconcileHook) {
let _ = RETRY_DRAINER.set(drain);
}
/// Repair drifted site-replication wiring, immediately and then on a timer.
///
/// The first pass runs inside the spawned task rather than on the caller's path: it walks
@@ -62,16 +71,38 @@ pub(crate) fn spawn_site_replication_reconcile_task(ctx: CancellationToken) {
return;
}
spawn_reconcile_loop(ctx.clone(), RECONCILE_INTERVAL, &RECONCILER, true);
if RETRY_DRAINER.get().is_none() {
warn!("site replication retry drainer is not registered; periodic retry drain disabled");
return;
}
spawn_reconcile_loop(ctx, RETRY_DRAIN_INTERVAL, &RETRY_DRAINER, false);
}
fn spawn_reconcile_loop(
ctx: CancellationToken,
interval: Duration,
hook: &'static OnceLock<ReconcileHook>,
run_immediately: bool,
) {
tokio::spawn(async move {
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL);
let first_tick = if run_immediately {
Instant::now()
} else {
Instant::now() + interval
};
let mut ticker = tokio::time::interval_at(first_tick, interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tokio::select! {
_ = ctx.cancelled() => break,
// The first tick fires immediately, which is the startup repair pass.
// The heavy reconciler owns the startup repair pass. The lightweight
// retry drain starts on its normal cadence so it cannot steal that
// first lifecycle lock and defer bucket/IAM repair for a full interval.
_ = ticker.tick() => {
if let Some(reconcile) = RECONCILER.get() {
if let Some(reconcile) = hook.get() {
reconcile().await;
}
}
@@ -79,3 +110,14 @@ pub(crate) fn spawn_site_replication_reconcile_task(ctx: CancellationToken) {
}
});
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn retry_drain_runs_faster_than_heavy_reconcile() {
assert!(RETRY_DRAIN_INTERVAL < RECONCILE_INTERVAL);
assert!(RETRY_DRAIN_INTERVAL <= Duration::from_secs(60));
}
}
rustfs/src/storage/rpc/node_service.rs
+200
View File
@@ -493,6 +493,13 @@ impl std::fmt::Debug for NodeService {
}
}
pub(crate) fn make_scanner_control_server() -> scanner_control_service_server::ScannerControlServiceServer<NodeService> {
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
scanner_control_service_server::ScannerControlServiceServer::new(make_server())
.max_decoding_message_size(limit)
.max_encoding_message_size(limit)
}
pub fn make_server() -> NodeService {
let context = runtime_sources::current_app_context();
make_server_for_context(context)
@@ -1087,6 +1094,74 @@ impl NodeService {
}
}
#[tonic::async_trait]
impl scanner_control_service_server::ScannerControlService for NodeService {
async fn scanner_scoped_dirty_usage_ack(
&self,
request: Request<ScannerScopedDirtyUsageAckRequest>,
) -> Result<Response<ScannerScopedDirtyUsageAckResponse>, Status> {
use rustfs_protos::scoped_dirty_usage::*;
static ADMISSION: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4);
let canonical =
canonical_scoped_dirty_usage_request(request.get_ref()).map_err(|err| Status::invalid_argument(err.to_string()))?;
verify_tonic_canonical_body_digest(&request, &canonical)
.map_err(|_| Status::permission_denied("scoped dirty usage authentication failed"))?;
let _admission = ADMISSION
.try_acquire()
.map_err(|_| Status::resource_exhausted("scoped dirty usage receiver is busy"))?;
let request = request.into_inner();
let store = self
.resolve_object_store()
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
if store.id.is_nil()
|| request.owner_id != store.id.to_string()
|| request.instance_id != rustfs_scanner::scanner_activity_epoch()
{
return Err(Status::failed_precondition("scoped dirty usage peer or process changed"));
}
let cleared = timeout(Duration::from_secs(30), async {
// Strict bucket order is validated before admission. Acquire every
// lifecycle/metadata fence before clearing any dirty record.
let mut guards = Vec::with_capacity(request.entries.len());
for entry in &request.entries {
let incarnation = Uuid::from_slice(entry.bucket_incarnation.as_ref())
.map_err(|_| Status::invalid_argument("invalid bucket incarnation"))?;
let guard =
crate::storage::storage_api::acquire_scanner_bucket_incarnation_fence(&entry.bucket, incarnation, store.id)
.await
.map_err(|_| Status::failed_precondition("trusted bucket incarnation is unavailable"))?;
guards.push(guard);
}
let entries = guards
.iter()
.zip(&request.entries)
.map(|(guard, entry)| (guard, entry.generation))
.collect::<Vec<_>>();
rustfs_scanner::acknowledge_scoped_dirty_usage(&request.instance_id, &entries, request.probe_only)
.map_err(|err| Status::failed_precondition(err.to_string()))
})
.await
.map_err(|_| Status::deadline_exceeded("scoped dirty usage incarnation validation timed out"))??;
let mut response = ScannerScopedDirtyUsageAckResponse {
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
owner_id: request.owner_id,
instance_id: request.instance_id,
supported: true,
max_entries: SCOPED_DIRTY_USAGE_MAX_ENTRIES,
max_request_bytes: SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES,
cleared,
response_proof: Bytes::new(),
};
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
.map_err(|_| Status::internal("scoped dirty usage response is too large"))?;
response.response_proof = sign_tonic_rpc_response_proof(&body)
.map_err(|_| Status::unavailable("scoped dirty usage response authentication is unavailable"))?
.into();
Ok(Response::new(response))
}
}
#[tonic::async_trait]
impl Node for NodeService {
async fn ping(&self, request: Request<PingRequest>) -> Result<Response<PingResponse>, Status> {
@@ -2623,6 +2698,7 @@ mod tests {
use rustfs_kms::KmsServiceManager;
use rustfs_protos::CanonicalMutationBody as _;
use rustfs_protos::models::PingBodyBuilder;
use rustfs_protos::proto_gen::node_service::scanner_control_service_server::ScannerControlService as _;
use rustfs_protos::proto_gen::node_service::{
BackgroundHealStatusRequest, BatchGenerallyLockRequest, CancelDecommissionRequest, CheckPartsRequest,
ClearDecommissionRequest, ControlPlaneErrorCode, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest,
@@ -5990,6 +6066,74 @@ mod tests {
);
}
fn scoped_dirty_usage_request() -> rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
challenge: vec![7; 16].into(),
protocol_version: 1,
owner_id: "11111111-1111-1111-1111-111111111111".into(),
instance_id: "a".repeat(32),
scope: 1,
probe_only: false,
entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
bucket: "photos".into(),
bucket_incarnation: vec![1; 16].into(),
generation: 8,
}],
}
}
#[tokio::test]
async fn scoped_dirty_usage_authenticates_before_storage_and_rejects_tampering() {
use rustfs_protos::scoped_dirty_usage::canonical_scoped_dirty_usage_request;
let service = create_test_node_service();
let unsigned = service
.scanner_scoped_dirty_usage_ack(Request::new(scoped_dirty_usage_request()))
.await
.expect_err("unsigned ACK must not access storage");
assert_eq!(unsigned.code(), tonic::Code::PermissionDenied);
for field in 0..9 {
let mut signed = Request::new(scoped_dirty_usage_request());
let canonical = canonical_scoped_dirty_usage_request(signed.get_ref()).expect("canonical request");
set_tonic_canonical_body_digest(&mut signed, &canonical).expect("digest");
mark_v2_authenticated(&mut signed);
match field {
0 => signed.get_mut().challenge = vec![3; 16].into(),
1 => signed.get_mut().owner_id = "22222222-2222-2222-2222-222222222222".into(),
2 => signed.get_mut().instance_id = "b".repeat(32),
3 => signed.get_mut().probe_only = true,
4 => signed.get_mut().entries[0].bucket = "videos".into(),
5 => signed.get_mut().entries[0].bucket_incarnation = vec![2; 16].into(),
6 => signed.get_mut().entries[0].generation += 1,
7 => signed.get_mut().scope += 1,
_ => signed.get_mut().protocol_version += 1,
}
let error = service
.scanner_scoped_dirty_usage_ack(signed)
.await
.expect_err("tampered ACK must fail");
assert_eq!(
error.code(),
if field < 7 {
tonic::Code::PermissionDenied
} else {
tonic::Code::InvalidArgument
}
);
}
let mut signed = Request::new(scoped_dirty_usage_request());
let canonical = canonical_scoped_dirty_usage_request(signed.get_ref()).expect("canonical request");
set_tonic_canonical_body_digest(&mut signed, &canonical).expect("digest");
mark_v2_authenticated(&mut signed);
assert_eq!(
service
.scanner_scoped_dirty_usage_ack(signed)
.await
.expect_err("missing owner cannot advertise capability")
.code(),
tonic::Code::Unavailable
);
}
#[tokio::test]
async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() {
let service = create_test_node_service();
@@ -6485,6 +6629,62 @@ mod tests {
)
}
#[tokio::test]
async fn scoped_dirty_usage_transport_rejects_oversized_unknown_and_duplicate_fields() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind scoped ACK transport test");
let addr = listener.local_addr().expect("test listener address");
let (shutdown, stopped) = tokio::sync::oneshot::channel();
let server = tokio::spawn(async move {
tonic::transport::Server::builder()
.add_service(super::make_scanner_control_server())
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
let _ = stopped.await;
})
.await
.expect("scoped ACK transport server");
});
let client = reqwest::Client::builder()
.no_proxy()
.http2_prior_knowledge()
.build()
.expect("HTTP/2 client");
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
for tag in [0x78, 0x0a] {
// Unknown varint field 15, or repeated empty singular challenge:
// both decode to a tiny default struct despite the large wire body.
for oversized in [false, true] {
let mut payload = [tag, 0].repeat(if oversized { (limit - 4) / 2 } else { limit / 2 });
if oversized {
// Unknown fixed32 field 15 makes a valid cap+1 protobuf.
payload.extend_from_slice(&[0x7d, 0, 0, 0, 0]);
}
assert_eq!(payload.len(), limit + usize::from(oversized));
let mut frame = vec![0];
frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test payload").to_be_bytes());
frame.extend_from_slice(&payload);
let response = client
.post(format!("http://{addr}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"))
.header("content-type", "application/grpc")
.header("te", "trailers")
.body(frame)
.send()
.await
.expect("send raw protobuf frame");
let status = response.headers().get("grpc-status").expect("gRPC failure status");
assert_eq!(
status.to_str().expect("status text"),
if oversized { "11" } else { "3" },
"cap+1 must fail in the codec, while cap bytes reach request validation"
);
}
}
drop(client);
shutdown.send(()).expect("stop test server");
server.await.expect("join test server");
}
#[tokio::test]
async fn heal_control_transport_enforces_codec_limit_and_fails_closed() {
let Some(mut client) = connect_test_heal_control_client().await else {
rustfs/src/storage/storage_api.rs
+9 -1
View File
@@ -379,7 +379,7 @@ pub(crate) mod tonic_service_consumer {
#[cfg(test)]
pub(crate) use super::super::tonic_service::{heal_topology_fingerprint, make_heal_control_server_for_source};
pub(crate) use super::super::tonic_service::{
make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server,
make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server,
};
}
@@ -1704,6 +1704,14 @@ pub(crate) async fn acquire_bucket_metadata_transaction_lock(
ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await
}
pub(crate) async fn acquire_scanner_bucket_incarnation_fence(
bucket: &str,
incarnation: uuid::Uuid,
owner_id: uuid::Uuid,
) -> Result<ecstore_bucket::metadata_sys::BucketMetadataMutationGuard> {
ecstore_bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence(bucket, incarnation, owner_id).await
}
pub(crate) async fn update_bucket_targets_under_transaction_lock(
guard: &ecstore_bucket::metadata_sys::BucketMetadataMutationGuard,
bucket: &str,
rustfs/src/storage/tonic_service.rs
+1
View File
@@ -13,6 +13,7 @@
// limitations under the License.
pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_cache;
pub(crate) use crate::storage::rpc::node_service::make_scanner_control_server;
#[cfg(test)]
pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source};
pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server};
rustfs/src/storage_api.rs
+3 -3
View File
@@ -176,7 +176,7 @@ pub(crate) mod server {
heal_topology_fingerprint, make_heal_control_server_for_source,
};
pub(crate) use crate::storage::storage_api::tonic_service_consumer::{
make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server,
make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server,
};
}
}
@@ -244,8 +244,8 @@ pub(crate) mod site_replication {
pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints};
pub(crate) use crate::storage::storage_api::{
ECStore, EndpointServerPools, StorageError, delete_config_no_lock, lock_bucket_targets_metadata, read_config,
read_config_no_lock, save_config_no_lock, with_config_object_read_lock, with_config_object_write_lock,
ECStore, EndpointServerPools, StorageError, delete_config_no_lock, is_err_bucket_not_found, lock_bucket_targets_metadata,
read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock, with_config_object_write_lock,
};
pub(crate) mod metadata_sys {
scripts/check_scheduled_validation_freshness.py
+292 -106
View File
@@ -1,10 +1,11 @@
#!/usr/bin/env python3
"""Fail when a critical scheduled validation has not started recently."""
"""Require recent scheduled attempts and completed successes on the default branch."""
from __future__ import annotations
import argparse
from datetime import datetime, timedelta, timezone
import io
import json
import os
from pathlib import Path
@@ -13,7 +14,7 @@ import sys
import tempfile
import unittest
from unittest import mock
from urllib.parse import quote, urlencode
from urllib.parse import parse_qs, quote, urlencode, urlsplit
from urllib.request import Request, urlopen
@@ -75,15 +76,8 @@ def stale_reason(
run: dict[str, object] | None,
now: datetime,
max_age_hours: int,
never_ran_grace_until: datetime | None = None,
) -> str | None:
if run is None:
# The grace deadline only covers a workflow whose first scheduled slot
# has not arrived yet (for example a monthly cron enabled mid-month).
# A recorded-but-old run proves the schedule used to fire and stopped,
# so the grace never masks that case.
if never_ran_grace_until is not None and now <= never_ran_grace_until:
return None
return "no scheduled run has been recorded"
created_at = parse_timestamp(run.get("created_at"))
age = now - created_at
@@ -93,14 +87,23 @@ def stale_reason(
def fetch_latest_scheduled_run(
repository: str, workflow: str, token: str, api_url: str
repository: str,
workflow: str,
token: str,
api_url: str,
default_branch: str,
successful: bool = False,
) -> dict[str, object] | None:
owner, repo = repository.split("/", 1)
workflow_name = Path(workflow).name
query = {"event": "schedule", "branch": default_branch, "per_page": 1}
if successful:
# Filter on the server: the last success may be beyond a page of failures.
query["status"] = "success"
endpoint = (
f"{api_url.rstrip('/')}/repos/{quote(owner, safe='')}/{quote(repo, safe='')}"
f"/actions/workflows/{quote(workflow_name, safe='')}/runs?"
+ urlencode({"event": "schedule", "per_page": 1})
+ urlencode(query)
)
request = Request(
endpoint,
@@ -110,57 +113,104 @@ def fetch_latest_scheduled_run(
"X-GitHub-Api-Version": "2022-11-28",
},
)
with urlopen(request, timeout=30) as response:
# Two requests per manifest entry must fit the watchdog's ten-minute job.
with urlopen(request, timeout=15) as response:
payload = json.load(response)
runs = payload.get("workflow_runs")
runs = payload.get("workflow_runs") if isinstance(payload, dict) else None
if not isinstance(runs, list):
raise ValueError(f"GitHub returned no workflow_runs list for {workflow}")
total_count = payload.get("total_count")
if not isinstance(total_count, int) or isinstance(total_count, bool) or total_count < len(runs):
raise ValueError(f"GitHub returned an invalid run count for {workflow}")
if not runs:
if total_count:
raise ValueError(f"GitHub returned an empty first page with recorded runs for {workflow}")
return None
if not isinstance(runs[0], dict):
run = runs[0]
if not isinstance(run, dict):
raise ValueError(f"GitHub returned an invalid workflow run for {workflow}")
return runs[0]
if run.get("event") != "schedule" or run.get("head_branch") != default_branch:
raise ValueError(f"GitHub returned a run outside the scheduled default-branch query for {workflow}")
if not isinstance(run.get("status"), str) or not run["status"]:
raise ValueError(f"GitHub returned no run status for {workflow}")
conclusion = run.get("conclusion")
if (conclusion is not None and not isinstance(conclusion, str)) or (
run["status"] == "completed" and not conclusion
):
raise ValueError(f"GitHub returned an invalid run conclusion for {workflow}")
if successful and (run["status"] != "completed" or conclusion != "success"):
raise ValueError(f"GitHub returned a run without a completed success for {workflow}")
parse_timestamp(run.get("created_at"))
if not isinstance(run.get("html_url"), str) or not run["html_url"]:
raise ValueError(f"GitHub returned no run URL for {workflow}")
return run
def write_report(path: Path, failures: list[tuple[str, int, str, str]]) -> None:
lines = ["## Scheduled validation freshness"]
if not failures:
lines.append("")
lines.append("All critical scheduled validations have a recent scheduled run.")
else:
lines.extend(
[
"",
"The following critical validations are stale or could not be inspected:",
"",
"| Workflow | Limit | Result | Last run |",
"| --- | ---: | --- | --- |",
]
)
for workflow, max_age_hours, reason, run_url in failures:
link = f"[open]({run_url})" if run_url else "—"
lines.append(f"| `{workflow}` | {max_age_hours}h | {reason} | {link} |")
def describe_run(run: dict[str, object] | None) -> str:
if run is None:
return "No recorded run"
outcome = run["status"]
if run.get("conclusion"):
outcome = f"{outcome}/{run['conclusion']}"
return f"[{outcome}]({run['html_url']}) — created {run['created_at']}"
def write_report(path: Path, rows: list[tuple[str, int, str, str, str]], default_branch: str) -> None:
lines = [
"## Scheduled validation freshness",
"",
f"Default branch: `{default_branch}`. Ages use scheduled-run creation time; rerunning an old commit does not refresh its evidence.",
"Attempt outcomes are shown independently of successful-run freshness.",
"Success is the GitHub workflow run conclusion; suite completeness remains the responsibility of each workflow.",
"",
"| Workflow | Limit | Freshness | Last attempt | Last completed success |",
"| --- | ---: | --- | --- | --- |",
]
for workflow, max_age_hours, result, attempt, success in rows:
cells = [f"`{workflow}`", f"{max_age_hours}h", result, attempt, success]
lines.append("| " + " | ".join(cell.replace("|", "\\|").replace("\n", " ") for cell in cells) + " |")
path.write_text("\n".join(lines) + "\n")
def check_freshness(
config: Path, report: Path, repository: str, token: str, api_url: str
config: Path, report: Path, repository: str, token: str, api_url: str, default_branch: str
) -> int:
now = datetime.now(timezone.utc)
failures: list[tuple[str, int, str, str]] = []
rows: list[tuple[str, int, str, str, str]] = []
failed = False
for workflow, max_age_hours, never_ran_grace_until in load_validations(config):
try:
run = fetch_latest_scheduled_run(repository, workflow, token, api_url)
reason = stale_reason(run, now, max_age_hours, never_ran_grace_until)
if reason is not None:
run_url = str(run.get("html_url", "")) if run else ""
failures.append((workflow, max_age_hours, reason, run_url))
except Exception as error:
failures.append(
(workflow, max_age_hours, f"inspection failed: {error}", "")
)
write_report(report, failures)
return 1 if failures else 0
runs: dict[str, dict[str, object] | None] = {}
reasons: list[str] = []
for label, successful in (("Last attempt", False), ("Last completed success", True)):
try:
runs[label] = fetch_latest_scheduled_run(
repository, workflow, token, api_url, default_branch, successful
)
except Exception as error:
reasons.append(f"{label}: inspection failed: {error}")
# A failed inspection or any recorded attempt ends first-run grace.
initial_grace = (
len(runs) == 2
and all(run is None for run in runs.values())
and never_ran_grace_until is not None
and now <= never_ran_grace_until
)
if not initial_grace:
for label, run in runs.items():
reason = stale_reason(run, now, max_age_hours)
if reason is not None:
reasons.append(f"{label}: {reason}")
failed |= bool(reasons)
result = "; ".join(reasons) if reasons else "Fresh"
if initial_grace:
result = f"Initial grace until {never_ran_grace_until.isoformat()}"
evidence = [
describe_run(runs[label]) if label in runs else "Inspection failed"
for label in ("Last attempt", "Last completed success")
]
rows.append((workflow, max_age_hours, result, *evidence))
write_report(report, rows, default_branch)
return 1 if failed else 0
class SelfTests(unittest.TestCase):
@@ -173,15 +223,6 @@ class SelfTests(unittest.TestCase):
self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36))
self.assertIsNotNone(stale_reason(None, self.NOW, 36))
def test_never_ran_grace_only_covers_missing_runs(self) -> None:
future_grace = self.NOW + timedelta(hours=1)
past_grace = self.NOW - timedelta(seconds=1)
self.assertIsNone(stale_reason(None, self.NOW, 36, future_grace))
self.assertIsNone(stale_reason(None, self.NOW, 36, self.NOW))
self.assertIsNotNone(stale_reason(None, self.NOW, 36, past_grace))
stale_run = {"created_at": "2026-08-20T23:59:59Z"}
self.assertIsNotNone(stale_reason(stale_run, self.NOW, 36, future_grace))
def test_config_rejects_duplicate_and_invalid_entries(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "validations.json"
@@ -238,63 +279,205 @@ class SelfTests(unittest.TestCase):
],
)
def test_check_reports_missing_runs(self) -> None:
@staticmethod
def run_fixture(**overrides: object) -> dict[str, object]:
return {
"status": "completed",
"conclusion": "success",
"event": "schedule",
"head_branch": "release/current",
"created_at": "2026-08-22T00:00:00Z",
"html_url": "https://github.test/rustfs/rustfs/actions/runs/1",
**overrides,
}
def check_payloads(
self, payloads: list[object], *, grace: str | None = None, workflows: int = 1
) -> tuple[int, str, list]:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
config = root / "validations.json"
report = root / "report.md"
config.write_text(
json.dumps(
[
{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36},
{"workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36},
{"workflow": ".github/workflows/mint.yml", "max_age_hours": 36},
]
)
)
with mock.patch(
__name__ + ".fetch_latest_scheduled_run",
side_effect=[
{"created_at": "2999-01-01T00:00:00Z"},
None,
RuntimeError("API unavailable"),
],
entries = [
{"workflow": f".github/workflows/check-{index}.yml", "max_age_hours": 36}
for index in range(workflows)
]
if grace is not None:
entries[0]["never_ran_grace_until"] = grace
config.write_text(json.dumps(entries))
responses = []
for payload in payloads:
if isinstance(payload, dict) and isinstance(payload.get("workflow_runs"), list):
payload = {"total_count": len(payload["workflow_runs"]), **payload}
responses.append(payload if isinstance(payload, Exception) else io.StringIO(json.dumps(payload)))
with (
mock.patch(__name__ + ".urlopen", side_effect=responses) as request,
mock.patch(__name__ + ".datetime", wraps=datetime) as clock,
):
self.assertEqual(
check_freshness(
config,
report,
"rustfs/rustfs",
"token",
"https://api.github.test",
),
1,
clock.now.return_value = self.NOW
status = check_freshness(
config, report, "rustfs/rustfs", "test-token",
"https://api.github.test", "release/current",
)
contents = report.read_text()
self.assertIn(".github/workflows/fuzz.yml", contents)
self.assertIn("inspection failed: API unavailable", contents)
self.assertNotIn(".github/workflows/ci.yml`", contents)
return status, report.read_text(), request.call_args_list
config.write_text(
json.dumps(
[{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}]
def test_requests_filter_schedule_default_branch_and_success_on_server(self) -> None:
attempt = self.run_fixture(status="in_progress", conclusion=None)
success = self.run_fixture(html_url="https://github.test/rustfs/rustfs/actions/runs/2")
status, report, calls = self.check_payloads([
{"workflow_runs": [attempt], "total_count": 1001},
{"workflow_runs": [success], "total_count": 1},
])
self.assertEqual(status, 0)
self.assertEqual(len(calls), 2)
for call, successful in zip(calls, (False, True)):
request = call.args[0]
url = urlsplit(request.full_url)
self.assertEqual(url.path, "/repos/rustfs/rustfs/actions/workflows/check-0.yml/runs")
expected = {"event": ["schedule"], "branch": ["release/current"], "per_page": ["1"]}
if successful:
expected["status"] = ["success"]
self.assertEqual(parse_qs(url.query), expected)
self.assertEqual(request.get_header("Authorization"), "Bearer test-token")
self.assertEqual(call.kwargs, {"timeout": 15})
self.assertIn("[in_progress]", report)
self.assertIn(str(attempt["html_url"]), report)
self.assertIn(str(success["html_url"]), report)
def test_cancelled_attempt_cannot_refresh_expired_success(self) -> None:
attempt = self.run_fixture(conclusion="cancelled")
success = self.run_fixture(
created_at="2026-08-20T23:59:59Z", updated_at="2026-08-22T11:59:59Z",
html_url="https://github.test/rustfs/rustfs/actions/runs/2",
)
status, report, _ = self.check_payloads([
{"workflow_runs": [attempt]}, {"workflow_runs": [success]},
])
self.assertEqual(status, 1)
self.assertIn("Last completed success: last scheduled run is", report)
self.assertIn("[completed/cancelled]", report)
for run in (attempt, success):
self.assertIn(str(run["html_url"]), report)
self.assertIn(str(run["created_at"]), report)
def test_attempt_outcome_does_not_replace_recent_success(self) -> None:
success = self.run_fixture(created_at="2026-08-21T00:00:00Z")
for state, conclusion in (
("completed", "failure"), ("completed", "cancelled"),
("completed", "timed_out"), ("completed", "success"),
("queued", None), ("in_progress", None),
):
with self.subTest(state=state, conclusion=conclusion):
status, report, _ = self.check_payloads([
{"workflow_runs": [self.run_fixture(status=state, conclusion=conclusion)]},
{"workflow_runs": [success]},
])
self.assertEqual(status, 0)
self.assertIn(f"[{state}" + (f"/{conclusion}" if conclusion else "") + "]", report)
self.assertIn("Fresh", report)
self.assertNotIn("All critical scheduled validations", report)
def test_grace_requires_two_successful_queries_with_no_history(self) -> None:
for attempt, success, grace, expected in (
(None, None, "2026-08-22T12:00:00Z", 0),
(None, None, "2026-08-22T11:59:59Z", 1),
(self.run_fixture(conclusion="failure"), None, "2026-08-23T00:00:00Z", 1),
(self.run_fixture(status="queued", conclusion=None), None, "2026-08-23T00:00:00Z", 1),
(None, self.run_fixture(), "2026-08-23T00:00:00Z", 1),
):
with self.subTest(attempt=attempt, success=success, grace=grace):
status, report, _ = self.check_payloads([
{"workflow_runs": [] if attempt is None else [attempt]},
{"workflow_runs": [] if success is None else [success]},
], grace=grace)
self.assertEqual(status, expected)
self.assertEqual("Initial grace until" in report, expected == 0)
def test_api_failures_preserve_other_evidence_and_never_enter_grace(self) -> None:
good = {"workflow_runs": [self.run_fixture()]}
for first, second in (
(RuntimeError("API unavailable"), good),
(good, RuntimeError("API unavailable")),
(RuntimeError("API unavailable"), {"workflow_runs": []}),
):
with self.subTest(first=first, second=second):
status, report, calls = self.check_payloads(
[first, second], grace="2026-08-23T00:00:00Z"
)
)
with mock.patch(
__name__ + ".fetch_latest_scheduled_run",
return_value={"created_at": "2999-01-01T00:00:00Z"},
):
self.assertEqual(
check_freshness(
config,
report,
"rustfs/rustfs",
"token",
"https://api.github.test",
),
0,
)
self.assertIn("All critical scheduled validations", report.read_text())
self.assertEqual(status, 1)
self.assertEqual(len(calls), 2)
self.assertIn("inspection failed: API unavailable", report)
self.assertNotIn("Initial grace until", report)
if first is good or second is good:
self.assertIn(str(self.run_fixture()["html_url"]), report)
def test_invalid_api_evidence_fails_closed(self) -> None:
malformed = [
[], {}, {"workflow_runs": {}}, {"workflow_runs": [None]},
{"workflow_runs": [], "total_count": 1},
{"workflow_runs": [], "total_count": -1},
{"workflow_runs": [], "total_count": None},
{"workflow_runs": [], "total_count": True},
*({"workflow_runs": [self.run_fixture(**override)]} for override in (
{"event": "workflow_dispatch"}, {"head_branch": "other"},
{"created_at": "invalid"}, {"created_at": "2026-08-22T00:00:00"},
{"status": None}, {"conclusion": None}, {"conclusion": 1},
{"html_url": ""},
)),
]
for payload in malformed:
for index, label in enumerate(("Last attempt", "Last completed success")):
with self.subTest(payload=payload, label=label):
payloads = [{"workflow_runs": [self.run_fixture()]} for _ in range(2)]
payloads[index] = payload
status, report, _ = self.check_payloads(payloads, grace="2026-08-23T00:00:00Z")
self.assertEqual(status, 1)
self.assertIn(f"{label}: inspection failed", report)
self.assertNotIn("Initial grace until", report)
self.assertIn(str(self.run_fixture()["html_url"]), report)
for state, conclusion in (("in_progress", "success"), ("completed", "failure"), ("completed", "skipped")):
with self.subTest(state=state, conclusion=conclusion):
status, report, _ = self.check_payloads([
{"workflow_runs": [self.run_fixture()]},
{"workflow_runs": [self.run_fixture(status=state, conclusion=conclusion)]},
])
self.assertEqual(status, 1)
self.assertIn("without a completed success", report)
def test_report_retains_every_workflow(self) -> None:
status, report, calls = self.check_payloads([
{"workflow_runs": [self.run_fixture()]}, {"workflow_runs": [self.run_fixture()]},
{"workflow_runs": []}, {"workflow_runs": []},
RuntimeError("API unavailable"), {"workflow_runs": [self.run_fixture()]},
], workflows=3)
self.assertEqual(status, 1)
self.assertEqual(len(calls), 6)
for index in range(3):
self.assertEqual(report.count(f"`.github/workflows/check-{index}.yml`"), 1)
self.assertIn("No recorded run", report)
self.assertIn("Inspection failed", report)
def test_cli_requires_the_repository_default_branch(self) -> None:
from check_test_wiring import yaml_block
workflow = (ROOT / ".github/workflows/scheduled-validation-freshness.yml").read_text().splitlines()
job = yaml_block(workflow, "check-freshness", 2)
self.assertIsNotNone(job)
start = job.index(" - name: Check latest scheduled runs")
end = next((index for index in range(start + 1, len(job)) if job[index].startswith(" - ")), len(job))
environment = yaml_block(job[start:end], "env", 8)
self.assertIsNotNone(environment)
self.assertIn(" RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}", environment)
with (
mock.patch.dict(os.environ, {"GITHUB_REPOSITORY": "rustfs/rustfs", "GH_TOKEN": "test-token"}, clear=True),
mock.patch.object(sys, "argv", ["checker", "--report", "unused.md"]),
mock.patch("sys.stderr", new=io.StringIO()) as stderr,
self.assertRaises(SystemExit) as error,
):
main()
self.assertEqual(error.exception.code, 2)
self.assertIn("RUSTFS_DEFAULT_BRANCH", stderr.getvalue())
def main() -> int:
@@ -318,11 +501,14 @@ def main() -> int:
repository = os.environ.get("GITHUB_REPOSITORY", "")
token = os.environ.get("GH_TOKEN", "")
api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com")
default_branch = os.environ.get("RUSTFS_DEFAULT_BRANCH", "")
if not re.fullmatch(r"[^/\s]+/[^/\s]+", repository):
parser.error("GITHUB_REPOSITORY must be owner/repository")
if not token:
parser.error("GH_TOKEN is required")
return check_freshness(args.config, args.report, repository, token, api_url)
if not default_branch or any(character.isspace() for character in default_branch):
parser.error("RUSTFS_DEFAULT_BRANCH is required and must name the repository default branch")
return check_freshness(args.config, args.report, repository, token, api_url, default_branch)
if __name__ == "__main__":
scripts/test_security_workflow.py
+197
View File
@@ -0,0 +1,197 @@
#!/usr/bin/env python3
"""Run the security workflow's evidence and result steps without remote VMs."""
from __future__ import annotations
import os
import re
import subprocess
import tempfile
import unittest
from pathlib import Path
from check_test_wiring import yaml_block
ROOT = Path(__file__).resolve().parents[1]
WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml"
CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |"
class SecurityWorkflowTests(unittest.TestCase):
def setUp(self) -> None:
self.source = WORKFLOW.read_text()
self.job = yaml_block(self.source.splitlines(), "security-test", 2)
self.assertIsNotNone(self.job)
starts = [i for i, line in enumerate(self.job) if line.startswith(" - name: ")]
self.steps = {
self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end]
for start, end in zip(starts, starts[1:] + [len(self.job)])
}
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.directory = Path(self.temp.name)
self.context = {
"runner.temp": self.temp.name,
"github.server_url": "https://github.com",
"github.repository": "rustfs/rustfs",
"github.run_id": "314159",
"github.run_attempt": "2",
"github.sha": "0123456789abcdef0123456789abcdef01234567",
"github.event_name": "workflow_dispatch",
"github.workspace": self.temp.name,
"inputs.package_url": "",
"inputs.rustfs_version": "test-version",
"inputs.topology": "all",
"inputs.oidc_live": "false",
"steps.evidence.outcome": "skipped",
"steps.test.outcome": "skipped",
"steps.report.outcome": "skipped",
}
self.env = {
**os.environ, "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"),
"GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name, "TMPDIR": self.temp.name,
}
for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
self.context["env.SECURITY_ARTIFACTS_DIR"] = ""
self.artifacts = self.directory / "rustfs-security-314159-2"
suite = self.directory / "auto-testing/rustfs-security-test.sh"
suite.parent.mkdir()
suite.write_text(
'#!/usr/bin/env bash\nset -euo pipefail\n'
'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n'
'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n'
'case "$FAKE_REPORT" in\n'
f' present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n'
' empty) : > "$REPORT_FILE" ;;\n'
'esac\n'
'echo "UNWRAPPED SUITE SUMMARY" >> "$GITHUB_STEP_SUMMARY"\n'
'exit "$FAKE_EXIT"\n'
)
def render(self, value: str) -> str:
return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
result = {}
for line in yaml_block(lines, "env", indent) or []:
if line.strip() and not line.lstrip().startswith("#"):
key, value = line.strip().split(": ", 1)
result[key] = self.render(value.strip("'\""))
return result
def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
lines = self.steps[name]
start = lines.index(" run: |") + 1
shell_lines = []
for line in lines[start:]:
if line.strip() and not line.startswith(" "):
break
shell_lines.append(line[10:])
self.assertTrue(shell_lines, f"missing literal shell body: {name}")
result = subprocess.run(
["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render("\n".join(shell_lines))],
cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
)
for line in lines:
if line.startswith(" id: "):
self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
if Path(self.env["GITHUB_ENV"]).exists():
for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
key, value = line.split("=", 1)
self.env[key] = value
self.context[f"env.{key}"] = value
return result
def test_workflow_wiring(self) -> None:
names = list(self.steps)
self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)"))
self.assertNotIn(" continue-on-error: true", self.job)
self.assertIn(" continue-on-error: true", self.steps["Run security suite"])
for name in ("Initialize security evidence", "Generate report"):
self.assertNotIn(" continue-on-error: true", self.steps[name])
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps["Generate report"])
self.assertNotIn("/tmp/rustfs-security", self.source)
for name in ("Upload functional report to dashboard", "File failure issue in rustfs/backlog"):
report = next(line for line in self.steps[name] if line.strip().startswith("REPORT_FILE:"))
self.assertIn("${{ env.SECURITY_ARTIFACTS_DIR }}/report.md", report)
for name in ("Upload functional report to dashboard", "Upload report and logs"):
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name])
artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8)
self.assertIn(" path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings)
self.assertIn(" if-no-files-found: error", artifact_settings)
def test_suite_report_and_result_matrix(self) -> None:
for outcome, mode, exit_code in (
("success", "present", 0), ("failure", "present", 7), ("failure", "missing", 7),
("success", "missing", 0), ("success", "empty", 0),
("skipped", "missing", 0), ("skipped", "present", 0),
("cancelled", "missing", 0), ("cancelled", "present", 0),
):
with self.subTest(outcome=outcome, report=mode):
self.setUp()
initialized = self.run_step("Initialize security evidence")
self.assertEqual(initialized.returncode, 0, initialized.stderr)
self.assertEqual(self.env["SECURITY_ARTIFACTS_DIR"], str(self.artifacts))
self.env.update(FAKE_REPORT=mode, FAKE_EXIT=str(exit_code))
if outcome != "skipped" or mode == "present":
suite = self.run_step("Run security suite")
self.assertEqual(suite.returncode, exit_code, suite.stderr)
logs = list(self.artifacts.glob("rustfs-security.*/suite.log"))
self.assertEqual(len(logs), 1)
self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n")
self.context["steps.test.outcome"] = outcome
report = self.run_step("Generate report")
success = outcome == "success" and mode == "present"
self.assertEqual(report.returncode == 0, success, report.stderr)
contents = (self.artifacts / "report.md").read_text()
for expected in (
"https://github.com/rustfs/rustfs/actions/runs/314159", "Attempt: 2",
f"Workflow Commit: {self.context['github.sha']}", "Trigger: workflow_dispatch",
f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}",
):
self.assertIn(expected, contents)
self.assertEqual(CASE_ROW in contents, success)
self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, success)
if mode == "present":
raw = (self.artifacts / "suite-report.md").read_text()
self.assertEqual(raw, f"CURRENT SUITE DIAGNOSTIC\n{CASE_ROW}\n")
summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text()
self.assertEqual(summary, contents)
self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary)
def test_existing_evidence_directory_is_rejected(self) -> None:
self.artifacts.mkdir()
stale = self.artifacts / "suite-report.md"
stale.write_text("OLD RUN REPORT")
self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
self.assertEqual(stale.read_text(), "OLD RUN REPORT")
self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
(self.artifacts / "report.md").write_text("OLD RUN REPORT")
self.context.update({
"env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
})
fake_bin = self.directory / "bin"
fake_bin.mkdir()
gh = fake_bin / "gh"
gh.write_text(
'#!/usr/bin/env bash\nset -euo pipefail\n'
'if [ "$1 $2" = "issue create" ]; then\n'
' while [ "$#" -gt 0 ]; do\n'
' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
' shift\n'
' done\n'
'fi\n'
)
gh.chmod(0o755)
body = self.directory / "issue-body.md"
self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
result = self.run_step("File failure issue in rustfs/backlog")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertNotIn("OLD RUN REPORT", body.read_text())
self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
if __name__ == "__main__":
unittest.main()