gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-05 11:45:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into codex/test-lifecycle-adversarial

Browse Source
This commit is contained in:
Zhengchao An
2026-09-05 18:55:09 +08:00
committed by GitHub
parent 8943f38390 e8a7f4bc4a
commit 0f0d2d16fa
115 changed files with 12501 additions and 2800 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/e2e-full-selection.txt
+1 -1
View File
@@ -1,2 +1,2 @@
sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535
sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
.config/ecstore-required-tests.json
+72
View File
@@ -0,0 +1,72 @@
{
"lane": "ci/test-and-lint",
"tests": [
{
"invariant": "write-quorum",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::inline_put_commit_path_tests::inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one"
},
{
"invariant": "metadata-rollback",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::write_unique_file_info_reverts_metadata_when_write_quorum_fails"
},
{
"invariant": "stale-writer",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::put_object_no_lock_aborts_after_outer_namespace_lock_loss"
},
{
"invariant": "range-body",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::transition_upload_integrity_tests::transitioned_compressed_object_range_get_returns_plaintext_slice"
},
{
"invariant": "multipart-cancellation",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::multipart::tests::cancelled_complete_keeps_upload_lock_through_tail_cleanup"
},
{
"invariant": "list-uncommitted-version",
"suite": "rustfs-filemeta",
"name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge"
},
{
"invariant": "minio-object-fixture",
"suite": "rustfs-filemeta",
"name": "filemeta::test::parses_real_minio_object_xlmeta"
},
{
"invariant": "corrupt-part-arrays",
"suite": "rustfs-filemeta",
"name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
}
],
"fixtures": [
{
"path": "crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex",
"sha256": "e8093767806d701e639b48d023190e858fbc4cde69bcfd83c22af8cba8452ce5",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex",
"sha256": "2a415ad3a3be5a9440035d4026ff880e0e8c1ec1701be9f4e077734e8dce03da",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex",
"sha256": "7f21f50c326dd8b0228deb6dbdb7052b3d0a3f8ee6c85d43486f0e6bb7a97261",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex",
"sha256": "f2b6e260aff106adf6039feb1c645686e84e75404ff725491fb18668be5db203",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex",
"sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
}
]
}
.config/make/tests.mak
+1
View File
@@ -40,6 +40,7 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
.github/workflows/ci-docs-only.yml
+1
View File
@@ -129,6 +129,7 @@ jobs:
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
.github/workflows/ci.yml
+8
View File
@@ -167,6 +167,7 @@ jobs:
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
@@ -269,6 +270,7 @@ jobs:
CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
run: |
mkdir -p artifacts/test-and-lint
rm -f target/nextest/ci/junit.xml
./scripts/ci/resource_sampler.sh start nextest
trap './scripts/ci/resource_sampler.sh stop' EXIT
set +e
@@ -277,6 +279,12 @@ jobs:
--status-level all --final-status-level all \
2>&1 | tee artifacts/test-and-lint/nextest.log
status=${PIPESTATUS[0]}
if [[ "${status}" -eq 0 ]]; then
cargo nextest list --profile ci --all --exclude e2e_test --message-format json \
> artifacts/test-and-lint/core-test-listing.json \
&& python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \
&& test -s target/nextest/ci/junit.xml || status=$?
fi
{
echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
echo "exit_status=${status}"
.github/workflows/rustfs-security-test.yml
+55 -28
View File
@@ -74,10 +74,23 @@ env:
jobs:
security-test:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Initialize security evidence
id: evidence
run: |
set -euo pipefail
umask 077
SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${SECURITY_ARTIFACTS_DIR}"
printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry)
@@ -98,11 +111,6 @@ jobs:
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
exit 1
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Show environment
run: |
uname -a
@@ -135,7 +143,8 @@ jobs:
id: test
continue-on-error: true
env:
REPORT_FILE: /tmp/rustfs-security-report.md
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
run: |
set -euo pipefail
@@ -159,29 +168,48 @@ jobs:
else
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
fi
./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
- name: Generate report
if: always()
id: report
if: ${{ always() && steps.evidence.outcome == 'success' }}
env:
TEST_OUTCOME: ${{ steps.test.outcome }}
run: |
set -euo pipefail
if [ ! -f /tmp/rustfs-security-report.md ]; then
{
echo "# RustFS security test report"
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Test Step Outcome: failure (suite did not produce a report)"
} > /tmp/rustfs-security-report.md
RESULT=failure
if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
RESULT=success
fi
cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}"
{
echo "# RustFS security test report"
echo ""
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${TEST_OUTCOME}"
echo ""
# The dashboard prioritizes case rows over the step outcome.
# Keep partial case results in the artifact when the suite fails.
if [ "${RESULT}" = "success" ]; then
cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics."
else
echo "The suite did not produce a non-empty report."
fi
} > "${SECURITY_ARTIFACTS_DIR}/report.md"
cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true
env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: /tmp/rustfs-security-report.md
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
SUITE: security
run: |
set -euo pipefail
@@ -210,8 +238,9 @@ jobs:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
SUITE: 'security'
SUITE_LABEL: 'Security'
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-security-report.md'
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
LOG_FILE: ''
run: |
set -euo pipefail
@@ -245,7 +274,7 @@ jobs:
echo ""
echo "## Report (errors and symptoms)"
echo ""
if [ -s "${REPORT_FILE}" ]; then
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)"
@@ -263,14 +292,12 @@ jobs:
echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-security-test-${{ github.run_id }}
path: |
/tmp/rustfs-security-report.md
/tmp/rustfs-security.*/*
if-no-files-found: ignore
name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
if-no-files-found: error
retention-days: 3
- name: Cleanup environment (after)
.github/workflows/rustfs-upgrade-test.yml
+26 -2
View File
@@ -18,7 +18,7 @@ on:
workflow_dispatch:
inputs:
from_version:
description: 'OLD RustFS release tag (must ship a .deb asset, e.g. 1.0.0-rc.3)'
description: 'OLD RustFS release tag, e.g. 1.0.0-rc.3 (its release must ship a .deb asset). Leave empty for the default.'
required: false
default: '1.0.0-rc.3'
from_url:
@@ -26,7 +26,7 @@ on:
required: false
type: string
to_version:
description: 'NEW RustFS release tag (leave empty for latest nightly)'
description: 'NEW RustFS release tag, e.g. 1.0.0-rc.5 (any version with a .deb asset). Leave empty for latest nightly.'
required: false
to_url:
description: 'NEW .deb URL. Overrides to_version / nightly default.'
@@ -145,6 +145,7 @@ jobs:
continue-on-error: true
env:
LOG_FILE: /tmp/rustfs-upgrade.log
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
run: |
set -euo pipefail
chmod +x auto-testing/rustfs-upgrade-test.sh
@@ -175,6 +176,29 @@ jobs:
else
ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
fi
# Fail fast with a clear message when a requested release tag has
# no .deb asset (e.g. 1.0.0-rc.4 ships only zips), instead of
# letting the suite die mid-run on a 404.
check_release_asset() {
local version="$1" tag asset url
[ -n "${version}" ] && [ "${version}" != "null" ] || return 0
tag="${version#v}"
asset="rustfs_${tag//-/.}_amd64.deb"
url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}"
if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then
echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2
echo " ${url}" >&2
echo "Pick a tag whose release ships a .deb (check its release assets)." >&2
exit 1
fi
echo "resolved ${tag} -> ${url}"
}
if [ -z "${FROM_URL}" ]; then
check_release_asset "${FROM_VERSION}"
fi
if [ -z "${TO_URL}" ]; then
check_release_asset "${TO_VERSION}"
fi
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
- name: Generate report
.github/workflows/scheduled-validation-freshness.yml
+1
View File
@@ -42,6 +42,7 @@ jobs:
- name: Check latest scheduled runs
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: |
set +e
python3 scripts/check_scheduled_validation_freshness.py \
.gitignore
+1
View File
@@ -33,6 +33,7 @@ profile.json
*.zst
.secrets
*.go
!crates/zip/tests/fixtures/snowball/**/generate/*.go
*.pb
*.svg
deploy/logs/*.log.*
Cargo.lock
Generated
+324 -112
View File
File diff suppressed because it is too large Load Diff
Cargo.toml
+10 -6
View File
@@ -168,7 +168,7 @@ reqwest = "0.13.4"
rustfs-kafka-async = { version = "1.3.1" }
socket2 = { version = "0.6.5" }
tokio = { version = "1.53.1" }
tokio-rustls = { default-features = false, version = "0.26.4" }
tokio-rustls = { default-features = false, version = "0.26.5" }
tokio-stream = { version = "0.1.19" }
tokio-test = "0.4.5"
tokio-util = { version = "0.7.19" }
@@ -234,15 +234,19 @@ tokio-postgres-rustls = "0.14.0"
# Utilities and Tools
anyhow = "1.0.104"
arc-swap = "1.9.2"
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams.
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork.
astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
tar-codec = "0.0.14"
tar-framing = "0.0.14"
atoi = "3.1.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.11.0" }
aws-config = { version = "1.12.0" }
aws-credential-types = { version = "1.3.0" }
aws-sdk-kms = { default-features = false, version = "1.117.0" }
aws-sdk-s3 = { default-features = false, version = "1.144.0" }
aws-sdk-sts = { default-features = false, version = "1.113.0" }
aws-sdk-kms = { default-features = false, version = "1.118.0" }
aws-sdk-s3 = { default-features = false, version = "1.145.0" }
aws-sdk-sts = { default-features = false, version = "1.114.0" }
aws-smithy-async = { version = "1.3.0" }
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
aws-smithy-runtime-api = { version = "1.16.0" }
aws-smithy-types = { version = "1.6.3" }
crates/e2e_test/src/replication_extension_test.rs
+93
View File
@@ -6743,6 +6743,99 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua
Ok(())
}
#[tokio::test]
async fn test_site_replication_replays_bucket_created_during_peer_outage_real_dual_node() -> TestResult {
init_logging();
// Keep compilation outside the scenario timeout. Recovery itself waits
// for the production 30-second lightweight retry tick.
let _rustfs_binary = rustfs_binary_path();
match timeout(Duration::from_secs(150), async {
let mut site_env = replication_fast_env();
site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
let mut site_a_env = RustFSTestEnvironment::new().await?;
site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
let mut site_b_env = RustFSTestEnvironment::new().await?;
site_b_env.start_rustfs_server_without_cleanup_with_env(&site_env).await?;
let site_a_client = site_a_env.create_s3_client();
let site_b_client = site_b_env.create_s3_client();
let bucket = "site-repl-peer-outage";
let key = "after-recovery.txt";
let payload = b"site replication recovered the missed bucket".to_vec();
let add_status = site_replication_add(
&site_a_env,
&[
PeerSite {
name: "outage-site-a".to_string(),
endpoint: site_a_env.url.clone(),
access_key: site_a_env.access_key.clone(),
secret_key: site_a_env.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "outage-site-b".to_string(),
endpoint: site_b_env.url.clone(),
access_key: site_b_env.access_key.clone(),
secret_key: site_b_env.secret_key.clone(),
..Default::default()
},
],
)
.await?;
assert!(add_status.success, "unexpected site add result: {add_status:?}");
wait_for_site_replication_enabled(&site_a_env, 2).await?;
wait_for_site_replication_enabled(&site_b_env, 2).await?;
site_b_env.stop_server();
site_a_client.create_bucket().bucket(bucket).send().await?;
site_a_client.head_bucket().bucket(bucket).send().await?;
let queued = site_replication_info(&site_a_env)
.await?
.retry_stats
.ok_or("peer outage did not persist a site replication retry event")?;
assert!(queued.pending + queued.failed > 0, "peer outage retry queue was unexpectedly empty");
site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
let recovery_deadline = tokio::time::Instant::now() + Duration::from_secs(75);
loop {
let bucket_recovered = site_b_client.head_bucket().bucket(bucket).send().await.is_ok();
let queue_empty = site_replication_info(&site_a_env).await?.retry_stats.is_none();
if bucket_recovered && queue_empty {
break;
}
if tokio::time::Instant::now() >= recovery_deadline {
return Err(format!(
"site replication retry did not settle after peer recovery; bucket_recovered={bucket_recovered}, queue_empty={queue_empty}"
)
.into());
}
sleep(Duration::from_millis(250)).await;
}
site_a_client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from(payload.clone()))
.send()
.await?;
assert_eq!(wait_for_object_on_target(&site_b_client, bucket, key).await?, payload);
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err("site replication peer-outage recovery timed out after 150 seconds".into()),
}
}
/// Re-applying a site's own replication config must not disable the peer's reverse direction.
///
/// `PutBucketReplication` broadcasts the config to every peer — the console's replication
crates/ecstore/Cargo.toml
+1
View File
@@ -244,6 +244,7 @@ windows-sys = { workspace = true, features = [
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
[dev-dependencies]
aws-smithy-async.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
criterion = { workspace = true, features = ["html_reports"] }
temp-env = { workspace = true, features = ["async_closure"] }
crates/ecstore/src/api/mod.rs
+10 -9
View File
@@ -196,15 +196,16 @@ pub mod bucket {
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
pub use crate::bucket::metadata_sys::{
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config,
get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config,
get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata,
remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock,
acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
update_under_transaction_lock,
};
}
crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs
+5
View File
@@ -1170,6 +1170,7 @@ pub async fn save_manual_transition_job_record_if_current(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()),
..Default::default()
@@ -1242,6 +1243,7 @@ pub(crate) async fn save_manual_transition_worker_result_if_absent(
data,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -1270,6 +1272,7 @@ pub(crate) async fn save_manual_transition_task_if_absent(
data,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -1621,6 +1624,7 @@ pub async fn save_manual_transition_scope_admission_if_absent(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -1672,6 +1676,7 @@ pub async fn save_manual_transition_scope_admission_if_current(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()),
..Default::default()
crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs
+6
View File
@@ -1733,6 +1733,7 @@ async fn save_config_if_none_fenced(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -1832,6 +1833,7 @@ async fn save_decommission_manifest_checkpoint_if_match(
let mut opts = ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
no_lock: true,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(observed_etag),
@@ -1960,6 +1962,7 @@ async fn save_config_if_match_fenced(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag.to_string()),
..Default::default()
@@ -3780,6 +3783,7 @@ where
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -3869,6 +3873,7 @@ where
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag),
..Default::default()
@@ -3893,6 +3898,7 @@ where
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs
+15 -15
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use super::runtime_boundary as runtime_sources;
use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
@@ -72,9 +70,11 @@ static REMOTE_DELETE_BREAKER: LazyLock<Mutex<RemoteDeleteBreaker>> = LazyLock::n
});
#[cfg(test)]
static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<
std::sync::Mutex<Option<Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>>>,
> = std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
type RemoteTierDeleteTestHook = Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>;
#[cfg(test)]
static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<std::sync::Mutex<Option<RemoteTierDeleteTestHook>>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
#[derive(Debug)]
struct RemoteDeleteBreaker {
@@ -107,7 +107,7 @@ impl RemoteDeleteBreaker {
fn prune(&mut self, now: Instant) {
while let Some(ts) = self.failures.front().copied() {
if now.duration_since(ts) > self.window {
self.failures.pop_front();
let _ = self.failures.pop_front();
} else {
break;
}
@@ -137,10 +137,10 @@ fn is_signer_header_error(err: &std::io::Error) -> bool {
return false;
}
if let Some(source) = err.get_ref() {
if error_chain_contains_signer_header_marker(source) {
return true;
}
if let Some(source) = err.get_ref()
&& error_chain_contains_signer_header_marker(source)
{
return true;
}
let message = err.to_string().to_ascii_lowercase();
@@ -205,7 +205,7 @@ impl ObjSweeper {
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
self.version_id = vid.clone();
self.version_id = vid;
self
}
@@ -219,7 +219,7 @@ impl ObjSweeper {
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
let mut opts = ObjectOpts {
version_id: self.version_id.clone(),
version_id: self.version_id,
versioned: self.versioned,
version_suspended: self.suspended,
..Default::default()
@@ -388,8 +388,8 @@ impl Jentry {
impl ExpiryOp for Jentry {
fn op_hash(&self) -> u64 {
let mut hasher = Sha256::new();
hasher.update(format!("{}", self.tier_name).as_bytes());
hasher.update(format!("{}", self.obj_name).as_bytes());
hasher.update(self.tier_name.as_bytes());
hasher.update(self.obj_name.as_bytes());
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
}
@@ -436,7 +436,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
tier_name: &str,
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
) -> Result<(), std::io::Error> {
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
let lease = TierConfigMgr::acquire_operation_lease(tier_config_mgr, tier_name)
.await
.map_err(std::io::Error::other)?;
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await
crates/ecstore/src/bucket/lifecycle/transition_transaction.rs
+2
View File
@@ -612,6 +612,7 @@ pub(crate) async fn save_transition_transaction_record(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -658,6 +659,7 @@ pub(crate) async fn save_transition_transaction_record_if_current(
data.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag),
..Default::default()
crates/ecstore/src/bucket/metadata_sys.rs
+114 -7
View File
@@ -655,6 +655,12 @@ pub struct BucketMetadataMutationGuard {
}
impl BucketMetadataMutationGuard {
/// Returns the storage-verified identity while both incarnation fences remain valid.
pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
self.ensure_valid(&self.bucket)?;
Ok((&self.bucket, self.incarnation_id))
}
fn ensure_valid(&self, bucket: &str) -> Result<()> {
if self.bucket != bucket {
return Err(Error::other("bucket metadata mutation guard does not match bucket"));
@@ -674,6 +680,29 @@ async fn acquire_config_write_guard_for_incarnation(
sys: Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
expected_incarnation_id: Option<Uuid>,
) -> Result<BucketMetadataMutationGuard> {
acquire_config_write_guard_with_migration(sys, bucket, expected_incarnation_id, true).await
}
/// Scanner probes must not create an incarnation to make a capability available.
pub async fn acquire_scanner_bucket_incarnation_fence(
bucket: &str,
expected_incarnation_id: Uuid,
expected_owner_id: Uuid,
) -> Result<BucketMetadataMutationGuard> {
super::utils::check_valid_bucket_name(bucket)?;
let sys = get_bucket_metadata_sys()?;
if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() {
return Err(Error::other("scanner bucket incarnation owner does not match"));
}
acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
}
async fn acquire_config_write_guard_with_migration(
sys: Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
expected_incarnation_id: Option<Uuid>,
migrate: bool,
) -> Result<BucketMetadataMutationGuard> {
let metadata_sys = sys.read().await.clone();
let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?;
@@ -681,13 +710,15 @@ async fn acquire_config_write_guard_for_incarnation(
// Legacy buckets are migrated while the lifecycle fence prevents a
// same-name replacement. The second read under the write transaction is
// the CAS source of truth for the actual rewrite.
await_bucket_namespace_operation(
Some(&lifecycle_guard),
bucket,
"bucket config incarnation migration",
metadata_sys.get_bucket_incarnation_id(bucket),
)
.await?;
if migrate {
await_bucket_namespace_operation(
Some(&lifecycle_guard),
bucket,
"bucket config incarnation migration",
metadata_sys.get_bucket_incarnation_id(bucket),
)
.await?;
}
let transaction_guard = await_bucket_namespace_operation(
Some(&lifecycle_guard),
bucket,
@@ -3176,6 +3207,82 @@ mod tests {
);
}
#[tokio::test]
async fn scoped_dirty_usage_incarnation_probe_does_not_migrate_legacy_metadata() {
let (dirs, store) = isolated_store_over_temp_disks().await;
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
let bucket = "scoped-ack-legacy";
for dir in &dirs {
std::fs::create_dir_all(dir.path().join(bucket)).expect("create legacy bucket");
}
let mut metadata = BucketMetadata::new(bucket);
metadata.bucket_incarnation_id = Uuid::nil();
sys.read()
.await
.persist_and_set(metadata)
.await
.expect("persist legacy metadata");
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(Uuid::new_v4()), false)
.await
.is_err()
);
assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
assert!(
sys.read()
.await
.get_config_from_disk(bucket)
.await
.expect("read metadata")
.bucket_incarnation_id
.is_nil()
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn scoped_dirty_usage_incarnation_rejects_deleted_and_recreated_bucket() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
let bucket = "scoped-ack-recreated";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let old = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
let guard = acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.expect("trusted incarnation fence");
assert_eq!(guard.checked_bucket_incarnation().expect("valid fences"), (bucket, old));
drop(guard);
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete bucket");
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.is_err()
);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("recreate bucket");
let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation");
assert_ne!(old, new);
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.is_err()
);
assert!(
acquire_config_write_guard_with_migration(sys, bucket, Some(new), false)
.await
.is_ok()
);
}
#[tokio::test]
async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
crates/ecstore/src/bucket/on_demand_migration/backfill.rs
+1
View File
@@ -684,6 +684,7 @@ async fn write_checkpoint(
};
let opts = ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(preconditions),
..Default::default()
};
crates/ecstore/src/bucket/on_demand_migration/breaker.rs
+6 -1
View File
@@ -86,7 +86,12 @@ impl BreakerVerdict {
Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => {
BreakerVerdict::Failure
}
Some(SourceError::AccessDenied | SourceError::Unsupported(_) | SourceError::Other(_)) => BreakerVerdict::Neutral,
Some(
SourceError::AccessDenied
| SourceError::Unsupported(_)
| SourceError::InvalidPagination(_)
| SourceError::Other(_),
) => BreakerVerdict::Neutral,
}
}
}
crates/ecstore/src/bucket/on_demand_migration/list_through.rs
+291 -20
View File
@@ -189,8 +189,8 @@ pub enum SourceListPlan {
/// delimiter — the source's own roll-up boundary matches the request's.
Page { prefix: String },
/// `filter.prefix` reaches past a delimiter, so every key the source could
/// contribute rolls into this one common prefix. One bounded probe listing
/// decides whether it exists; there is nothing to paginate.
/// contribute rolls into this one common prefix. Bounded probes follow
/// empty progressing pages until a key proves existence or the source ends.
Folded { probe_prefix: String, common_prefix: String },
}
@@ -279,6 +279,29 @@ pub struct FetchRequest {
pub token: Option<String>,
}
/// Invalid pagination metadata. Opaque cursor values are never included in errors.
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
pub enum ListPageError {
#[error("truncated listing has no continuation token")]
Missing,
#[error("truncated listing has an empty continuation token")]
Empty,
#[error("truncated listing repeats a continuation token")]
Repeated,
}
pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> {
if is_truncated {
match next_token {
None => return Err(ListPageError::Missing),
Some("") => return Err(ListPageError::Empty),
Some(next) if Some(next) == token => return Err(ListPageError::Repeated),
Some(_) => {}
}
}
Ok(())
}
#[derive(Debug, Default)]
struct SideState {
start: SideCursor,
@@ -364,6 +387,11 @@ impl ListThroughMerger {
/// or `filter.prefix` excludes it.
pub fn disable_source(&mut self) {
self.source.disabled = true;
// A refill can fail after a valid first page. A local-only response
// must discard both that source payload and its ordering horizon.
self.source.entries.clear();
self.source.pages.clear();
self.source.more = false;
}
pub fn next_fetch(&self) -> Option<FetchRequest> {
@@ -378,7 +406,13 @@ impl ListThroughMerger {
/// Records one fetched page. `entries` must be sorted by `name` and already
/// filtered with [`Self::accepts`]; the caller keeps the matching payloads
/// in the same order.
pub fn push_page(&mut self, side: MergeSide, entries: Vec<ListEntryKey>, is_truncated: bool, next_token: Option<String>) {
pub fn push_page(
&mut self,
side: MergeSide,
entries: Vec<ListEntryKey>,
is_truncated: bool,
next_token: Option<String>,
) -> Result<(), ListPageError> {
let state = match side {
MergeSide::Local => &mut self.local,
MergeSide::Source => &mut self.source,
@@ -387,15 +421,19 @@ impl ListThroughMerger {
Some(last) => last.next_token.clone(),
None => state.start.token.clone(),
};
// A truncated page without a cursor cannot be continued; treating the
// side as finished is the only alternative to looping on it forever.
state.more = is_truncated && next_token.is_some();
validate_list_page(is_truncated, token.as_deref(), next_token.as_deref())?;
// Also reject a cycle through an earlier page in this bounded fetch.
if is_truncated && state.pages.iter().any(|page| page.token == next_token) {
return Err(ListPageError::Repeated);
}
state.more = is_truncated;
state.pages.push(FetchedPage {
token,
count: entries.len(),
next_token: is_truncated.then_some(next_token).flatten(),
});
state.entries.extend(entries);
Ok(())
}
pub fn finish(self) -> MergeOutcome {
@@ -599,9 +637,15 @@ mod tests {
let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys);
let kept: Vec<ListEntryKey> = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect();
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned());
merger.push_page(fetch.side, kept, truncated, next);
merger
.push_page(fetch.side, kept, truncated, next)
.expect("reference provider pages must advance");
}
let outcome = merger.finish();
assert_eq!(outcome.is_truncated, outcome.next_token.is_some());
if outcome.is_truncated {
assert_ne!(outcome.next_token, token, "every truncated merged page must make progress");
}
page_sizes.push(outcome.picks.len());
for pick in &outcome.picks {
let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone();
@@ -616,11 +660,25 @@ mod tests {
}
fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec<ListEntryKey> {
let mut all: Vec<String> = local.iter().chain(source.iter()).cloned().collect();
all.sort();
all.dedup();
let (entries, _, _) = reference_page(&all, prefix, delimiter, None, usize::MAX);
entries
// This oracle builds the complete namespace independently of the
// provider's page/marker helper and the production merger.
let mut namespace = std::collections::BTreeMap::new();
for key in local.iter().chain(source) {
let Some(suffix) = key.strip_prefix(prefix) else {
continue;
};
if let Some(delimiter) = delimiter.filter(|delimiter| !delimiter.is_empty())
&& let Some((directory, _)) = suffix.split_once(delimiter)
{
namespace.insert(format!("{prefix}{directory}{delimiter}"), true);
continue;
}
namespace.insert(key.clone(), false);
}
namespace
.into_iter()
.map(|(name, is_prefix)| ListEntryKey { name, is_prefix })
.collect()
}
#[test]
@@ -662,7 +720,9 @@ mod tests {
token: None
})
);
merger.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None);
merger
.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None)
.expect("local EOF is valid");
assert_eq!(merger.next_fetch(), None);
let outcome = merger.finish();
assert_eq!(outcome.picks.len(), 1);
@@ -683,12 +743,14 @@ mod tests {
};
let mut merger = ListThroughMerger::new(1, Some(&resume));
merger.disable_source();
merger.push_page(
MergeSide::Local,
vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
true,
Some("local-2".to_string()),
);
merger
.push_page(
MergeSide::Local,
vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
true,
Some("local-2".to_string()),
)
.expect("local cursor advances");
let outcome = merger.finish();
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("truncated page carries a token");
@@ -698,6 +760,212 @@ mod tests {
assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed");
}
#[test]
fn truncated_pages_require_a_nonempty_advancing_cursor() {
for side in [MergeSide::Local, MergeSide::Source] {
for entries in [vec![], vec![ListEntryKey::object("a")]] {
for (next, expected) in [
(None, Err(ListPageError::Missing)),
(Some(""), Err(ListPageError::Empty)),
(Some("stuck"), Err(ListPageError::Repeated)),
(Some("advances"), Ok(())),
] {
let resume = ListThroughToken::new(
SideCursor {
token: Some("stuck".into()),
done: false,
},
SideCursor {
token: Some("stuck".into()),
done: false,
},
None,
);
let mut merger = ListThroughMerger::new(2, Some(&resume));
let result = merger.push_page(side, entries.clone(), true, next.map(str::to_string));
assert_eq!(result, expected, "{side:?}, {entries:?}, {next:?}");
let state = if side == MergeSide::Local {
&merger.local
} else {
&merger.source
};
assert_eq!(state.pages.len(), usize::from(result.is_ok()), "invalid page must not be accepted");
}
}
}
}
#[test]
fn repeated_empty_cursor_is_rejected_before_an_identical_page_can_escape() {
let resume = ListThroughToken::new(
SideCursor { token: None, done: true },
SideCursor {
token: Some("stuck".into()),
done: false,
},
None,
);
let mut merger = ListThroughMerger::new(2, Some(&resume));
assert_eq!(
merger.next_fetch(),
Some(FetchRequest {
side: MergeSide::Source,
token: Some("stuck".into())
})
);
assert_eq!(
merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
Err(ListPageError::Repeated)
);
}
#[test]
fn empty_pages_may_advance_within_the_fetch_budget_until_eof() {
let mut merger = ListThroughMerger::new(2, None);
merger.push_page(MergeSide::Local, vec![], false, None).expect("local EOF");
for next in ["opaque-z", "opaque-a"] {
assert_eq!(merger.next_fetch().expect("bounded source fetch").side, MergeSide::Source);
merger
.push_page(MergeSide::Source, vec![], true, Some(next.into()))
.expect("opaque cursor advances regardless of sort order");
}
assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget");
let outcome = merger.finish();
assert!(outcome.picks.is_empty());
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("empty progressing page has a cursor");
assert_eq!(token.source.as_deref(), Some("opaque-a"));
let mut merger = ListThroughMerger::new(2, Some(&token));
assert_eq!(merger.next_fetch().expect("source resumes").token.as_deref(), Some("opaque-a"));
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None)
.expect("source EOF");
let outcome = merger.finish();
assert_eq!(
outcome.picks,
vec![MergePick {
side: MergeSide::Source,
index: 0
}]
);
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
}
#[test]
fn a_cursor_cycle_inside_the_fetch_budget_is_rejected() {
let resume = ListThroughToken::new(
SideCursor { token: None, done: true },
SideCursor {
token: Some("first".into()),
done: false,
},
None,
);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Source, vec![], true, Some("second".into()))
.expect("first page advances");
assert_eq!(
merger.push_page(MergeSide::Source, vec![], true, Some("first".into())),
Err(ListPageError::Repeated)
);
}
#[test]
fn source_refill_failure_discards_buffered_source_entries_and_horizon() {
let mut merger = ListThroughMerger::new(2, None);
merger
.push_page(MergeSide::Local, vec![ListEntryKey::object("z")], false, None)
.expect("local EOF");
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("a")], true, Some("stuck".into()))
.expect("first source page advances");
assert_eq!(merger.next_fetch().expect("source refill is required").token.as_deref(), Some("stuck"));
assert_eq!(
merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
Err(ListPageError::Repeated)
);
merger.disable_source();
let outcome = merger.finish();
assert_eq!(
outcome.picks,
vec![MergePick {
side: MergeSide::Local,
index: 0
}]
);
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
}
#[test]
fn list_through_static_namespace_boundary_matrix() {
let corpus = [
"a",
"a/",
"a/b",
"a/b/child",
"a0",
"b",
"b/leaf",
"quote\"&<",
"space key",
"z",
"é",
"中/文",
];
for count in [0, 1, 3, 4, corpus.len()] {
let keys: Vec<String> = corpus[..count].iter().map(|key| (*key).to_string()).collect();
for placement in 0..3 {
let (local, source): (Vec<_>, Vec<_>) =
keys.iter()
.enumerate()
.fold((vec![], vec![]), |(mut local, mut source), (index, key)| {
if placement != 1 || index % 2 == 0 {
local.push(key.clone());
}
if placement != 0 || index % 2 == 0 {
source.push(key.clone());
}
(local, source)
});
for prefix in ["", "a", "a/", "中/"] {
for delimiter in [None, Some("/")] {
for max_keys in [1, 3, 4] {
let oracle = expected(&local, &source, prefix, delimiter);
let (emitted, sizes) = walk(&local, &source, prefix, delimiter, max_keys);
assert_eq!(
emitted.iter().map(|(entry, _)| entry.clone()).collect::<Vec<_>>(),
oracle,
"count={count}, placement={placement}, prefix={prefix}, delimiter={delimiter:?}, max={max_keys}"
);
let expected_sizes: Vec<_> = if oracle.is_empty() {
vec![0]
} else {
oracle.chunks(max_keys).map(<[ListEntryKey]>::len).collect()
};
assert_eq!(sizes, expected_sizes, "exact max and max+1 boundaries must agree");
}
}
}
}
}
}
#[test]
fn list_through_large_overlap_walk_keeps_all_5300_keys() {
let source: Vec<_> = (0..5000).map(|index| format!("k{index:05}")).collect();
let local: Vec<_> = (4800..5300).map(|index| format!("k{index:05}")).collect();
let (emitted, sizes) = walk(&local, &source, "", None, 333);
assert_eq!(emitted.len(), 5300);
for (index, (entry, side)) in emitted.iter().enumerate() {
assert_eq!(entry.name, format!("k{index:05}"));
assert_eq!(*side, if index >= 4800 { MergeSide::Local } else { MergeSide::Source });
}
assert_eq!(sizes, [vec![333; 15], vec![305]].concat());
}
#[test]
fn token_round_trips_and_rejects_tampering() {
let token = ListThroughToken::new(
@@ -796,7 +1064,10 @@ mod tests {
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(256))]
#![proptest_config(ProptestConfig {
rng_seed: proptest::test_runner::RngSeed::Fixed(0xec5706),
..ProptestConfig::with_cases(256)
})]
/// Full pagination of a merged listing equals the sorted, deduplicated
/// union of both sides, with every shared key served by local, and no
crates/ecstore/src/bucket/on_demand_migration/source_client.rs
+106 -7
View File
@@ -25,6 +25,7 @@
//! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never
//! forwarded: v1 rejects SSE-C source objects outright.
use super::list_through::{ListPageError, validate_list_page};
use crate::bucket::remote_s3_client::{
PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config,
};
@@ -223,6 +224,8 @@ pub enum SourceError {
ServerError(u16),
#[error("unsupported source object: {0}")]
Unsupported(String),
#[error("invalid source listing: {0}")]
InvalidPagination(#[from] ListPageError),
#[error("source request failed: {0}")]
Other(String),
}
@@ -245,6 +248,7 @@ impl SourceError {
SourceError::Connect(_) => "connect",
SourceError::ServerError(_) => "server_error",
SourceError::Unsupported(_) => "unsupported",
SourceError::InvalidPagination(_) => "invalid_pagination",
SourceError::Other(_) => "other",
}
}
@@ -714,6 +718,7 @@ impl SourceClient {
..*request
})
.await?;
validate_list_page(page.is_truncated, request.continuation_token, page.next_continuation_token.as_deref())?;
page.objects = page
.objects
.into_iter()
@@ -800,11 +805,6 @@ impl SourceBackend for S3SourceBackend {
let is_truncated = output.is_truncated.unwrap_or(false);
let next_continuation_token = output.next_continuation_token;
if is_truncated && next_continuation_token.is_none() {
return Err(SourceError::Other(
"source reported a truncated listing without a continuation token".to_string(),
));
}
let objects = output
.contents
.unwrap_or_default()
@@ -1274,7 +1274,9 @@ mod tests {
<CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes>
<CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes>
</ListBucketResult>"#;
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await;
let next_body = body.replace("data/opaque", "data/next");
let (client, requests) =
scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), &next_body)]).await;
let first = client
.list_page(&SourceListRequest {
prefix: Some("photos/"),
@@ -1336,7 +1338,104 @@ mod tests {
.list_objects_v2(None, None, 10)
.await
.expect_err("truncated page without token is corrupt");
assert!(matches!(err, SourceError::Other(_)), "{err:?}");
assert!(matches!(err, SourceError::InvalidPagination(ListPageError::Missing)), "{err:?}");
}
#[tokio::test]
async fn list_page_validates_s3_cursor_progress_before_mapping_entries() {
for contents in ["", "<Contents><Key>data/a</Key><Size>1</Size></Contents>"] {
for (truncated, next, expected) in [
(true, None, Some(ListPageError::Missing)),
(true, Some(""), Some(ListPageError::Empty)),
(true, Some("stuck"), Some(ListPageError::Repeated)),
(true, Some("opaque-next"), None),
(false, None, None),
(false, Some("stuck"), None),
] {
let next_xml = next
.map(|next| format!("<NextContinuationToken>{next}</NextContinuationToken>"))
.unwrap_or_default();
let body = format!(
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><IsTruncated>{truncated}</IsTruncated>{next_xml}{contents}</ListBucketResult>"
);
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), &body)]).await;
let result = client
.list_page(&SourceListRequest {
continuation_token: Some("stuck"),
max_keys: 2,
..Default::default()
})
.await;
match expected {
Some(expected) => {
let error = result.expect_err("malformed pagination must fail at the provider boundary");
assert!(
matches!(&error, SourceError::InvalidPagination(actual) if *actual == expected),
"{error:?}"
);
assert_eq!(error.class_label(), "invalid_pagination");
assert!(!error.is_retryable());
assert!(!error.to_string().contains("stuck"), "errors must not echo opaque tokens");
}
None => {
let page = result.expect("progressing empty/nonempty pages and EOF are valid");
assert_eq!(page.is_truncated, truncated);
assert_eq!(page.next_continuation_token.as_deref(), next);
assert_eq!(page.objects.len(), usize::from(!contents.is_empty()));
if let Some(object) = page.objects.first() {
assert_eq!(object.key, "a");
}
}
}
let requests = recorded(&requests);
assert_eq!(requests.len(), 1, "invalid pagination must not be retried");
assert!(requests[0].uri.contains("continuation-token=stuck"));
}
}
}
struct ListOnlyBackend(SourcePage);
#[async_trait::async_trait]
impl SourceBackend for ListOnlyBackend {
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
assert_eq!(request.continuation_token, Some("stuck"), "opaque cursors reach every provider unchanged");
Ok(self.0.clone())
}
async fn head(&self, _key: &str) -> Result<SourceHead, SourceError> {
panic!("unexpected HEAD in list test")
}
async fn get(&self, _key: &str, _range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
panic!("unexpected GET in list test")
}
async fn tagging(&self, _key: &str) -> Result<HashMap<String, String>, SourceError> {
panic!("unexpected tagging in list test")
}
async fn probe(&self) -> Result<(), SourceError> {
panic!("unexpected probe in list test")
}
}
#[tokio::test]
async fn list_page_validates_non_s3_provider_cursors_at_the_common_boundary() {
for (next, expected) in [
(None, ListPageError::Missing),
(Some(""), ListPageError::Empty),
(Some("stuck"), ListPageError::Repeated),
] {
let mut client = prefix_client(Some("data/".into()));
client.backend = Box::new(ListOnlyBackend(SourcePage {
is_truncated: true,
next_continuation_token: next.map(str::to_string),
..Default::default()
}));
let error = client
.list_objects_v2(None, Some("stuck"), 2)
.await
.expect_err("all providers must advance pagination");
assert!(matches!(error, SourceError::InvalidPagination(actual) if actual == expected));
}
}
const TAGGING_BODY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
crates/ecstore/src/bucket/on_demand_migration/stats.rs
+1 -1
View File
@@ -177,7 +177,7 @@ impl From<&SourceError> for PullFailureReason {
SourceError::Connect(_) => PullFailureReason::SourceConnect,
SourceError::ServerError(_) => PullFailureReason::SourceServerError,
SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported,
SourceError::Other(_) => PullFailureReason::SourceOther,
SourceError::InvalidPagination(_) | SourceError::Other(_) => PullFailureReason::SourceOther,
}
}
}
crates/ecstore/src/bucket/remote_s3_client.rs
+170 -1
View File
@@ -652,9 +652,10 @@ async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
#[cfg(test)]
mod tests {
use super::*;
use aws_smithy_async::time::TimeSource;
use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
RemoteS3EndpointSpec {
@@ -824,6 +825,174 @@ mod tests {
);
}
#[derive(Clone, Debug)]
struct ClockSkewTimeSource(Arc<AtomicU64>);
impl TimeSource for ClockSkewTimeSource {
fn now(&self) -> SystemTime {
SystemTime::UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst))
}
}
#[derive(Clone, Debug)]
struct ClockSkewConnector {
request_headers: RecordedHeaders,
error_code: &'static str,
skew_seconds: i64,
clock: ClockSkewTimeSource,
}
fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {
headers
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(name))
.map(|(_, value)| value.as_str())
.unwrap_or_else(|| panic!("signed request must contain {name}"))
}
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
.expect("SDK signing timestamp must use the SigV4 format")
}
impl SmithyHttpConnector for ClockSkewConnector {
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
let mut headers = self.request_headers.lock().expect("clock skew request capture lock");
assert!(headers.len() < 3, "clock skew fixture must not exceed two GET attempts and one HEAD");
headers.push(
request
.headers()
.iter()
.map(|(key, value)| (key.to_string(), value.to_string()))
.collect(),
);
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
+ chrono::Duration::seconds(self.skew_seconds);
let (status, body) = if headers.len() == 1 {
(
403,
format!("<Error><Code>{}</Code><Message>Clock skew fixture</Message></Error>", self.error_code),
)
} else {
(200, String::new())
};
let response = http::Response::builder()
.status(status)
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
.header("content-type", "application/xml")
.header("content-length", body.len())
.body(SdkBody::from(body))
.expect("clock skew fixture response");
HttpConnectorFuture::ready(Ok(HttpResponse::try_from(response).expect("Smithy fixture response")))
}
}
async fn clock_skew_client(
error_code: &'static str,
skew_seconds: i64,
retry: RemoteS3RetryPolicy,
) -> (S3Client, RecordedHeaders, ClockSkewTimeSource) {
let headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
let clock = ClockSkewTimeSource(Arc::new(AtomicU64::new(1_700_000_000)));
let connector = SharedHttpConnector::new(ClockSkewConnector {
request_headers: Arc::clone(&headers),
error_code,
skew_seconds,
clock: clock.clone(),
});
let mut spec = spec("s3.example.com", true);
spec.retry = retry;
let config = build_remote_s3_config(&spec)
.await
.expect("clock skew fixture uses the production outbound configuration")
.http_client(http_client_fn(move |_settings, _components| connector.clone()))
.time_source(clock.clone())
.build();
(S3Client::from_conf(config), headers, clock)
}
#[tokio::test(start_paused = true)]
async fn remote_s3_clock_skew_retries_resign_and_seed_next_operation() {
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
for skew_seconds in [-600, 600] {
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
client
.get_object()
.bucket("bucket")
.key("object")
.send()
.await
.expect("clock skew GET must retry successfully");
assert_eq!(
headers.lock().expect("captured requests").len(),
2,
"{error_code}: GET needs exactly one retry"
);
clock.0.fetch_add(17, Ordering::SeqCst);
// SDK signing time is independent of Tokio's retry/scheduler clock.
tokio::time::advance(Duration::from_secs(61)).await;
client
.head_bucket()
.bucket("bucket")
.send()
.await
.expect("subsequent HEAD must use the client's cached skew");
let headers = headers.lock().expect("captured signed requests");
assert_eq!(headers.len(), 3, "subsequent operation must succeed on its first attempt");
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
assert_eq!(
signing_time(&headers[1]),
initial + chrono::Duration::seconds(skew_seconds),
"{error_code}: retry must apply the measured offset exactly"
);
assert_eq!(
signing_time(&headers[2]),
initial + chrono::Duration::seconds(skew_seconds + 17),
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
);
let signature = |index: usize| {
recorded_header(&headers[index], "authorization")
.rsplit_once("Signature=")
.expect("SigV4 authorization contains a signature")
.1
};
assert_ne!(
signature(0),
signature(1),
"{error_code}: retry must be signed again after adjusting its date"
);
}
}
}
#[tokio::test(start_paused = true)]
async fn remote_s3_clock_skew_respects_one_attempt_policy() {
use aws_smithy_types::error::metadata::ProvideErrorMetadata;
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
for retry in [
RemoteS3RetryPolicy::Disabled,
RemoteS3RetryPolicy::Standard { max_attempts: 1 },
] {
let (client, headers, _clock) = clock_skew_client(error_code, 600, retry).await;
let error = client
.get_object()
.bucket("bucket")
.key("object")
.send()
.await
.expect_err("clock skew must not override the caller's one-attempt budget");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some(error_code));
assert_eq!(
headers.lock().expect("captured requests").len(),
1,
"{error_code}: {retry:?} must send exactly one request"
);
}
}
}
#[test]
fn path_style_auto_and_path_force_path_style() {
assert!(PathStyle::Auto.force_path_style());
crates/ecstore/src/cluster/rpc/client.rs
+19
View File
@@ -30,6 +30,7 @@ use rustfs_protos::{
ChannelClass, create_new_channel, get_channel_for_class,
proto_gen::node_service::{
heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient,
scanner_control_service_client::ScannerControlServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
},
};
@@ -60,6 +61,24 @@ pub async fn node_service_time_out_client(
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
}
pub(crate) async fn scanner_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> crate::error::Result<ScannerControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr)
.await
.map_err(|err| crate::error::Error::other(err.to_string()))?,
};
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(limit)
.max_encoding_message_size(limit))
}
pub async fn heal_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+47
View File
@@ -2050,6 +2050,53 @@ impl PeerRestClient {
.await
}
/// Probe only: scoped ACK production requires a durable per-bucket proof.
pub async fn scanner_scoped_dirty_usage_capability(
&self,
owner_id: String,
instance_id: String,
entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
) -> Result<bool> {
use rustfs_protos::scoped_dirty_usage::*;
let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
owner_id,
instance_id,
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
probe_only: true,
entries,
};
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
self.finalize_result(
async {
let mut client = super::client::scanner_control_time_out_client(
&self.grid_host,
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
)
.await?;
let mut request = Request::new(payload.clone());
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|| response.owner_id != payload.owner_id
|| response.instance_id != payload.instance_id
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|| response.cleared != 0
{
return Err(Error::other("scoped dirty usage capability response does not match request"));
}
Ok(response.supported)
}
.await,
)
.await
}
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
let result = self
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
crates/ecstore/src/core/pools.rs
+4
View File
@@ -5493,6 +5493,7 @@ where
fence.ensure_held()?;
let mut opts = ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
no_lock: true,
http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
..Default::default()
@@ -14412,6 +14413,7 @@ impl ECStore {
encoded.clone(),
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -14566,6 +14568,7 @@ impl ECStore {
encoded,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(http_preconditions),
..Default::default()
},
@@ -14957,6 +14960,7 @@ impl ECStore {
encoded,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(etag),
..Default::default()
crates/ecstore/src/disk/disk_store.rs
+77 -8
View File
@@ -317,6 +317,22 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
dst_path: &str,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> Result<RenameDataResp> {
self.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
.await
.result
}
}
impl LocalDiskWrapper {
pub(in crate::disk) async fn rename_data_observed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
external_guard: Option<Arc<dyn Send + Sync>>,
) -> super::RenameDataObservation {
let operation = self.clone();
let src_volume = src_volume.to_owned();
let src_path = src_path.to_owned();
@@ -333,22 +349,35 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
} else {
get_max_timeout_duration()
};
run_owned_mutation(external_guard, move || async move {
operation
let observed = run_owned_mutation(external_guard, move || async move {
let mut preflight_rejection = None;
let result = operation
.track_disk_health_mutation(
"rename_data",
DiskMetricMutation::Write,
|| async {
operation
.disk
.rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path)
.await
// Preserve the former DiskAPI future's single boxing boundary.
let observed =
Box::pin(
operation
.disk
.rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
)
.await;
preflight_rejection = observed.preflight_rejection;
observed.result
},
timeout_duration,
)
.await
.await;
// Health tracking must observe the real disk error, not an Ok tuple.
Ok(super::RenameDataObservation {
result,
preflight_rejection,
})
})
.await
.await;
observed.unwrap_or_else(|error| super::RenameDataObservation::unknown(Err(error)))
}
}
@@ -2588,6 +2617,46 @@ mod tests {
assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1));
}
#[tokio::test]
async fn rename_preflight_evidence_preserves_health_errors_and_owned_reply() {
for source_exists in [false, true] {
for guarded in [false, true] {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8"))
.expect("endpoint should parse");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
if source_exists {
disk.make_volume("source").await.expect("source volume should exist");
}
let wrapper = LocalDiskWrapper::new(disk, false);
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let external_guard = guarded.then(|| Arc::new(DropProbe(Arc::clone(&drops))) as Arc<dyn Send + Sync>);
let mut file_info = FileInfo::new("object", 1, 0);
file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
file_info.erasure.index = 1;
let observed = wrapper
.rename_data_observed("source", "object", &file_info, "missing-destination", "object", external_guard)
.await;
assert!(observed.rejected_before_publication(), "normal access rejection must carry proof");
assert!(matches!(observed.result, Err(DiskError::VolumeNotFound)));
let snapshot = wrapper.metrics_snapshot();
assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1));
assert_eq!(snapshot.total_writes, 0, "health tracking must not observe the rejection as Ok");
assert_eq!(drops.load(Ordering::SeqCst), usize::from(guarded));
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
let observed = wrapper
.rename_data_observed("source", "object", &file_info, "missing-destination", "object", None)
.await;
assert!(!observed.rejected_before_publication(), "wrapper errors carry no local preflight proof");
assert!(matches!(observed.result, Err(DiskError::FaultyDisk)));
let snapshot = wrapper.metrics_snapshot();
assert_eq!(snapshot.total_errors_availability, 1);
assert_eq!(snapshot.total_writes, 0);
}
}
}
#[tokio::test]
async fn local_disk_health_wrapper_counts_returned_availability_errors() {
let dir = tempfile::tempdir().expect("temp dir should be created");
crates/ecstore/src/disk/local.rs
+202 -1143
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/disk/local/commit.rs
+1233
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/disk/mod.rs
+49
View File
@@ -75,6 +75,25 @@ use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncWrite};
use uuid::Uuid;
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
pub(crate) struct RenameDataObservation {
pub(crate) result: Result<RenameDataResp>,
preflight_rejection: Option<local::LocalRenamePreflightRejection>,
}
impl RenameDataObservation {
fn unknown(result: Result<RenameDataResp>) -> Self {
Self {
result,
preflight_rejection: None,
}
}
pub(crate) fn rejected_before_publication(&self) -> bool {
self.result.is_err() && self.preflight_rejection.is_some()
}
}
const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
@@ -711,6 +730,36 @@ impl Disk {
.await
}
pub(crate) async fn rename_data_borrowed_with_fence_observed(
&self,
src_volume: &str,
src_path: &str,
fi: &FileInfo,
dst_volume: &str,
dst_path: &str,
scanner_publication_lease_token: Option<Uuid>,
) -> RenameDataObservation {
match self {
Disk::Local(local_disk) => {
local_disk
.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
.await
}
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
remote_disk
.rename_data_borrowed_with_fence(
src_volume,
src_path,
fi,
dst_volume,
dst_path,
scanner_publication_lease_token,
)
.await,
),
}
}
pub(crate) async fn rename_data_borrowed_with_fence(
&self,
src_volume: &str,
crates/ecstore/src/object_api/types.rs
+16
View File
@@ -870,6 +870,18 @@ impl TierFreeVersionReceiptSink {
}
}
/// Internal PUT completion boundary; this does not change fsync or write quorum.
#[doc(hidden)]
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub enum WriteCompletion {
/// Return at write quorum when the commit owner can retain its guards.
#[default]
Quorum,
/// Drain the rename fan-out before returning. Minority failures still heal
/// after a successful quorum commit; this does not require every disk to succeed.
TailDrained,
}
#[derive(Default, Clone)]
pub struct ObjectOptions {
// Use the maximum parity (N/2), used when saving server configuration files
@@ -896,6 +908,10 @@ pub struct ObjectOptions {
/// Persisted bucket incarnation observed before authorization.
pub expected_bucket_incarnation_id: Option<Uuid>,
pub no_lock: bool,
/// Control-plane writers that immediately read or CAS the same namespace
/// key use TailDrained without changing namespace lock ownership.
#[doc(hidden)]
pub write_completion: WriteCompletion,
/// True when an upper layer already holds the object read lock before
/// forwarding a no_lock read to the set layer.
pub metadata_cache_safe: bool,
crates/ecstore/src/services/tier/tier_admin.rs
+2 -4
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
@@ -145,7 +143,7 @@ mod tests {
assert_eq!(creds.access_key, "access");
assert_eq!(creds.secret_key, "secret");
assert_eq!(creds.creds_json.as_slice(), &service_account[..]);
assert_eq!(creds.creds_json.as_slice(), service_account);
let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
assert_eq!(wire["access"], "access");
@@ -162,7 +160,7 @@ mod tests {
.expect("the former RustFS field names and byte-array encoding should remain readable");
assert_eq!(legacy.access_key, "legacy-access");
assert_eq!(legacy.secret_key, "legacy-secret");
assert_eq!(legacy.creds_json.as_slice(), &service_account[..]);
assert_eq!(legacy.creds_json.as_slice(), service_account);
}
#[test]
crates/ecstore/src/services/tier/tier_mutation_intent.rs
+2
View File
@@ -460,6 +460,7 @@ where
data,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -556,6 +557,7 @@ where
data,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(current_etag.to_string()),
..Default::default()
crates/ecstore/src/services/tier/tier_probe_intent.rs
+2
View File
@@ -494,6 +494,7 @@ where
data,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_none_match: Some("*".to_string()),
..Default::default()
@@ -549,6 +550,7 @@ where
data,
&ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: Some(current.record_etag.clone()),
..Default::default()
crates/ecstore/src/services/tier/warm_backend_aliyun.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_azure.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_gcs.rs
+8 -10
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::{HashMap, HashSet};
use std::future::Future;
@@ -146,11 +144,11 @@ pub struct WarmBackendGCS {
impl WarmBackendGCS {
pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> {
if conf.creds == "" {
if conf.creds.is_empty() {
return Err(std::io::Error::other("both access and secret keys are required"));
}
if conf.bucket == "" {
if conf.bucket.is_empty() {
return Err(std::io::Error::other("no bucket name was provided"));
}
@@ -195,11 +193,11 @@ impl WarmBackendGCS {
}
pub fn get_dest(&self, object: &str) -> String {
let mut dest_obj = object.to_string();
if self.prefix != "" {
dest_obj = format!("{}/{}", &self.prefix, object);
if self.prefix.is_empty() {
object.to_string()
} else {
format!("{}/{}", self.prefix, object)
}
return dest_obj;
}
}
@@ -223,7 +221,7 @@ impl WarmBackend for WarmBackendGCS {
let bucket = gcs_bucket_resource_name(&self.bucket);
let Ok(res) = Box::pin(
self.client
.write_object(&bucket, &self.get_dest(object), Bytes::from(d))
.write_object(&bucket, self.get_dest(object), Bytes::from(d))
.send_buffered(),
)
.await
@@ -240,7 +238,7 @@ impl WarmBackend for WarmBackendGCS {
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let bucket = gcs_bucket_resource_name(&self.bucket);
let mut req = self.client.read_object(&bucket, &self.get_dest(object));
let mut req = self.client.read_object(&bucket, self.get_dest(object));
let mut max_response_bytes = None;
if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation);
crates/ecstore/src/services/tier/warm_backend_huaweicloud.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_minio.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_r2.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_rustfs.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/services/tier/warm_backend_tencent.rs
-2
View File
@@ -15,8 +15,6 @@
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use std::collections::HashMap;
crates/ecstore/src/set_disk/core/io_primitives.rs
+994 -527
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/set_disk/core/metadata_quorum.rs
+385
View File
@@ -0,0 +1,385 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Pure metadata quorum and early-stop decisions for `SetDisks` reads.
//!
//! Disk scheduling, coalescing, cancellation, and late shard materialization
//! remain with their existing owners; this module only classifies observations.
use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
};
use crate::disk::error::DiskError;
use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS;
use crate::set_disk::file_info_is_valid_for_metadata;
use rustfs_filemeta::FileInfo;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
pub(in crate::set_disk) reason: &'static str,
}
#[derive(Clone, Debug)]
pub(in crate::set_disk) struct MetadataQuorumAccumulator {
pub(in crate::set_disk) total_disks: usize,
pub(in crate::set_disk) default_parity_count: usize,
pub(in crate::set_disk) allow_early_stop: bool,
pub(in crate::set_disk) valid_responses: usize,
pub(in crate::set_disk) not_found_responses: usize,
pub(in crate::set_disk) version_not_found_responses: usize,
pub(in crate::set_disk) ignored_errors: usize,
pub(in crate::set_disk) hard_errors: usize,
pub(in crate::set_disk) candidate: Option<FileInfo>,
pub(in crate::set_disk) candidate_votes: usize,
// Bitset of shard indexes whose metadata matches the candidate. Erasure
// layouts are capped at 16 shards, so this stays allocation-free on the
// GET metadata hot path.
candidate_shard_mask: u16,
pub(in crate::set_disk) conflicting_metadata: bool,
pub(in crate::set_disk) delete_marker_seen: bool,
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
pub(in crate::set_disk) delete_marker_votes: usize,
pub(in crate::set_disk) requested_version_id: String,
pub(in crate::set_disk) matching_version_votes: usize,
}
impl MetadataQuorumAccumulator {
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
Self {
total_disks,
default_parity_count,
allow_early_stop,
valid_responses: 0,
not_found_responses: 0,
version_not_found_responses: 0,
ignored_errors: 0,
hard_errors: 0,
candidate: None,
candidate_votes: 0,
candidate_shard_mask: 0,
conflicting_metadata: false,
delete_marker_seen: false,
delete_marker_candidates: Vec::new(),
delete_marker_votes: 0,
requested_version_id: String::new(),
matching_version_votes: 0,
}
}
pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
self.requested_version_id = version_id.to_string();
self
}
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
self.observe_file_info_with_index(None, file_info);
}
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
self.observe_file_info_with_index(Some(disk_index), file_info);
}
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
if !file_info_is_valid_for_metadata(file_info) {
self.hard_errors = self.hard_errors.saturating_add(1);
return;
}
self.valid_responses = self.valid_responses.saturating_add(1);
// Track version match for versioned requests
if !self.requested_version_id.is_empty()
&& let Some(ref vid) = file_info.version_id
&& vid.to_string() == self.requested_version_id
{
self.matching_version_votes = self.matching_version_votes.saturating_add(1);
}
if file_info.is_canonical_delete_marker() {
self.delete_marker_seen = true;
if let Some((_, votes)) = self
.delete_marker_candidates
.iter_mut()
.find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
{
*votes = votes.saturating_add(1);
} else {
self.delete_marker_candidates.push((file_info.clone(), 1));
}
self.delete_marker_votes = self
.delete_marker_candidates
.iter()
.map(|(_, votes)| *votes)
.max()
.unwrap_or_default();
self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
return;
}
match &self.candidate {
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
self.candidate_votes = self.candidate_votes.saturating_add(1);
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
Some(_) => {
self.conflicting_metadata = true;
}
None => {
self.candidate = Some(file_info.clone());
self.candidate_votes = 1;
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
}
}
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
return None;
}
Some(1u16 << (erasure_index - 1))
}
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
self.candidate_read_reserve_target()
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
}
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
let candidate = self.candidate.as_ref()?;
Some(
candidate
.erasure
.data_blocks
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
)
}
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
match err {
DiskError::FileNotFound | DiskError::VolumeNotFound => {
self.not_found_responses = self.not_found_responses.saturating_add(1);
}
DiskError::FileVersionNotFound => {
self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
}
_ if is_metadata_fanout_ignored_error(err) => {
self.ignored_errors = self.ignored_errors.saturating_add(1);
}
_ => {
self.hard_errors = self.hard_errors.saturating_add(1);
}
}
}
pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.delete_marker_votes >= self.default_write_quorum() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
});
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self
.candidate
.as_ref()
.and_then(|candidate| self.candidate_latest_quorum(candidate))
.is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
{
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
});
}
None
}
/// Check if a versioned request can early-stop because the requested
/// version_id has reached quorum across disks.
pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.requested_version_id.is_empty() {
return None;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self.matching_version_votes >= self.read_quorum_for_version() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
});
}
None
}
pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
if !self.allow_early_stop {
return false;
}
if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
return true;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return false;
}
if !self.requested_version_id.is_empty()
&& self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
{
return true;
}
match &self.candidate {
Some(candidate) => self
.candidate_latest_quorum(candidate)
.is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
None => pending >= self.default_write_quorum(),
}
}
/// Compute the read quorum threshold for version-aware early-stop.
/// Uses `total_disks / 2` (like `missing_response_quorum`) when
/// `default_parity_count` is set, otherwise requires all disks.
pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
self.missing_response_quorum()
}
pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
if !self.allow_early_stop {
return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
}
if self.conflicting_metadata {
return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
}
if self.delete_marker_seen {
return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
}
let missing_response_quorum = self.missing_response_quorum();
if self.version_not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
}
if self.not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
}
if self.hard_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_ERROR;
}
if self.ignored_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
}
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
}
pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
if self.default_parity_count == 0 {
return Some(self.total_disks);
}
if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
return None;
}
let data_blocks = candidate.erasure.data_blocks;
Some(if data_blocks == candidate.erasure.parity_blocks {
data_blocks.saturating_add(1)
} else {
data_blocks
})
}
pub(crate) fn default_write_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
return self.total_disks;
}
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
if data_blocks == self.default_parity_count {
data_blocks.saturating_add(1)
} else {
data_blocks
}
}
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
self.total_disks
} else {
self.total_disks / 2
}
}
}
pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
left.volume == right.volume
&& left.name == right.name
&& left.version_id == right.version_id
&& left.is_latest == right.is_latest
&& left.deleted == right.deleted
&& left.mark_deleted == right.mark_deleted
&& left.transition_status == right.transition_status
&& left.transitioned_objname == right.transitioned_objname
&& left.transition_tier == right.transition_tier
&& left.transition_version_id == right.transition_version_id
&& left.transition_version == right.transition_version
&& left.transition_version_state == right.transition_version_state
&& left.expire_restored == right.expire_restored
&& left.size == right.size
&& left.mod_time == right.mod_time
&& left.mode == right.mode
&& left.written_by_version == right.written_by_version
&& left.metadata == right.metadata
&& left.replication_state_internal == right.replication_state_internal
&& left.parts == right.parts
&& left.checksum == right.checksum
&& left.versioned == right.versioned
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.data_dir == right.data_dir
&& left.erasure.algorithm == right.erasure.algorithm
&& left.erasure.data_blocks == right.erasure.data_blocks
&& left.erasure.parity_blocks == right.erasure.parity_blocks
&& left.erasure.block_size == right.erasure.block_size
&& left.erasure.distribution == right.erasure.distribution
}
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
}
crates/ecstore/src/set_disk/core/mod.rs
+1
View File
@@ -18,3 +18,4 @@
//! duplicating read/write/erasure logic.
pub(crate) mod io_primitives;
mod metadata_quorum;
crates/ecstore/src/set_disk/ops/object.rs
+444 -31
View File
@@ -299,11 +299,11 @@ use crate::error::is_err_invalid_upload_id;
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
use crate::object_api::{
NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion,
};
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal};
use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal};
#[cfg(test)]
use crate::storage_api_contracts::namespace::NamespaceLocking;
#[cfg(test)]
@@ -3548,6 +3548,7 @@ impl SetDisks {
(None, None, None)
};
let mut tmp_cleanup_owned = false;
let rollback_receipt = RenameRollbackReceipt::default();
let operation = async {
let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
@@ -4256,6 +4257,7 @@ impl SetDisks {
let commit_bucket = bucket.to_owned();
let commit_object = object.to_owned();
let commit_tmp_dir = tmp_dir.clone();
let commit_rollback_receipt = rollback_receipt.clone();
let commit_object_lock_guard = object_lock_guard.take();
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
let commit_publication_guard = publication_commit_guard.take();
@@ -4266,13 +4268,17 @@ impl SetDisks {
// complete rename fan-out drains. Keep this path synchronous so
// its terminal state is known before the coordinator releases
// remote leases.
let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
&& (commit_object_lock_guard.is_some()
|| commit_decommission_object_lock_guard.is_some()
|| commit_publication_guard.is_some())
let commit_owns_namespace_guard = commit_object_lock_guard.is_some()
|| commit_decommission_object_lock_guard.is_some()
|| commit_publication_guard.is_some();
let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
&& commit_owns_namespace_guard
&& commit_scanner_publication_scope.is_none();
// Full-tail callers also transfer owned guards to the coordinator:
// cancelling their ACK waiter must not cancel an in-flight rename.
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|| commit_allows_early_ack
|| commit_owns_namespace_guard
|| commit_bucket_lifecycle_guard.is_some()
|| quota_mutation_fence;
let commit_write_path_label = write_path.metric_label();
@@ -4452,7 +4458,8 @@ impl SetDisks {
write_quorum,
commit_scanner_publication_lease_tokens.as_ref(),
)
.with_publication_scope(commit_scanner_publication_scope.clone()),
.with_publication_scope(commit_scanner_publication_scope.clone())
.with_rollback_receipt(commit_rollback_receipt.clone()),
)
.await;
if let Some(scope) = commit_scanner_publication_scope.as_ref() {
@@ -4585,6 +4592,11 @@ impl SetDisks {
let rename_commit = match rename_result {
Ok(commit) => commit,
Err(err) => {
if commit_rollback_receipt.is_incomplete() {
// Incomplete undo retains the staging source and
// rollback backup for recovery; cleanup is unsafe.
return Err(err.into());
}
if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await {
warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data");
} else if issue3031_diag_enabled() {
@@ -4617,9 +4629,8 @@ impl SetDisks {
request.object_version_id = committed_version_id
.or_else(|| commit_version_suspended.then(Uuid::nil))
.map(|version_id| version_id.to_string());
tokio::spawn(async move {
let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await;
});
let heal_set = commit_set.clone();
tokio::spawn(async move { heal_set.submit_rename_tail_heal(request).await });
}
let rename_stage_elapsed = rename_stage_start.elapsed();
@@ -4885,7 +4896,7 @@ impl SetDisks {
);
}
});
} else {
} else if !rollback_receipt.is_incomplete() {
// Failure path (quorum loss / rollback): keep the cleanup inline so
// a failed PUT never returns while its tmp shards are still on disk
// (state-residue hardening tracked by backlog#864 / backlog#898).
@@ -17494,27 +17505,69 @@ mod put_object_tmp_cleanup_tests {
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn put_object_failure_cleans_tmp_workspace_inline() {
let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "tmp-clean-missing-bucket";
let object = "orphan-object";
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
let writer = Arc::clone(&set_disks);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("missing-bucket PUT must stage before rename");
let staged = non_trash_tmp_entries(&temp_dirs).await;
assert_eq!(staged.len(), 4, "every disk must have a staged workspace before rejection");
for workspace in staged {
let mut entries = tokio::fs::read_dir(&workspace)
.await
.expect("staged workspace should be readable");
let mut shards = 0;
while let Some(entry) = entries.next_entry().await.expect("staged data directory should be readable") {
if entry.file_type().await.expect("staged entry type").is_dir() {
let part = tokio::fs::metadata(entry.path().join("part.1"))
.await
.expect("staging must contain an actual erasure shard");
assert!(part.len() > 0, "the shard must be written before the missing-bucket failure");
shards += 1;
}
}
assert_eq!(shards, 1);
}
assert!(temp_dirs.iter().all(|dir| !dir.path().join(bucket).exists()));
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("missing-bucket PUT must finish")
.expect("PUT task should join")
.expect_err("put_object into a missing bucket volume must fail");
assert!(matches!(err, StorageError::VolumeNotFound), "original disk error expected: {err}");
// The bucket volume is never created, so the shards are written into
// the tmp workspace and the commit fails at rename_data with a quorum
// error — exercising the failure-path cleanup.
let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
let err = set_disks
.put_object("tmp-clean-missing-bucket", "orphan-object", &mut reader, &ObjectOptions::default())
.await
.expect_err("put_object into a missing bucket volume must fail");
// No polling: the failure path must clean the tmp workspace inline,
// before put_object returns (backlog#864 / backlog#898 hardening).
let leftovers = non_trash_tmp_entries(&temp_dirs).await;
assert!(
leftovers.is_empty(),
"failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
);
drop(temp_dirs);
// No polling: known pre-publication rejection must clean staging
// inline, before PUT returns (backlog#864 / backlog#898).
let leftovers = non_trash_tmp_entries(&temp_dirs).await;
assert!(
leftovers.is_empty(),
"failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
);
}
})
.await;
}
#[tokio::test]
@@ -18157,6 +18210,354 @@ mod put_object_tmp_cleanup_tests {
.await;
}
async fn make_completion_test_bucket(disks: &[DiskStore], bucket: &str) {
for disk in disks {
disk.make_volume(bucket)
.await
.expect("completion test bucket should be created");
}
}
/// Observe the actual metadata quorum while the remaining rename is parked.
/// A completed task count alone can race tasks that have not started yet.
async fn wait_for_paused_tail_metadata_quorum(disks: &[DiskStore], bucket: &str, object: &str) {
tokio::time::timeout(Duration::from_secs(30), async {
loop {
let mut committed = 0;
for disk in disks {
match disk.read_version("", bucket, object, "", &ReadOptions::default()).await {
Ok(_) => committed += 1,
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {}
Err(err) => panic!("unexpected metadata error while observing {bucket}/{object}: {err}"),
}
}
if committed == 3 {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("three disks must publish metadata while the fourth rename remains paused");
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for size in [4096, 1024 * 1024] {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-full-tail-cas";
let object = "full-tail-cas-object";
make_completion_test_bucket(&disks, bucket).await;
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; size]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("full-tail PUT must reach the rename barrier");
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
assert!(!put.is_finished(), "full-tail PUT must remain pending after metadata quorum");
let mut lock_probe = Box::pin(set.acquire_write_lock_diag("full_tail_probe", bucket, object));
assert!(
futures::poll!(lock_probe.as_mut()).is_pending(),
"the owned namespace guard must remain held"
);
barrier.release();
let written = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("full-tail PUT should finish after release")
.expect("full-tail PUT task should join")
.expect("full-tail PUT must commit");
assert_eq!(tasks.running(), 0, "full-tail response must follow every rename task");
drop(
tokio::time::timeout(Duration::from_secs(5), lock_probe)
.await
.expect("same-key lock should be available on return")
.expect("same-key lock probe should succeed"),
);
for disk in &disks {
disk.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("successful full-tail PUT must publish on every healthy disk");
}
drop(barrier);
let mut replacement = PutObjReader::from_vec(b"cas successor".to_vec());
set.put_object(
bucket,
object,
&mut replacement,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
http_preconditions: Some(HTTPPreconditions {
if_match: written.etag,
..Default::default()
}),
..Default::default()
},
)
.await
.expect("immediate same-key CAS must acquire the namespace guard");
let mut read = set
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("CAS successor must be immediately readable");
let mut body = Vec::new();
read.stream.read_to_end(&mut body).await.expect("successor body must drain");
assert_eq!(body, b"cas successor");
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_preserves_quorum_success_and_heals_failed_tail() {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-full-tail-heal";
let object = "full-tail-heal-object";
make_completion_test_bucket(&disks, bucket).await;
let mut heals = set.capture_test_rename_tail_heals();
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let _fault = rename_fault_injection::fail_rename_on(object, &[0]);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("failed tail must first reach the rename barrier");
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
assert!(!put.is_finished(), "committed quorum must still wait for the failing tail");
barrier.release();
tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("failed tail should drain")
.expect("PUT task should join")
.expect("a minority tail error must not negate committed quorum");
assert_eq!(tasks.running(), 0);
let heal = tokio::time::timeout(Duration::from_secs(30), heals.recv())
.await
.expect("failed tail must schedule heal")
.expect("heal capture must remain connected");
assert_eq!(heal.bucket, bucket);
assert_eq!(heal.object_prefix.as_deref(), Some(object));
let info = set
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("committed object must remain readable despite the failed tail");
assert_eq!(info.size, TEST_OBJECT_SIZE as i64);
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_rejects_quorum_minus_one() {
let (_dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-full-tail-no-quorum";
let object = "full-tail-no-quorum-object";
make_completion_test_bucket(&disks, bucket).await;
let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]);
let tasks = rename_fanout_barrier::observe_tasks(object);
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
let err = set
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect_err("draining two successful disks cannot satisfy write quorum three");
assert!(
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
"original quorum error expected: {err}"
);
assert_eq!(tasks.running(), 0, "failed fan-out and rollback must complete before return");
assert!(
set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(),
"failed fresh write must not become visible"
);
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() {
use crate::set_disk::core::io_primitives::rollback_fault_injection;
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
for fault in [
rollback_fault_injection::Fault::Io,
rollback_fault_injection::Fault::VolumeNotFoundAfterRename,
] {
let (dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-incomplete-undo";
let object = "incomplete-undo-object";
make_completion_test_bucket(&disks, bucket).await;
let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
set.put_object(
bucket,
object,
&mut old_reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("old generation should be completely committed");
wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await;
let old = disks[0]
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("old metadata must be readable");
let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory");
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
let _undo_fault = rollback_fault_injection::arm(object, 0, fault);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("overwrite must enter the actual rename fan-out before failure injection");
barrier.release();
let err = tokio::time::timeout(Duration::from_secs(30), put)
.await
.expect("incomplete undo must return without hanging")
.expect("PUT task should join")
.expect_err("two renamed disks cannot satisfy write quorum three");
assert!(
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
"original quorum error expected: {err}"
);
assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return");
let leftovers = non_trash_tmp_entries(&dirs).await;
assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery");
let backups = dirs
.iter()
.filter(|dir| {
dir.path()
.join(bucket)
.join(object)
.join(old_data_dir.to_string())
.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP)
.exists()
})
.count();
assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup");
// The remaining three disks still serve the old generation;
// the failed minority must never become an acknowledged write.
let mut read = set
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("old generation must remain readable after incomplete rollback");
let mut body = Vec::new();
read.stream
.read_to_end(&mut body)
.await
.expect("old generation should stream");
assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]);
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn tail_drained_put_owned_commit_survives_waiter_cancellation() {
let (dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = RUSTFS_META_BUCKET;
let object = "full-tail-cancelled-receipt";
// Internal config writes do not own a bucket lifecycle guard. The object
// guard alone must keep the full-tail coordinator alive after cancellation.
let tasks = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let writer = Arc::clone(&set);
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
writer
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("cancelled receipt must first reach the rename barrier");
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
put.abort();
assert!(put.await.expect_err("ACK waiter should cancel").is_cancelled());
let mut lock_probe = Box::pin(set.acquire_write_lock_diag("cancelled_full_tail_probe", bucket, object));
assert!(
futures::poll!(lock_probe.as_mut()).is_pending(),
"owned coordinator must retain the namespace guard after waiter cancellation"
);
barrier.release();
drop(
tokio::time::timeout(Duration::from_secs(30), lock_probe)
.await
.expect("cancelled coordinator must eventually release its guard")
.expect("post-commit lock probe should succeed"),
);
assert_eq!(tasks.running(), 0, "cancelled coordinator must reap every rename task");
for disk in &disks {
disk.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("caller cancellation must not interrupt committed receipt materialization");
}
wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn no_lock_put_waits_for_rename_tail_under_outer_guard() {
@@ -18184,6 +18585,7 @@ mod put_object_tmp_cleanup_tests {
&mut reader,
&ObjectOptions {
no_lock: true,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
@@ -18209,7 +18611,18 @@ mod put_object_tmp_cleanup_tests {
put.await
.expect("no-lock PUT task should join")
.expect("no-lock PUT should commit after the rename tail releases");
let mut lock_probe = Box::pin(set_disks.acquire_write_lock_diag("borrowed_full_tail_probe", bucket, object));
assert!(
futures::poll!(lock_probe.as_mut()).is_pending(),
"full-tail PUT must not release the caller's outer guard"
);
drop(outer_guard);
drop(
tokio::time::timeout(Duration::from_secs(5), lock_probe)
.await
.expect("outer owner releasing its guard should unblock the probe")
.expect("post-outer-guard probe should succeed"),
);
})
.await;
}
crates/ecstore/src/set_disk/transition_matrix_tests.rs
+3 -2
View File
@@ -18,6 +18,7 @@ use super::{
};
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
use crate::ecstore_validation_blackbox::make_local_set_disks;
use crate::object_api::WriteCompletion;
use crate::services::tier::test_util::register_mock_tier;
use crate::storage_api_contracts::bucket::BucketOperations;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
@@ -72,7 +73,7 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
object,
&mut reader,
&ObjectOptions {
no_lock: true,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
@@ -185,7 +186,7 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot
object,
&mut reader,
&ObjectOptions {
no_lock: true,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
crates/ecstore/src/store/init.rs
+63 -27
View File
@@ -2979,6 +2979,33 @@ mod tests {
#[cfg(feature = "test-util")]
const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object";
fn decommission_retry_fault_hook(
bucket: &str,
object: &str,
faults: Arc<AtomicUsize>,
) -> crate::core::pools::DecommissionTestFaultDecision {
let target_bucket = bucket.to_string();
let target_object = object.to_string();
Arc::new(move |stage, bucket, object, _attempt, succeeded| {
if !succeeded
|| stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
|| bucket != target_bucket
|| object != target_object
{
return false;
}
// Entry retries reset the local attempt; real copy errors can skip
// successful attempts. Only injected faults spend this global budget.
faults
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
.then_some(faults.saturating_add(1))
})
.is_ok()
})
}
async fn seed_decommission_source(
store: &Arc<crate::store::ECStore>,
bucket: &str,
@@ -5120,6 +5147,33 @@ mod tests {
shutdown.cancel();
}
#[test]
fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
for attempts in [[1, 2, 3], [1, 1, 2], [1, 3, 3]] {
let faults = Arc::new(AtomicUsize::new(0));
let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
for (stage, bucket, object, succeeded) in [
("other-stage", "bucket", "object", true),
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true),
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true),
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false),
] {
assert!(!hook(stage, bucket, object, 1, succeeded));
}
assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
for (index, attempt) in attempts.into_iter().enumerate() {
assert_eq!(
hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, true),
index < 2,
"attempts={attempts:?}, index={index}"
);
}
assert_eq!(faults.load(Ordering::SeqCst), 2, "attempts={attempts:?}");
}
}
#[test]
#[serial_test::serial(storage_class_env)]
fn decommission_entry_retries_source_changed_without_canceling_other_bucket() {
@@ -5214,31 +5268,8 @@ mod tests {
));
let ordinary_faults = Arc::new(AtomicUsize::new(0));
let ordinary_faults_for_hook = Arc::clone(&ordinary_faults);
let fault_bucket = other_bucket.clone();
let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
move |stage, bucket, object, attempt, succeeded| {
let candidate = succeeded
&& stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
&& bucket == fault_bucket.as_str()
&& object == other_object;
if !candidate {
return false;
}
// Keep the fault budget global across any
// entry-level re-list; its inner attempt counter
// restarts after SourceChanged.
ordinary_faults_for_hook
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
let next_fault = faults.saturating_add(1);
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1)
&& attempt == next_fault)
.then_some(next_fault)
})
.is_ok()
},
));
let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults));
let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook);
let rx = CancellationToken::new();
let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
@@ -8045,10 +8076,15 @@ mod tests {
);
assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone())
let full_tail = ObjectOptions {
max_parity: true,
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
};
com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail)
.await
.expect("second page receipt should restore");
com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec())
com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail)
.await
.expect("second page receipt should corrupt deterministically");
let corrupt = store
crates/heal/src/heal/manager.rs
+60 -12
View File
@@ -45,6 +45,11 @@ use tracing::{debug, error, info, warn};
use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
// Each cache includes alias tokens in its count and byte budget. Eviction
// removes every token sharing a snapshot; neither cache retains repair state.
const MAX_COMPLETED_HEAL_TOKENS: usize = 1024;
const MAX_COMPLETED_HEAL_BYTES: usize = 64 * 1024 * 1024;
const MAX_COMPLETED_HEAL_RESULT_BYTES: usize = 1024 * 1024;
const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again";
const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
@@ -180,6 +185,8 @@ fn record_displaced_terminal(
request: &HealRequest,
) -> Arc<CompletedHealStatus> {
let terminal = Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(),
status: HealTaskStatus::Failed {
error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"),
@@ -193,6 +200,7 @@ fn record_displaced_terminal(
let mut terminals = lock_displaced_terminals(registry);
prune_completed_heal_statuses(&mut terminals);
terminals.insert(request.id.clone(), Arc::clone(&terminal));
prune_completed_heal_statuses(&mut terminals);
terminal
}
@@ -209,9 +217,15 @@ async fn remove_displaced_task_aliases(
.collect::<Vec<_>>();
let mut displaced_terminals = lock_displaced_terminals(terminals);
prune_completed_heal_statuses(&mut displaced_terminals);
for alias_id in alias_ids {
displaced_terminals.insert(alias_id, Arc::clone(terminal));
if displaced_terminals
.get(task_id)
.is_some_and(|current| Arc::ptr_eq(current, terminal))
{
for alias_id in alias_ids {
displaced_terminals.insert(alias_id, Arc::clone(terminal));
}
}
prune_completed_heal_statuses(&mut displaced_terminals);
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
}
@@ -222,6 +236,36 @@ async fn remove_task_aliases_for_task(registry: &Arc<Mutex<HashMap<String, HealT
.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
}
// Callers hold active ownership until publication. Lock order is active ->
// retrying (when needed) -> aliases -> completed; queries release aliases
// before looking up active state. Publishing aliases before removing their
// mapping keeps both an already-resolved token and a new lookup valid.
async fn publish_completed_heal(
completed_heals: &Mutex<HashMap<String, Arc<CompletedHealStatus>>>,
task_aliases: &Mutex<HashMap<String, HealTaskAlias>>,
task_id: &str,
completed: CompletedHealStatus,
terminal: bool,
) {
let completed = Arc::new(completed);
completed.retained_bytes();
let mut aliases = task_aliases.lock().await;
let mut retained = completed_heals.lock().await;
if let Some(previous) = retained.get(task_id).cloned() {
for entry in retained.values_mut().filter(|entry| Arc::ptr_eq(entry, &previous)) {
*entry = Arc::clone(&completed);
}
}
retained.insert(task_id.to_owned(), Arc::clone(&completed));
if terminal {
for (alias_id, _) in aliases.iter().filter(|(_, alias)| alias.task_id == task_id) {
retained.insert(alias_id.clone(), Arc::clone(&completed));
}
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
}
prune_completed_heal_statuses(&mut retained);
}
#[derive(Debug, Clone)]
pub struct HealTaskReport {
pub status: HealTaskStatus,
@@ -268,7 +312,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
let result_items = match since {
None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(),
Some(cursor) => {
if cursor + 1 < completed.min_seq {
if cursor.saturating_add(1) < completed.min_seq {
lagged = true;
}
completed
@@ -283,7 +327,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
status: completed.status.clone(),
result_items,
result_items_truncated: completed.result_items_truncated || lagged,
progress: None,
progress: completed.progress.clone(),
next_seq: completed.next_seq,
min_seq: completed.min_seq,
}
@@ -1847,14 +1891,14 @@ impl HealManager {
pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
let canonical_task_id = self.canonical_task_id(task_id).await;
let active_heals = self.active_heals.lock().await;
if let Some(task) = active_heals.get(&canonical_task_id) {
Ok(task.get_progress().await)
} else {
Err(Error::TaskNotFound {
task_id: task_id.to_string(),
})
}
let progress = match self.lookup_task_state(&canonical_task_id, None).await {
TaskStateLookup::Active(task) => Some(task.get_progress().await),
TaskStateLookup::Completed(completed) => completed.progress.clone(),
_ => None,
};
progress.ok_or_else(|| Error::TaskNotFound {
task_id: task_id.to_string(),
})
}
/// Cancel task
@@ -1864,6 +1908,8 @@ impl HealManager {
let mut active_heals = self.active_heals.lock().await;
if let Some(task) = active_heals.get(&canonical_task_id) {
task.cancel().await?;
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await;
active_heals.remove(&canonical_task_id);
publish_active_heal_count(&active_heals);
info!(
@@ -1940,6 +1986,8 @@ impl HealManager {
for task_id in &task_ids {
if let Some(task) = active_heals.get(task_id) {
task.cancel().await?;
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await;
}
active_heals.remove(task_id);
cancelled += 1;
crates/heal/src/heal/manager/queue.rs
+129
View File
@@ -82,6 +82,8 @@ pub(super) enum QueuePushOutcome {
pub(super) struct CompletedHealStatus {
pub(super) heal_type: HealType,
pub(super) status: HealTaskStatus,
pub(super) progress: Option<HealProgress>,
pub(super) retained_bytes: std::sync::OnceLock<usize>,
pub(super) result_items_truncated: bool,
pub(super) completed_at: SystemTime,
/// Sequence-stamped retained window, archived with the completion so
@@ -92,6 +94,133 @@ pub(super) struct CompletedHealStatus {
pub(super) min_seq: u64,
}
impl CompletedHealStatus {
// Account for owned capacities, including nested drive arrays. Aliases
// conservatively charge the shared allocation again, keeping both token
// count and retained payload bounded without a second ownership index.
pub(super) fn retained_bytes(&self) -> usize {
*self.retained_bytes.get_or_init(|| self.measure_retained_bytes())
}
fn measure_retained_bytes(&self) -> usize {
let mut bytes = size_of::<Self>();
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
match &self.heal_type {
HealType::Cluster => {}
HealType::Bucket { bucket } => add(bucket.capacity()),
HealType::Object {
bucket,
object,
version_id,
}
| HealType::ECDecode {
bucket,
object,
version_id,
} => {
add(bucket.capacity());
add(object.capacity());
add(version_id.as_ref().map_or(0, String::capacity));
}
HealType::Prefix { bucket, prefix } => {
add(bucket.capacity());
add(prefix.capacity());
}
HealType::Metadata { bucket, object } => {
add(bucket.capacity());
add(object.capacity());
}
HealType::ErasureSet { buckets, set_disk_id } => {
add(buckets.capacity().saturating_mul(size_of::<String>()));
for bucket in buckets {
add(bucket.capacity());
}
add(set_disk_id.capacity());
}
}
if let HealTaskStatus::Failed { error } | HealTaskStatus::Retrying { error, .. } = &self.status {
add(error.capacity());
}
add(self
.progress
.as_ref()
.and_then(|progress| progress.current_object.as_ref())
.map_or(0, String::capacity));
add(self.seqed_items.capacity().saturating_mul(size_of::<(u64, HealResultItem)>()));
for (_, item) in &self.seqed_items {
add(Self::result_item_heap_bytes(item));
}
bytes
}
fn result_item_heap_bytes(item: &HealResultItem) -> usize {
let mut bytes = 0usize;
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
for value in [
&item.heal_item_type,
&item.bucket,
&item.object,
&item.version_id,
&item.detail,
] {
add(value.capacity());
}
for infos in [&item.before, &item.after] {
add(infos
.drives
.capacity()
.saturating_mul(size_of::<rustfs_madmin::heal_commands::HealDriveInfo>()));
for drive in &infos.drives {
add(drive.uuid.capacity());
add(drive.endpoint.capacity());
add(drive.state.capacity());
}
}
bytes
}
pub(super) fn bound_result_window(&mut self) {
let mut bytes = 0usize;
let retained = self
.seqed_items
.iter()
.rev()
.take_while(|(_, item)| {
bytes = bytes
.saturating_add(size_of::<(u64, HealResultItem)>())
.saturating_add(Self::result_item_heap_bytes(item));
bytes <= MAX_COMPLETED_HEAL_RESULT_BYTES
})
.count();
let truncated = retained < self.seqed_items.len();
if truncated {
self.seqed_items.drain(..self.seqed_items.len() - retained);
self.seqed_items.shrink_to_fit();
self.min_seq = self.seqed_items.first().map_or(self.next_seq, |(seq, _)| *seq);
self.result_items_truncated = true;
self.retained_bytes.take();
}
}
pub(super) async fn snapshot(task: &HealTask, status: HealTaskStatus) -> Self {
let seqed_items = task.get_seqed_result_items().await;
let (next_seq, min_seq) = task.result_seq_cursors();
let mut snapshot = Self {
heal_type: task.heal_type.clone(),
status,
progress: Some(task.get_progress().await),
retained_bytes: std::sync::OnceLock::new(),
result_items_truncated: task.result_items_truncated(),
completed_at: SystemTime::now(),
seqed_items,
next_seq,
min_seq,
};
snapshot.bound_result_window();
snapshot
}
}
#[derive(Debug, Clone)]
pub(super) struct HealTaskAlias {
pub(super) task_id: String,
crates/heal/src/heal/manager/scheduler.rs
+75 -39
View File
@@ -264,7 +264,7 @@ impl HealManager {
error: error.clone(),
retry_attempt: request.retry_attempts,
});
let retry_request_for_queue = retry_request;
let mut retry_request_for_queue = retry_request;
let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new());
if retry_request_for_queue.is_none() {
replacement_recovery_anchors_clone
@@ -272,7 +272,35 @@ impl HealManager {
.unwrap_or_else(|poisoned| poisoned.into_inner())
.remove(&task_id);
}
let mut completed_status = match retry_request_for_status {
Some(status) => status,
None => task.get_status().await,
};
let mut completed_status_entry = CompletedHealStatus::snapshot(&task, completed_status.clone()).await;
let completed_progress = task.get_progress().await;
#[cfg(test)]
tests::pause_completed_retention_before_publish(&task_id, &completed_status).await;
let mut active_heals_guard = active_heals_clone.lock().await;
let owns_completion = active_heals_guard.contains_key(&task_id);
let cancelled_completion = if owns_completion {
false
} else {
// Cancellation can win while a finished worker waits
// for active ownership. It must not resurrect a retry
// or replace an acknowledged cancellation with success.
retry_request_for_queue = None;
completed_heals_clone
.lock()
.await
.get(&task_id)
.is_some_and(|completed| completed.status == HealTaskStatus::Cancelled)
};
if cancelled_completion {
completed_status = HealTaskStatus::Cancelled;
completed_status_entry.status = HealTaskStatus::Cancelled;
}
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
// Keep retry ownership continuous: status snapshots acquire
// these locks in the same active -> retrying order.
let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
@@ -295,6 +323,16 @@ impl HealManager {
} else {
None
};
if owns_completion || cancelled_completion {
publish_completed_heal(
&completed_heals_clone,
&task_aliases_clone,
&task_id,
completed_status_entry,
terminal_completion,
)
.await;
}
let completed_task = active_heals_guard.remove(&task_id);
if let Some(completed_task) = completed_task.as_ref() {
publish_active_heal_count(&active_heals_guard);
@@ -304,33 +342,10 @@ impl HealManager {
drop(retrying_heals_guard.take());
drop(active_heals_guard);
if let Some(completed_task) = completed_task {
let completed_status = if let Some(status) = retry_request_for_status {
status
} else {
completed_task.get_status().await
};
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
let completed_progress = completed_task.get_progress().await;
// Single snapshot of the retained window: the task is
// finished and already off the active map, so there is
// no concurrent writer to race with.
let seqed_items = completed_task.get_seqed_result_items().await;
let (next_seq, min_seq) = completed_task.result_seq_cursors();
let completed_status_entry = CompletedHealStatus {
heal_type: completed_task.heal_type.clone(),
status: completed_status.clone(),
result_items_truncated: completed_task.result_items_truncated(),
completed_at: SystemTime::now(),
seqed_items,
next_seq,
min_seq,
};
let mut completed_heals_guard = completed_heals_clone.lock().await;
prune_completed_heal_statuses(&mut completed_heals_guard);
completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry));
drop(completed_heals_guard);
#[cfg(test)]
tests::pause_completed_retention_handoff(&task_id).await;
if completed_task.is_some() {
// update statistics
let mut stats = statistics_clone.write().await;
match completed_status {
@@ -352,10 +367,6 @@ impl HealManager {
} else {
release_mrf_repair_notice_targets(notice_targets);
}
task_aliases_clone
.lock()
.await
.retain(|alias_id, alias| alias_id != &task_id && alias.task_id != task_id);
}
}
@@ -718,17 +729,42 @@ pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
}
pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>) {
let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else {
return;
};
prune_completed_heal_statuses_at(completed_heals, SystemTime::now());
}
pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>, now: SystemTime) {
completed_heals.retain(|_, completed| {
completed
.completed_at
.duration_since(SystemTime::UNIX_EPOCH)
.map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION)
now.duration_since(completed.completed_at)
.map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
.unwrap_or(false)
});
let entry_bytes = |key: &String, value: &Arc<CompletedHealStatus>| {
key.capacity()
.saturating_add(size_of::<(String, Arc<CompletedHealStatus>)>())
.saturating_add(value.retained_bytes())
};
let mut bytes = completed_heals
.iter()
.fold(0usize, |total, (key, value)| total.saturating_add(entry_bytes(key, value)));
while completed_heals.len() > MAX_COMPLETED_HEAL_TOKENS || bytes > MAX_COMPLETED_HEAL_BYTES {
let Some(oldest) = completed_heals
.iter()
.min_by(|(left_id, left), (right_id, right)| {
left.completed_at.cmp(&right.completed_at).then_with(|| left_id.cmp(right_id))
})
.map(|(_, value)| Arc::clone(value))
else {
break;
};
completed_heals.retain(|key, value| {
if Arc::ptr_eq(value, &oldest) {
bytes = bytes.saturating_sub(entry_bytes(key, value));
false
} else {
true
}
});
}
}
pub(super) fn can_schedule_request(
crates/heal/src/heal/manager/tests.rs
+348 -3
View File
@@ -101,6 +101,326 @@ async fn process_manager_queue_once(manager: &HealManager) {
struct MockStorage;
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
CompletedHealStatus {
heal_type: HealType::Cluster,
status: HealTaskStatus::Completed,
progress: Some(HealProgress {
objects_scanned: 9,
objects_healed: 8,
objects_failed: 1,
..Default::default()
}),
retained_bytes: std::sync::OnceLock::new(),
result_items_truncated: false,
completed_at,
seqed_items: vec![(3, HealResultItem::default()), (4, HealResultItem::default())],
next_seq: 5,
min_seq: 3,
}
}
#[test]
fn completed_retention_cursor_boundaries_preserve_progress() {
let completed = completed_retention_fixture(SystemTime::now());
for (cursor, count, lagged) in [
(0, 2, true),
(1, 2, true),
(2, 2, false),
(3, 1, false),
(4, 0, false),
(5, 0, false),
(u64::MAX, 0, false),
] {
let report = completed_task_report(&completed, Some(cursor));
assert_eq!(report.result_items.len(), count, "cursor={cursor}");
assert_eq!(report.result_items_truncated, lagged, "cursor={cursor}");
assert_eq!(report.progress, completed.progress);
assert_eq!((report.next_seq, report.min_seq), (5, 3));
}
assert_eq!(completed_task_report(&completed, None).result_items.len(), 2);
}
#[tokio::test]
async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() {
let manager = HealManager::new(Arc::new(MockStorage), None);
let request = HealRequest::bucket("bucket".to_string());
manager.insert_task_alias("alias", &request.id).await;
let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
lock_displaced_terminals(&manager.displaced_terminals).remove(&request.id);
remove_displaced_task_aliases(&manager.task_aliases, &manager.displaced_terminals, &request.id, &terminal).await;
for token in [&request.id, &"alias".to_string()] {
assert!(matches!(manager.get_task_report(token).await, Err(Error::TaskNotFound { .. })));
}
assert!(manager.task_aliases.lock().await.is_empty());
assert!(lock_displaced_terminals(&manager.displaced_terminals).is_empty());
}
#[test]
fn completed_retention_count_ttl_and_alias_eviction_are_bounded() {
let now = SystemTime::now();
let mut entries = HashMap::new();
let oldest = Arc::new(completed_retention_fixture(now - KEEP_HEAL_TASK_STATUS_DURATION));
entries.insert("oldest".to_string(), Arc::clone(&oldest));
entries.insert("oldest-alias".to_string(), Arc::clone(&oldest));
for index in 2..MAX_COMPLETED_HEAL_TOKENS {
entries.insert(format!("task-{index}"), Arc::new(completed_retention_fixture(now)));
}
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS);
entries.insert("cap-plus-one".to_string(), Arc::new(completed_retention_fixture(now)));
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1);
assert!(!entries.contains_key("oldest"));
assert!(!entries.contains_key("oldest-alias"));
entries.clear();
entries.insert("ttl-boundary".to_string(), oldest);
entries.insert(
"expired".to_string(),
Arc::new(completed_retention_fixture(
now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1),
)),
);
entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1))));
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), 1);
assert!(entries.contains_key("ttl-boundary"));
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1));
assert!(entries.is_empty());
}
#[test]
fn completed_retention_total_byte_cap_and_cap_plus_one() {
let now = SystemTime::now();
let key = "large".to_string();
let mut entry = completed_retention_fixture(now);
let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
entry.retained_bytes.take();
entry.status = HealTaskStatus::Failed {
error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
};
assert_eq!(
entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
MAX_COMPLETED_HEAL_BYTES
);
let mut entries = HashMap::from([(key, Arc::new(entry))]);
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), 1, "exact byte cap remains retained");
let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
over.retained_bytes.take();
if let HealTaskStatus::Failed { error } = &mut over.status {
*error = "x".repeat(error.len() + 1);
}
entries.insert("large".to_string(), Arc::new(over));
prune_completed_heal_statuses_at(&mut entries, now);
assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
}
#[tokio::test]
async fn completed_retention_large_window_keeps_cursors_and_progress() {
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), Arc::new(MockStorage));
let mut snapshot = completed_retention_fixture(SystemTime::now());
snapshot.seqed_items[0].1.detail = "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES);
snapshot.bound_result_window();
assert_eq!(snapshot.seqed_items.len(), 1);
assert_eq!((snapshot.min_seq, snapshot.next_seq), (4, 5));
assert!(snapshot.result_items_truncated);
assert!(snapshot.retained_bytes() < MAX_COMPLETED_HEAL_RESULT_BYTES);
let report = completed_task_report(&snapshot, Some(0));
assert_eq!(report.progress.expect("progress retained").objects_scanned, 9);
assert!(report.result_items_truncated);
let active_max = task.get_result_items_since(Some(u64::MAX)).await;
assert!(active_max.items.is_empty());
assert!(!active_max.lagged);
}
#[test]
fn completed_retention_result_byte_cap_and_cap_plus_one() {
for extra in [0, 1] {
let mut snapshot = completed_retention_fixture(SystemTime::now());
snapshot.seqed_items = vec![(
4,
HealResultItem {
detail: "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES - size_of::<(u64, HealResultItem)>() + extra),
..Default::default()
},
)];
snapshot.min_seq = 4;
snapshot.bound_result_window();
assert_eq!(snapshot.seqed_items.len(), 1 - extra);
assert_eq!(snapshot.result_items_truncated, extra == 1);
assert_eq!(snapshot.min_seq, if extra == 0 { 4 } else { 5 });
assert_eq!(snapshot.next_seq, 5);
assert_eq!(snapshot.progress.as_ref().expect("progress retained").objects_scanned, 9);
}
}
#[derive(Default)]
struct CompletedRetentionHook {
started: Notify,
execute: Notify,
handoff: Notify,
finish: Notify,
pause_before_publish: bool,
before_publish: Notify,
publish: Notify,
prepared_status: Mutex<Option<HealTaskStatus>>,
}
static COMPLETED_RETENTION_HOOKS: LazyLock<Mutex<HashMap<String, Arc<CompletedRetentionHook>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
pub(super) async fn pause_completed_retention_handoff(task_id: &str) {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
if let Some(hook) = hook {
hook.handoff.notify_one();
hook.finish.notified().await;
}
}
pub(super) async fn pause_completed_retention_before_publish(task_id: &str, status: &HealTaskStatus) {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
if let Some(hook) = hook.filter(|hook| hook.pause_before_publish) {
*hook.prepared_status.lock().await = Some(status.clone());
hook.before_publish.notify_one();
hook.publish.notified().await;
}
}
#[tokio::test]
async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
let bucket = "completed-retention-retry-cancel";
let manager = HealManager::new(Arc::new(MockStorage), None);
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
let task_id = request.id.clone();
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
let alias = duplicate.id.clone();
let hook = Arc::new(CompletedRetentionHook {
pause_before_publish: true,
..Default::default()
});
{
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
hooks.insert(bucket.to_string(), Arc::clone(&hook));
hooks.insert(task_id.clone(), Arc::clone(&hook));
}
manager.submit_heal_request(request).await.expect("admit original");
manager.submit_heal_request(duplicate).await.expect("admit alias");
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
.await
.expect("scheduler starts");
let task = manager.active_heals.lock().await.get(&task_id).cloned().expect("active task");
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
hook.execute.notify_one();
tokio::time::timeout(Duration::from_secs(5), hook.before_publish.notified())
.await
.expect("retry snapshot prepared");
manager.cancel_task(&alias).await.expect("cancel wins active ownership");
assert!(matches!(*hook.prepared_status.lock().await, Some(HealTaskStatus::Retrying { .. })));
hook.publish.notify_one();
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
.await
.expect("scheduler finishes handoff");
for token in [&task_id, &alias] {
let report = manager.get_task_report(token).await.expect("cancelled token retained");
assert_eq!(report.status, HealTaskStatus::Cancelled);
assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
}
assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id));
hook.finish.notify_one();
COMPLETED_RETENTION_HOOKS
.lock()
.await
.retain(|key, _| key != bucket && key != &task_id);
}
#[tokio::test]
async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_handoff() {
for outcome in ["success", "failed", "cancelled"] {
let bucket = format!("completed-retention-{outcome}");
let hook = Arc::new(CompletedRetentionHook::default());
let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None));
let request = HealRequest::object(bucket.clone(), "object".to_string(), None);
let task_id = request.id.clone();
let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None);
let alias = duplicate.id.clone();
{
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
hooks.insert(bucket.clone(), Arc::clone(&hook));
hooks.insert(task_id.clone(), Arc::clone(&hook));
}
manager.submit_heal_request(request).await.expect("admit original");
manager.submit_heal_request(duplicate).await.expect("admit alias");
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
.await
.expect("scheduler reaches storage");
let task = manager
.active_heals
.lock()
.await
.get(&task_id)
.cloned()
.expect("task is active");
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
let before = manager.get_task_report(&alias).await.expect("alias resolves active progress");
assert_eq!(before.progress.as_ref().expect("active progress").objects_scanned, 1);
let poll_manager = Arc::clone(&manager);
let poll_alias = alias.clone();
let stop = CancellationToken::new();
let poll_stop = stop.clone();
let polling = tokio::spawn(async move {
while !poll_stop.is_cancelled() {
let report = poll_manager
.get_task_report(&poll_alias)
.await
.expect("handoff must never return NotFound");
assert!(report.progress.expect("progress never disappears").objects_scanned >= 1);
tokio::task::yield_now().await;
}
});
if outcome == "cancelled" {
manager.cancel_task(&alias).await.expect("cancel active task by alias");
} else {
hook.execute.notify_one();
}
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
.await
.expect("scheduler archives terminal");
assert!(!manager.active_heals.lock().await.contains_key(&task_id));
let expected = task.get_progress().await;
for token in [&task_id, &alias] {
assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
let report = manager
.get_task_report_for_path_since(&format!("{bucket}/object"), token, Some(u64::MAX))
.await
.expect("terminal token remains queryable at handoff");
assert_eq!(report.progress.as_ref(), Some(&expected));
assert!(report.result_items.is_empty());
match outcome {
"success" => assert_eq!(report.status, HealTaskStatus::Completed),
"failed" => assert!(matches!(report.status, HealTaskStatus::Failed { .. })),
_ => assert_eq!(report.status, HealTaskStatus::Cancelled),
}
}
let retained = manager.completed_heals.lock().await;
assert!(Arc::ptr_eq(&retained[&task_id], &retained[&alias]));
drop(retained);
stop.cancel();
polling.await.expect("concurrent polling succeeds");
// Archived progress must not alias a mutable live progress object.
task.progress.write().await.objects_scanned = 999;
assert_eq!(manager.get_task_report(&alias).await.expect("frozen report").progress, Some(expected));
hook.finish.notify_one();
COMPLETED_RETENTION_HOOKS
.lock()
.await
.retain(|key, _| key != &bucket && key != &task_id);
}
}
#[async_trait::async_trait]
impl HealStorageAPI for MockStorage {
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> {
@@ -123,6 +443,12 @@ impl HealStorageAPI for MockStorage {
}
async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(bucket).cloned();
if let Some(hook) = hook {
hook.started.notify_one();
hook.execute.notified().await;
return Ok(true);
}
Ok(bucket == "retry-transition")
}
@@ -133,13 +459,18 @@ impl HealStorageAPI for MockStorage {
_version_id: Option<&str>,
_opts: &HealOpts,
) -> Result<(HealResultItem, Option<Error>)> {
if bucket == "completed-retention-failed" {
return Err(Error::TaskExecutionFailed {
message: "retention fixture failure".to_string(),
});
}
if let Some(hook) = manager_recovery_test_hook() {
*hook
.heal_object_calls
.lock()
.expect("manager recovery object call lock should not poison") += 1;
}
if bucket == "retry-transition" {
if matches!(bucket, "retry-transition" | "completed-retention-retry-cancel") {
return Ok((
HealResultItem::default(),
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
@@ -1145,7 +1476,13 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
.expect("duplicate token should cancel merged active task");
assert!(manager.active_heals.lock().await.get(&active_task_id).is_none());
assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. })));
assert_eq!(
manager
.get_task_status(&active_task_id)
.await
.expect("cancelled task remains queryable"),
HealTaskStatus::Cancelled
);
}
#[tokio::test]
@@ -1638,6 +1975,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
manager.completed_heals.lock().await.insert(
task_id,
Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type,
status: HealTaskStatus::Retrying {
error: "Lock acquisition timeout".to_string(),
@@ -2053,7 +2392,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() {
"the overlapping admin task must be cancelled (removed from the active table) before the new one starts"
);
assert!(
matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })),
matches!(manager.get_task_status(&old_id).await, Ok(HealTaskStatus::Cancelled)),
"a cancelled task must no longer resolve as an active heal"
);
}
@@ -2360,6 +2699,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
manager.completed_heals.lock().await.insert(
task_id.clone(),
Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(),
status: HealTaskStatus::Retrying {
error: "transient disk failure".to_string(),
@@ -2395,6 +2736,8 @@ async fn test_get_task_status_reads_recent_completed_status() {
manager.completed_heals.lock().await.insert(
"completed-token".to_string(),
Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Bucket {
bucket: "bucket".to_string(),
},
@@ -2424,6 +2767,8 @@ async fn test_get_task_report_for_path_reads_completed_items() {
manager.completed_heals.lock().await.insert(
"completed-token".to_string(),
Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Object {
bucket: "bucket".to_string(),
object: "object".to_string(),
crates/heal/src/heal/task.rs
+1 -1
View File
@@ -999,7 +999,7 @@ impl HealTask {
let items = match since {
None => result_items.iter().map(|(_, item)| item.clone()).collect::<Vec<_>>(),
Some(cursor) => {
if cursor + 1 < min_seq {
if cursor.saturating_add(1) < min_seq {
lagged = true;
}
result_items
crates/madmin/src/client.rs
+72
View File
@@ -182,6 +182,9 @@ pub struct BackgroundHealStatus {
pub heal_active_tasks: u64,
#[serde(default)]
pub cluster_status_complete: bool,
/// Missing on older servers; absent coverage or counts mean unknown.
#[serde(default)]
pub coverage: Option<BackgroundHealCoverage>,
#[serde(default)]
pub progress: Option<serde_json::Value>,
/// Remaining wire fields (flattened `BackgroundHealInfo` plus the
@@ -190,6 +193,22 @@ pub struct BackgroundHealStatus {
pub extra: serde_json::Map<String, serde_json::Value>,
}
/// Node coverage of a background heal status snapshot. Counters describe only
/// nodes with usable snapshots; unknown peers may still be running heal work.
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BackgroundHealCoverage {
#[serde(default)]
pub expected: Option<usize>,
#[serde(default)]
pub responded: Option<usize>,
#[serde(default)]
pub unknown: Option<usize>,
/// Stable reason codes; unknown future codes are preserved verbatim.
#[serde(default)]
pub reasons: Vec<String>,
}
/// `GET /v3/scanner/status` response, typed at the fields operators branch
/// on; everything else passes through verbatim.
#[derive(Debug, Clone, Deserialize)]
@@ -630,9 +649,39 @@ mod tests {
assert_eq!(status.state, "active");
assert_eq!(status.heal_queue_length, 3);
assert!(status.cluster_status_complete);
assert!(status.coverage.is_none(), "legacy payloads have unknown coverage");
assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through");
}
#[test]
fn background_heal_status_missing_coverage_fields_remain_unknown() {
for raw in [json!({"state": "degraded"}), json!({"state": "degraded", "coverage": {}})] {
let status: BackgroundHealStatus = serde_json::from_value(raw).expect("partial legacy payload decodes");
assert!(!status.cluster_status_complete);
if let Some(coverage) = status.coverage {
assert_eq!(coverage.expected, None);
assert_eq!(coverage.responded, None);
assert_eq!(coverage.unknown, None);
}
}
}
#[test]
fn background_heal_status_preserves_future_fields_and_reasons() {
let raw = json!({
"state": "degraded", "clusterStatusComplete": false,
"coverage": {"expected": 3, "responded": 1, "unknown": 2, "reasons": ["future_reason"], "futureCoverage": true},
"futureStatus": {"value": 7}
});
let status: BackgroundHealStatus = serde_json::from_value(raw).expect("future additive fields decode");
assert_eq!(status.extra["futureStatus"]["value"], 7);
let coverage = status.coverage.expect("coverage supplied");
assert_eq!(coverage.expected, Some(3));
assert_eq!(coverage.responded, Some(1));
assert_eq!(coverage.unknown, Some(2));
assert_eq!(coverage.reasons, ["future_reason"]);
}
#[test]
fn scanner_status_defaults_freshness_to_unknown() {
let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}});
@@ -721,6 +770,7 @@ mod tests {
let status = client.background_heal_status().await.expect("status decodes");
assert_eq!(status.state, "idle");
assert!(status.coverage.is_none(), "older HTTP responses retain unknown coverage");
let request = server.recorded();
// The server registers this route POST-only; a GET here answers 405.
assert_eq!(request.method, "POST");
@@ -728,6 +778,28 @@ mod tests {
assert_eq!(request.query, "");
}
#[tokio::test]
async fn background_heal_status_decodes_partial_coverage_over_http() {
let body = r#"{"state":"degraded","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":false,"coverage":{"expected":3,"responded":1,"unknown":2,"reasons":["notification_system_unavailable"]},"futureStatus":true}"#;
let server = TestServer::spawn(body, 200).await;
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").expect("client builds");
let status = client
.background_heal_status()
.await
.expect("partial status is a successful response");
assert_eq!(status.state, "degraded");
assert!(!status.cluster_status_complete);
assert_eq!(status.extra["futureStatus"], true);
let coverage = status.coverage.expect("partial coverage supplied");
assert_eq!(coverage.expected, Some(3));
assert_eq!(coverage.responded, Some(1));
assert_eq!(coverage.unknown, Some(2));
assert_eq!(coverage.reasons, ["notification_system_unavailable"]);
let request = server.recorded();
assert_eq!(request.method, "POST");
assert_eq!(request.query, "", "reading status must not send heal control parameters");
}
#[tokio::test]
async fn http_error_status_maps_to_a_typed_error_with_body() {
let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await;
crates/protos/src/generated/proto_gen/node_service.rs
+286
View File
@@ -1283,6 +1283,54 @@ pub struct ScannerDirtyUsageSnapshotResponse {
#[prost(bytes = "bytes", tag = "7")]
pub response_proof: ::prost::bytes::Bytes,
}
/// Receiver-only protocol. Producers must retain whole-cycle ACK until they
/// have a durable per-bucket publication proof.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerScopedDirtyUsageEntry {
#[prost(string, tag = "1")]
pub bucket: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "2")]
pub bucket_incarnation: ::prost::bytes::Bytes,
#[prost(uint64, tag = "3")]
pub generation: u64,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ScannerScopedDirtyUsageAckRequest {
#[prost(bytes = "bytes", tag = "1")]
pub challenge: ::prost::bytes::Bytes,
#[prost(uint32, tag = "2")]
pub protocol_version: u32,
#[prost(string, tag = "3")]
pub owner_id: ::prost::alloc::string::String,
#[prost(string, tag = "4")]
pub instance_id: ::prost::alloc::string::String,
/// Only scope 1 (a complete bucket) is supported; zero is invalid.
#[prost(uint32, tag = "5")]
pub scope: u32,
#[prost(bool, tag = "6")]
pub probe_only: bool,
#[prost(message, repeated, tag = "7")]
pub entries: ::prost::alloc::vec::Vec<ScannerScopedDirtyUsageEntry>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerScopedDirtyUsageAckResponse {
#[prost(uint32, tag = "1")]
pub protocol_version: u32,
#[prost(string, tag = "2")]
pub owner_id: ::prost::alloc::string::String,
#[prost(string, tag = "3")]
pub instance_id: ::prost::alloc::string::String,
#[prost(bool, tag = "4")]
pub supported: bool,
#[prost(uint32, tag = "5")]
pub max_entries: u32,
#[prost(uint32, tag = "6")]
pub max_request_bytes: u32,
#[prost(uint64, tag = "7")]
pub cleared: u64,
#[prost(bytes = "bytes", tag = "8")]
pub response_proof: ::prost::bytes::Bytes,
}
/// A short-lived storage-owned read admission used only around a final
/// scanner metadata publication. It is intentionally separate from the
/// ScannerActivity observation wire so v6/v7 rolling compatibility remains
@@ -6282,6 +6330,244 @@ pub mod node_service_server {
}
}
/// Generated client implementations.
pub mod scanner_control_service_client {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::http::Uri;
use tonic::codegen::*;
#[derive(Debug, Clone)]
pub struct ScannerControlServiceClient<T> {
inner: tonic::client::Grpc<T>,
}
impl ScannerControlServiceClient<tonic::transport::Channel> {
/// Attempt to create a new client by connecting to a given endpoint.
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
where
D: TryInto<tonic::transport::Endpoint>,
D::Error: Into<StdError>,
{
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
Ok(Self::new(conn))
}
}
impl<T> ScannerControlServiceClient<T>
where
T: tonic::client::GrpcService<tonic::body::Body>,
T::Error: Into<StdError>,
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
{
pub fn new(inner: T) -> Self {
let inner = tonic::client::Grpc::new(inner);
Self { inner }
}
pub fn with_origin(inner: T, origin: Uri) -> Self {
let inner = tonic::client::Grpc::with_origin(inner, origin);
Self { inner }
}
pub fn with_interceptor<F>(inner: T, interceptor: F) -> ScannerControlServiceClient<InterceptedService<T, F>>
where
F: tonic::service::Interceptor,
T::ResponseBody: Default,
T: tonic::codegen::Service<
http::Request<tonic::body::Body>,
Response = http::Response<<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody>,
>,
<T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
Into<StdError> + std::marker::Send + std::marker::Sync,
{
ScannerControlServiceClient::new(InterceptedService::new(inner, interceptor))
}
/// Compress requests with the given encoding.
///
/// This requires the server to support it otherwise it might respond with an
/// error.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.send_compressed(encoding);
self
}
/// Enable decompressing responses.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.accept_compressed(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_decoding_message_size(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_encoding_message_size(limit);
self
}
pub async fn scanner_scoped_dirty_usage_ack(
&mut self,
request: impl tonic::IntoRequest<super::ScannerScopedDirtyUsageAckRequest>,
) -> std::result::Result<tonic::Response<super::ScannerScopedDirtyUsageAckResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.ScannerControlService/ScannerScopedDirtyUsageAck");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.ScannerControlService", "ScannerScopedDirtyUsageAck"));
self.inner.unary(req, path, codec).await
}
}
}
/// Generated server implementations.
pub mod scanner_control_service_server {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::*;
/// Generated trait containing gRPC methods that should be implemented for use with ScannerControlServiceServer.
#[async_trait]
pub trait ScannerControlService: std::marker::Send + std::marker::Sync + 'static {
async fn scanner_scoped_dirty_usage_ack(
&self,
request: tonic::Request<super::ScannerScopedDirtyUsageAckRequest>,
) -> std::result::Result<tonic::Response<super::ScannerScopedDirtyUsageAckResponse>, tonic::Status>;
}
#[derive(Debug)]
pub struct ScannerControlServiceServer<T> {
inner: Arc<T>,
accept_compression_encodings: EnabledCompressionEncodings,
send_compression_encodings: EnabledCompressionEncodings,
max_decoding_message_size: Option<usize>,
max_encoding_message_size: Option<usize>,
}
impl<T> ScannerControlServiceServer<T> {
pub fn new(inner: T) -> Self {
Self::from_arc(Arc::new(inner))
}
pub fn from_arc(inner: Arc<T>) -> Self {
Self {
inner,
accept_compression_encodings: Default::default(),
send_compression_encodings: Default::default(),
max_decoding_message_size: None,
max_encoding_message_size: None,
}
}
pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
where
F: tonic::service::Interceptor,
{
InterceptedService::new(Self::new(inner), interceptor)
}
/// Enable decompressing requests with the given encoding.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.accept_compression_encodings.enable(encoding);
self
}
/// Compress responses with the given encoding, if the client supports it.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.send_compression_encodings.enable(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.max_decoding_message_size = Some(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.max_encoding_message_size = Some(limit);
self
}
}
impl<T, B> tonic::codegen::Service<http::Request<B>> for ScannerControlServiceServer<T>
where
T: ScannerControlService,
B: Body + std::marker::Send + 'static,
B::Error: Into<StdError> + std::marker::Send + 'static,
{
type Response = http::Response<tonic::body::Body>;
type Error = std::convert::Infallible;
type Future = BoxFuture<Self::Response, Self::Error>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, req: http::Request<B>) -> Self::Future {
match req.uri().path() {
"/node_service.ScannerControlService/ScannerScopedDirtyUsageAck" => {
#[allow(non_camel_case_types)]
struct ScannerScopedDirtyUsageAckSvc<T: ScannerControlService>(pub Arc<T>);
impl<T: ScannerControlService> tonic::server::UnaryService<super::ScannerScopedDirtyUsageAckRequest>
for ScannerScopedDirtyUsageAckSvc<T>
{
type Response = super::ScannerScopedDirtyUsageAckResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::ScannerScopedDirtyUsageAckRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as ScannerControlService>::scanner_scoped_dirty_usage_ack(&inner, request).await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = ScannerScopedDirtyUsageAckSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
_ => Box::pin(async move {
let mut response = http::Response::new(tonic::body::Body::default());
let headers = response.headers_mut();
headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
Ok(response)
}),
}
}
}
impl<T> Clone for ScannerControlServiceServer<T> {
fn clone(&self) -> Self {
let inner = self.inner.clone();
Self {
inner,
accept_compression_encodings: self.accept_compression_encodings,
send_compression_encodings: self.send_compression_encodings,
max_decoding_message_size: self.max_decoding_message_size,
max_encoding_message_size: self.max_encoding_message_size,
}
}
}
/// Generated gRPC service name
pub const SERVICE_NAME: &str = "node_service.ScannerControlService";
impl<T> tonic::server::NamedService for ScannerControlServiceServer<T> {
const NAME: &'static str = SERVICE_NAME;
}
}
/// Generated client implementations.
pub mod heal_control_service_client {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::http::Uri;
crates/protos/src/lib.rs
+2
View File
@@ -541,6 +541,8 @@ pub fn canonical_scanner_activity_v7_response_body(
Ok(body)
}
pub mod scoped_dirty_usage;
pub fn canonical_scanner_dirty_usage_snapshot_request_body(
request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
crates/protos/src/node.proto
+34
View File
@@ -903,6 +903,36 @@ message ScannerDirtyUsageSnapshotResponse {
bytes response_proof = 7;
}
// Receiver-only protocol. Producers must retain whole-cycle ACK until they
// have a durable per-bucket publication proof.
message ScannerScopedDirtyUsageEntry {
string bucket = 1;
bytes bucket_incarnation = 2;
uint64 generation = 3;
}
message ScannerScopedDirtyUsageAckRequest {
bytes challenge = 1;
uint32 protocol_version = 2;
string owner_id = 3;
string instance_id = 4;
// Only scope 1 (a complete bucket) is supported; zero is invalid.
uint32 scope = 5;
bool probe_only = 6;
repeated ScannerScopedDirtyUsageEntry entries = 7;
}
message ScannerScopedDirtyUsageAckResponse {
uint32 protocol_version = 1;
string owner_id = 2;
string instance_id = 3;
bool supported = 4;
uint32 max_entries = 5;
uint32 max_request_bytes = 6;
uint64 cleared = 7;
bytes response_proof = 8;
}
// A short-lived storage-owned read admission used only around a final
// scanner metadata publication. It is intentionally separate from the
// ScannerActivity observation wire so v6/v7 rolling compatibility remains
@@ -1245,6 +1275,10 @@ service NodeService {
rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only
}
service ScannerControlService {
rpc ScannerScopedDirtyUsageAck(ScannerScopedDirtyUsageAckRequest) returns (ScannerScopedDirtyUsageAckResponse) {}; // auth-policy: body-bound
}
service HealControlService {
rpc HealControl(HealControlRequest) returns (HealControlResponse) {};
}
crates/protos/src/scoped_dirty_usage.rs
+213
View File
@@ -0,0 +1,213 @@
// Copyright 2024 RustFS Team
// Licensed under the Apache License, Version 2.0.
//! Bounded, authenticated receiver contract for per-bucket dirty acknowledgements.
use crate::CanonicalBodyBuilder;
use crate::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
use prost::Message;
pub const SCOPED_DIRTY_USAGE_PROTOCOL_VERSION: u32 = 1;
pub const SCOPED_DIRTY_USAGE_BUCKET_SCOPE: u32 = 1;
pub const SCOPED_DIRTY_USAGE_MAX_ENTRIES: u32 = 32;
pub const SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES: u32 = 8192;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScopedDirtyUsageRequestError {
UnsupportedProtocol,
UnsupportedScope,
InvalidIdentity,
InvalidGeneration,
InvalidEntries,
TooLarge,
}
impl std::fmt::Display for ScopedDirtyUsageRequestError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(match self {
Self::UnsupportedProtocol => "unsupported scoped dirty usage protocol",
Self::UnsupportedScope => "unsupported scoped dirty usage scope",
Self::InvalidIdentity => "invalid scoped dirty usage identity",
Self::InvalidGeneration => "invalid scoped dirty usage generation",
Self::InvalidEntries => "scoped dirty usage entries must be nonempty and strictly ordered",
Self::TooLarge => "scoped dirty usage request exceeds its budget",
})
}
}
impl std::error::Error for ScopedDirtyUsageRequestError {}
pub fn validate_scoped_dirty_usage_request(
request: &ScannerScopedDirtyUsageAckRequest,
) -> Result<(), ScopedDirtyUsageRequestError> {
use ScopedDirtyUsageRequestError as E;
if request.entries.len() > SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize
|| request.encoded_len() > SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize
{
return Err(E::TooLarge);
}
if request.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION {
return Err(E::UnsupportedProtocol);
}
if request.scope != SCOPED_DIRTY_USAGE_BUCKET_SCOPE {
return Err(E::UnsupportedScope);
}
if request.challenge.len() != 16 || request.owner_id.len() != 36 || request.instance_id.len() != 32 {
return Err(E::InvalidIdentity);
}
if request.entries.is_empty() || request.entries.windows(2).any(|pair| pair[0].bucket >= pair[1].bucket) {
return Err(E::InvalidEntries);
}
for entry in &request.entries {
if entry.bucket.is_empty()
|| entry.bucket.len() > 63
|| entry.bucket_incarnation.len() != 16
|| entry.bucket_incarnation.iter().all(|byte| *byte == 0)
{
return Err(E::InvalidIdentity);
}
if entry.generation == 0 || entry.generation == u64::MAX {
return Err(E::InvalidGeneration);
}
}
Ok(())
}
pub fn canonical_scoped_dirty_usage_request(
request: &ScannerScopedDirtyUsageAckRequest,
) -> Result<Vec<u8>, ScopedDirtyUsageRequestError> {
validate_scoped_dirty_usage_request(request)?;
let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-request-v1\0");
let encode = |_: std::num::TryFromIntError| ScopedDirtyUsageRequestError::TooLarge;
body.push_bytes(request.challenge.as_ref()).map_err(encode)?;
body.push_u32(request.protocol_version);
body.push_str(&request.owner_id).map_err(encode)?;
body.push_str(&request.instance_id).map_err(encode)?;
body.push_u32(request.scope);
body.push_bool(request.probe_only);
body.push_count(request.entries.len()).map_err(encode)?;
for entry in &request.entries {
body.push_str(&entry.bucket).map_err(encode)?;
body.push_bytes(entry.bucket_incarnation.as_ref()).map_err(encode)?;
body.push_u64(entry.generation);
}
Ok(body.finish())
}
pub fn canonical_scoped_dirty_usage_response(
request_body: &[u8],
response: &ScannerScopedDirtyUsageAckResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-response-v1\0");
body.push_bytes(request_body)?;
body.push_u32(response.protocol_version);
body.push_str(&response.owner_id)?;
body.push_str(&response.instance_id)?;
body.push_bool(response.supported);
body.push_u32(response.max_entries);
body.push_u32(response.max_request_bytes);
body.push_u64(response.cleared);
Ok(body.finish())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
fn request() -> ScannerScopedDirtyUsageAckRequest {
ScannerScopedDirtyUsageAckRequest {
challenge: vec![1; 16].into(),
protocol_version: 1,
owner_id: "11111111-1111-1111-1111-111111111111".into(),
instance_id: "a".repeat(32),
scope: 1,
probe_only: false,
entries: vec![ScannerScopedDirtyUsageEntry {
bucket: "photos".into(),
bucket_incarnation: vec![2; 16].into(),
generation: 8,
}],
}
}
#[test]
fn scoped_dirty_usage_binds_every_request_field() {
let base = request();
let baseline = canonical_scoped_dirty_usage_request(&base).expect("valid request");
for field in 0..9 {
let mut changed = base.clone();
match field {
0 => changed.challenge = vec![3; 16].into(),
1 => changed.protocol_version += 1,
2 => changed.owner_id = "22222222-2222-2222-2222-222222222222".into(),
3 => changed.instance_id = "b".repeat(32),
4 => changed.scope += 1,
5 => changed.probe_only = true,
6 => changed.entries[0].bucket = "videos".into(),
7 => changed.entries[0].bucket_incarnation = vec![3; 16].into(),
_ => changed.entries[0].generation += 1,
}
assert!(canonical_scoped_dirty_usage_request(&changed).map_or(true, |body| body != baseline));
}
}
#[test]
fn scoped_dirty_usage_binds_capability_and_ack_to_exact_request() {
let request = canonical_scoped_dirty_usage_request(&request()).expect("valid request");
let response = ScannerScopedDirtyUsageAckResponse {
protocol_version: 1,
owner_id: "owner".into(),
instance_id: "process".into(),
supported: true,
max_entries: 32,
max_request_bytes: 8192,
cleared: 1,
response_proof: vec![1; 32].into(),
};
let baseline = canonical_scoped_dirty_usage_response(&request, &response).expect("valid response");
for field in 0..7 {
let mut changed = response.clone();
match field {
0 => changed.protocol_version += 1,
1 => changed.owner_id.push('x'),
2 => changed.instance_id.push('x'),
3 => changed.supported = false,
4 => changed.max_entries += 1,
5 => changed.max_request_bytes += 1,
_ => changed.cleared += 1,
}
assert_ne!(
canonical_scoped_dirty_usage_response(&request, &changed).expect("response variant"),
baseline
);
}
assert_ne!(
canonical_scoped_dirty_usage_response(b"another request", &response).expect("request variant"),
baseline
);
}
#[test]
fn scoped_dirty_usage_rejects_overflow_unknown_and_duplicate_entries() {
let base = request();
let mut invalid = base.clone();
invalid.entries = vec![base.entries[0].clone(); SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize + 1];
assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge));
invalid = base.clone();
invalid.entries[0].bucket = "x".repeat(SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize);
assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge));
invalid = base.clone();
invalid.entries.push(base.entries[0].clone());
assert_eq!(
validate_scoped_dirty_usage_request(&invalid),
Err(ScopedDirtyUsageRequestError::InvalidEntries)
);
invalid = base;
invalid.entries[0].bucket_incarnation = vec![0; 16].into();
assert_eq!(
validate_scoped_dirty_usage_request(&invalid),
Err(ScopedDirtyUsageRequestError::InvalidIdentity)
);
}
}
crates/scanner/src/lib.rs
+3 -2
View File
@@ -90,8 +90,9 @@ pub use scanner::{
};
pub use scanner_io::{
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
scanner_maintenance_generation,
};
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
use std::sync::atomic::{AtomicU64, Ordering};
crates/scanner/src/remote_scanner.rs
+3
View File
@@ -52,6 +52,9 @@ static REMOTE_SCANNER_CYCLE_REFRESH: LazyLock<AsyncMutex<()>> = LazyLock::new(||
mod stream;
#[cfg(test)]
pub(crate) use stream::checkpoint_fixture_partial_return;
pub use stream::{RemoteScannerAdmission, RemoteScannerRequest, serve_remote_scanner_request};
pub(crate) use stream::{RemoteScannerOutcome, RemoteScannerScanSpec, scan_remote_bucket};
use stream::{RemoteScannerReplayCache, RemoteScannerRequestWire, RemoteScannerValidatedCycle};
crates/scanner/src/remote_scanner/stream.rs
+42
View File
@@ -1017,6 +1017,48 @@ fn finish_remote_scanner_stream(
#[cfg(test)]
const TEST_NEXT_CYCLE: u64 = 11;
#[cfg(test)]
pub(crate) async fn checkpoint_fixture_partial_return(progress: (u64, u64), entries_visited: u64) {
let request_id = Uuid::new_v4();
let writer_auth = FrameAuthenticator::for_test(request_id);
let reader_auth = FrameAuthenticator::for_test(request_id);
let mut bytes = Vec::new();
write_frame(
&mut bytes,
&writer_auth,
&mut 0,
&RemoteScannerFrame::terminal(
RemoteScannerProgress {
objects_scanned: progress.0,
directories_started: progress.1,
entries_visited,
},
RemoteScannerFrameResult::Partial,
),
)
.await
.expect("checkpoint partial frame must encode");
let frame = read_frame(&mut std::io::Cursor::new(bytes.as_slice()), &reader_auth, &mut 0)
.await
.expect("checkpoint progress frame must authenticate");
assert_eq!(frame.progress.entries_visited, entries_visited);
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
let result = consume_remote_scanner_stream(
std::io::Cursor::new(bytes),
parent,
budget.clone(),
"bucket",
DataUsageCacheSource::new(0, 0),
DataUsageScanPlanDigest([17; 32]),
reader_auth,
)
.await
.expect("checkpoint partial frame must decode");
assert!(matches!(result, RemoteScannerOutcome::Partial));
assert_eq!(budget.progress(), progress);
}
#[cfg(test)]
async fn consume_remote_scanner_stream<R>(
reader: R,
crates/scanner/src/scanner.rs
+15 -8
View File
@@ -1703,14 +1703,18 @@ where
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
let done_cycle = Metrics::time(Metric::ScanCycle);
let scan_result = crate::scanner_io::nsscanner_with_storage_status(
let scan_result = crate::scanner_io::nsscanner_with_storage_status_scoped(
storeapi.as_ref(),
cycle_budget.token(),
cycle_budget.clone(),
sender,
cycle_info.current,
leader_epoch,
scan_mode,
crate::scanner_io::ScannerCycleRequest {
ctx: cycle_budget.token(),
budget: cycle_budget.clone(),
updates: sender,
want_cycle: cycle_info.current,
leader_epoch,
scan_mode,
scan_scope: crate::scanner_io::ScannerBucketScanScope::default(),
persisted_usage_baseline: usage_persist_baseline.data.clone(),
},
)
.await;
let publication_defer_reason = match &scan_result {
@@ -3424,10 +3428,13 @@ use cycle_state::*;
use leadership::*;
use usage_store::*;
#[cfg(test)]
pub(crate) use activity::scanner_activity_snapshot_digest;
pub use activity::scanner_topology_digest;
pub(crate) use activity::{
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_structural_digest,
scanner_dirty_usage_acknowledgements,
};
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
pub use backlog::{
crates/scanner/src/scanner/activity.rs
+41
View File
@@ -902,6 +902,7 @@ where
observation
}
#[cfg(test)]
pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
let mut hasher = Sha256::new();
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
@@ -925,6 +926,30 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
hasher.finalize().into()
}
/// Hash the activity inputs that make an existing scanner cache unsafe to
/// reuse. Regular namespace writes and dirty-usage generations are omitted:
/// their affected buckets are tracked separately and may be refreshed from a
/// complete authoritative cache baseline.
pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
let mut hasher = Sha256::new();
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
for (host, activity) in snapshot {
let host = host.as_bytes();
let instance_id = activity.instance_id.as_bytes();
hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes());
hasher.update(host);
hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes());
hasher.update(instance_id);
hasher.update(activity.maintenance_generation.to_be_bytes());
hasher.update(activity.protocol_version.to_be_bytes());
hasher.update(activity.topology_digest);
hasher.update([u8::from(activity.data_movement_active)]);
hasher.update(activity.movement_generation.to_be_bytes());
hasher.update([u8::from(activity.publication_blocked)]);
}
hasher.finalize().into()
}
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
!snapshot.is_empty()
&& snapshot.values().all(|activity| {
@@ -955,6 +980,22 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna
.collect()
}
pub(crate) fn scanner_activity_dirty_usage_state_for_host<'a>(
snapshot: &'a ScannerActivitySnapshot,
host: &str,
) -> Option<(&'a str, u64, bool)> {
snapshot
.get(host)
.filter(|_| host != LOCAL_SCANNER_ACTIVITY_NODE)
.map(|activity| {
(
activity.instance_id.as_str(),
activity.dirty_usage_generation,
activity.dirty_usage_pending,
)
})
}
pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] {
let endpoint_pools = storeapi.endpoints();
let mut hasher = Sha256::new();
crates/scanner/src/scanner/cycle_state.rs
+280 -51
View File
@@ -379,6 +379,12 @@ pub(super) fn decode_recovery_marker_for_reset(
if !matches!(marker_revision, DataUsageCacheRevision::Etag(_)) {
return Err(ScannerError::Other("cycle recovery marker has no object revision".to_string()));
}
if let Ok(value) = serde_json::from_slice::<serde_json::Value>(data)
&& let Some(state) = value.get("state")
&& !matches!(state.as_str(), Some("blocked" | "cleanup-pending"))
{
return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string()));
}
let compat = serde_json::from_slice::<ScannerCycleRecoveryMarkerCompat>(data).ok();
let _schema_version = compat.as_ref().and_then(|marker| marker.schema_version);
let primary_revision = compat
@@ -406,7 +412,10 @@ pub(super) fn decode_recovery_marker_for_reset(
};
let state = match compat.as_ref().and_then(|marker| marker.state.as_deref()) {
Some("cleanup-pending") => "cleanup-pending",
_ => "blocked",
Some("blocked") | None => "blocked",
Some(_) => {
return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string()));
}
};
let now = unix_now_secs();
Ok(ScannerCycleRecoveryMarker {
@@ -721,17 +730,19 @@ async fn mark_cycle_recovery_cleanup_pending(
mut marker: ScannerCycleRecoveryMarker,
marker_revision: &DataUsageCacheRevision,
expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> {
marker.state = "cleanup-pending".to_string();
marker.last_attempt_at_unix_secs = unix_now_secs();
let bytes = serde_json::to_vec(&marker)
.map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?;
let info = save_config_with_publication_admission_for_epoch(
let info = save_reset_config(
storeapi.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
bytes,
marker_revision.preconditions(),
expected_epoch,
owns_reset,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?;
@@ -933,6 +944,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
.get_write_lock_quiet(Duration::from_secs(5))
.await
.map_err(|err| ScannerError::Other(format!("scanner leader lock is busy: {err}")))?;
let owns_reset = || !guard.is_lock_lost() && !ctx.is_cancelled();
if guard.is_lock_lost() {
return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string()));
@@ -952,7 +964,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
}
Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))),
};
let marker_data = marker_data.ok_or_else(|| ScannerError::Other("scanner cycle recovery marker is absent".to_string()))?;
let Some(marker_data) = marker_data else {
// A delete may commit before its reply is lost. Confirm both durable
// fences before treating a retry without its marker as completed.
let (cycle, epoch, revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
let floor = persisted_usage_floor(storeapi.clone()).await?;
if !matches!(revision, DataUsageCacheRevision::Etag(_))
|| epoch < floor.leader_epoch
|| cycle.next < floor.next_cycle
|| !owns_reset()
|| scanner_publication_admission_for_epoch(storeapi.clone(), reset_epoch)
.await
.is_none()
{
return Err(ScannerError::Other(
"scanner cycle recovery marker is absent without a completed reset fence".to_string(),
));
}
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
super::notify_scanner_cycle_recovery_wake();
return Ok(());
};
let (marker, force_full_rescan) = match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&marker_data) {
Ok(marker) if validate_recovery_marker(&marker).is_ok() => (marker, false),
_ => (decode_recovery_marker_for_reset(&marker_data, &marker_revision)?, true),
@@ -1026,8 +1058,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
}
};
if let Some((primary_cycle, primary_epoch)) = primary_state {
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
let (cleanup_marker, cleanup_marker_revision) =
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?;
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch, &owns_reset)
.await?;
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
let fence_epoch = primary_epoch
@@ -1047,12 +1081,14 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"preserved scanner cycle state exceeds the bounded object size".to_string(),
));
}
let preserved_info = save_config_with_publication_admission_for_epoch(
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
let preserved_info = save_reset_config(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
preserved_data,
primary_revision.preconditions(),
reset_epoch,
&owns_reset,
)
.await
.map_err(|err| {
@@ -1072,9 +1108,17 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after fencing newer cycle state".to_string(),
));
}
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch), false)
.await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
fence_scanner_usage_epoch_with_expected_epoch(
&ctx,
storeapi.clone(),
fence_epoch,
Some(reset_epoch),
false,
&owns_reset,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
if guard.is_lock_lost() {
return Err(ScannerError::Other(
"scanner leader lock was lost after fencing newer cycle state".to_string(),
@@ -1088,7 +1132,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner cycle state changed before recovery marker cleanup".to_string(),
));
}
delete_config_with_publication_admission_for_epoch(
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
delete_reset_config(
storeapi.clone(),
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
@@ -1100,6 +1145,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
..Default::default()
},
reset_epoch,
&owns_reset,
)
.await
.map_err(|err| {
@@ -1149,17 +1195,20 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
// Persist the cleanup-pending phase before rewriting the primary. If the
// process dies after the rewrite, startup still sees a durable fence and
// cannot mistake the partially completed reset for a healthy state.
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
let (marker, marker_revision) = if marker.state == "cleanup-pending" {
(marker, marker_revision)
} else {
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await?
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch, &owns_reset).await?
};
let rebuilt_info = save_config_with_publication_admission_for_epoch(
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
let rebuilt_info = save_reset_config(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
data,
primary_revision.preconditions(),
reset_epoch,
&owns_reset,
)
.await
.map_err(|err| {
@@ -1178,8 +1227,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after rebuilding cycle state".to_string(),
));
}
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
if let Err(err) =
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false).await
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false, &owns_reset)
.await
{
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
@@ -1249,7 +1300,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
));
}
if let Err(err) = delete_config_with_publication_admission_for_epoch(
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
if let Err(err) = delete_reset_config(
storeapi.clone(),
RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
@@ -1261,6 +1313,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
..Default::default()
},
reset_epoch,
&owns_reset,
)
.await
{
@@ -1310,6 +1363,57 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
Ok(())
}
async fn verify_cycle_reset_intent(
storeapi: Arc<impl ScannerObjectIO>,
expected_revision: &DataUsageCacheRevision,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<(), ScannerError> {
let revision = read_config_revision(storeapi, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to verify scanner cycle reset intent: {err}")))?;
if &revision != expected_revision {
return Err(ScannerError::Other("scanner cycle reset intent changed".to_string()));
}
if !owns_reset() {
return Err(ScannerError::Other("scanner cycle reset ownership was lost".to_string()));
}
Ok(())
}
async fn save_reset_config(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
path: &str,
data: Vec<u8>,
preconditions: crate::HTTPPreconditions,
expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<crate::ScannerObjectInfo, EcstoreError> {
let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
if !owns_reset() {
return Err(EcstoreError::other("scanner reset ownership was lost before write"));
}
save_config_with_preconditions(storeapi, path, data, preconditions).await
}
async fn delete_reset_config(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
bucket: &str,
path: &str,
options: ScannerObjectOptions,
expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<crate::ScannerObjectInfo, EcstoreError> {
let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
if !owns_reset() {
return Err(EcstoreError::other("scanner reset ownership was lost before delete"));
}
storeapi.delete_config_object(bucket, path, options).await
}
fn scanner_usage_state_reset_paths() -> Vec<String> {
vec![
DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
@@ -1333,8 +1437,14 @@ pub(super) async fn read_usage_state_reset_slots(
Ok(slots)
}
fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<PersistedUsageFloor, ScannerError> {
let mut floor = PersistedUsageFloor::default();
enum ScannerUsageResetFloor {
Missing,
Trusted(PersistedUsageFloor),
Corrupt,
}
fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<ScannerUsageResetFloor, ScannerError> {
let mut floor = None;
for slot in slots {
let Some(data) = slot.data.as_deref() else {
continue;
@@ -1342,9 +1452,21 @@ fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<Persi
let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(data) else {
continue;
};
update_persisted_usage_floor(&mut floor, &usage, &slot.path)?;
if !data_usage_info_has_persisted_baseline_identity(&usage)
&& !(slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str() && data_usage_info_is_bootstrap_pending(&usage))
&& legacy_incomplete_usage_fence(data, &usage)
.and_then(|fence| fence.claimable_epoch())
.is_none()
{
continue;
}
update_persisted_usage_floor(floor.get_or_insert_with(PersistedUsageFloor::default), &usage, &slot.path)?;
}
Ok(floor)
Ok(match floor {
Some(floor) => ScannerUsageResetFloor::Trusted(floor),
None if slots.iter().any(|slot| slot.data.is_some()) => ScannerUsageResetFloor::Corrupt,
None => ScannerUsageResetFloor::Missing,
})
}
async fn read_cycle_state_for_usage_reset(
@@ -1401,11 +1523,12 @@ async fn delete_usage_state_reset_slot(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
slot: &ScannerUsageStateResetSlot,
expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<bool, ScannerError> {
if matches!(slot.revision, DataUsageCacheRevision::Missing) {
return Ok(false);
}
let delete_result = delete_config_with_publication_admission_for_epoch(
let delete_result = delete_reset_config(
storeapi.clone(),
RUSTFS_META_BUCKET,
&slot.path,
@@ -1415,6 +1538,7 @@ async fn delete_usage_state_reset_slot(
..Default::default()
},
expected_epoch,
owns_reset,
)
.await;
match delete_result {
@@ -1486,21 +1610,24 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary(
expected_publication_epoch: u64,
leader_epoch: Option<u64>,
context: ScannerUsageBootstrapPublishContext,
owns_publication: impl Fn() -> bool + Sync,
) -> Result<(), ScannerError> {
async fn inner(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
expected_revision: &DataUsageCacheRevision,
expected_publication_epoch: u64,
leader_epoch: Option<u64>,
owns_publication: &(impl Fn() -> bool + Sync),
) -> Result<(), ScannerUsageBootstrapPublishError> {
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::now(), leader_epoch);
let data = serde_json::to_vec(&marker).map_err(ScannerUsageBootstrapPublishError::Encode)?;
let save_result = save_config_with_publication_admission_for_epoch(
let save_result = save_reset_config(
storeapi.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
data.clone(),
expected_revision.preconditions(),
expected_publication_epoch,
owns_publication,
)
.await;
if save_result
@@ -1524,7 +1651,7 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary(
})
}
inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch)
inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch, &owns_publication)
.await
.map_err(|err| err.into_scanner_error(context))
}
@@ -1534,32 +1661,108 @@ pub(super) async fn reset_scanner_usage_state_slots_for_full_rebuild(
slots: &[ScannerUsageStateResetSlot],
expected_epoch: u64,
leader_epoch: u64,
owns_reset: impl Fn() -> bool + Sync,
) -> Result<Vec<String>, ScannerError> {
let mut reset_paths = Vec::new();
let primary = slots
.iter()
.find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str())
.ok_or_else(|| ScannerError::Other("scanner usage reset primary slot was not inspected".to_string()))?;
publish_scanner_usage_bootstrap_primary(
storeapi.clone(),
&primary.revision,
expected_epoch,
Some(leader_epoch),
ScannerUsageBootstrapPublishContext::Reset,
)
.await?;
if !owns_reset() {
return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string()));
}
let resume_epoch = usage_state_reset_resume_epoch(slots)?;
match resume_epoch {
Some(epoch) if epoch == leader_epoch => {}
Some(_) => return Err(ScannerError::Other("scanner usage reset bootstrap epoch changed".to_string())),
None => {
publish_scanner_usage_bootstrap_primary(
storeapi.clone(),
&primary.revision,
expected_epoch,
Some(leader_epoch),
ScannerUsageBootstrapPublishContext::Reset,
&owns_reset,
)
.await?;
}
}
let (data, intent_revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to inspect scanner usage reset intent: {err}")))?;
data.as_deref()
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok())
.filter(|usage| data_usage_info_is_bootstrap_pending(usage) && usage.scanner_epoch == Some(leader_epoch))
.ok_or_else(|| ScannerError::Other("scanner usage reset intent changed before cleanup".to_string()))?;
if !matches!(intent_revision, DataUsageCacheRevision::Etag(_))
|| (resume_epoch.is_some() && intent_revision != primary.revision)
{
return Err(ScannerError::Other("scanner usage reset intent revision changed".to_string()));
}
reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string());
for slot in slots.iter().filter(|slot| slot.path != DATA_USAGE_OBJ_NAME_PATH.as_str()) {
if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch).await? {
if let Some(usage) = slot
.data
.as_deref()
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok())
&& usage_epoch(&usage) >= leader_epoch
{
return Err(ScannerError::Other(format!(
"scanner usage reset slot is not older than its intent: {}",
slot.path
)));
}
let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to verify scanner usage reset intent: {err}")))?;
if revision != intent_revision {
return Err(ScannerError::Other("scanner usage reset intent changed during cleanup".to_string()));
}
if !owns_reset() {
return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string()));
}
if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch, &owns_reset).await? {
reset_paths.push(slot.path.clone());
}
}
let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to confirm scanner usage reset intent: {err}")))?;
if revision != intent_revision || !owns_reset() {
return Err(ScannerError::Other(
"scanner usage reset intent or ownership changed before completion".to_string(),
));
}
invalidate_admin_data_usage_snapshot_cache().await;
invalidate_data_usage_snapshot_cache().await;
Ok(reset_paths)
}
fn usage_state_reset_resume_epoch(slots: &[ScannerUsageStateResetSlot]) -> Result<Option<u64>, ScannerError> {
let primary = slots.iter().find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str());
let usage = primary
.and_then(|slot| slot.data.as_deref())
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok());
match usage {
Some(usage) if usage.usage_snapshot_bootstrap_pending => {
if !data_usage_info_is_bootstrap_pending(&usage) {
return Err(ScannerError::Other("scanner usage reset bootstrap is invalid".to_string()));
}
if usage.scanner_epoch.is_none() {
// Initial bootstrap has no reset owner yet.
return Ok(None);
}
usage
.scanner_epoch
.filter(|epoch| *epoch > 0 && *epoch < u64::MAX)
.map(Some)
.ok_or_else(|| ScannerError::Other("scanner usage reset bootstrap has no valid epoch".to_string()))
}
_ => Ok(None),
}
}
pub async fn reset_scanner_usage_state_for_full_rebuild(
ctx: CancellationToken,
storeapi: Arc<ECStore>,
@@ -1584,12 +1787,31 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
};
let (cycle, cycle_epoch, cycle_revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
let slots = read_usage_state_reset_slots(storeapi.clone()).await?;
let usage_floor = usage_state_reset_floor(&slots)?;
let leader_epoch = cycle_epoch
.max(usage_floor.leader_epoch)
.checked_add(1)
.filter(|epoch| *epoch < u64::MAX)
.ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?;
let usage_floor = match usage_state_reset_floor(&slots)? {
ScannerUsageResetFloor::Trusted(floor) => floor,
ScannerUsageResetFloor::Corrupt if matches!(cycle_revision, DataUsageCacheRevision::Missing) => {
return Err(ScannerError::Other("scanner usage reset has no trusted cycle or usage floor".to_string()));
}
ScannerUsageResetFloor::Missing | ScannerUsageResetFloor::Corrupt => PersistedUsageFloor {
next_cycle: cycle.next,
leader_epoch: cycle_epoch,
},
};
let resume_epoch = usage_state_reset_resume_epoch(&slots)?;
let leader_epoch = if let Some(epoch) = resume_epoch {
if epoch != cycle_epoch || usage_floor.leader_epoch > epoch || usage_floor.next_cycle > cycle.next {
return Err(ScannerError::Other(
"scanner usage reset bootstrap conflicts with the persisted cycle fence".to_string(),
));
}
epoch
} else {
cycle_epoch
.max(usage_floor.leader_epoch)
.checked_add(1)
.filter(|epoch| *epoch < u64::MAX)
.ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?
};
let rebuilt_cycle = CurrentCycle {
next: cycle.next.max(usage_floor.next_cycle),
..Default::default()
@@ -1602,21 +1824,24 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
"scanner leader lock was lost before fencing usage reset cycle state".to_string(),
));
}
save_config_with_publication_admission_for_epoch(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
cycle_data,
cycle_revision.preconditions(),
reset_epoch,
)
.await
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}"))
}
})?;
if resume_epoch.is_none() {
save_reset_config(
storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
cycle_data,
cycle_revision.preconditions(),
reset_epoch,
&|| !guard.is_lock_lost() && !ctx.is_cancelled(),
)
.await
.map_err(|err| {
if scanner_publication_epoch_changed(&err) {
ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string())
} else {
ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}"))
}
})?;
}
if guard.is_lock_lost() {
return Err(ScannerError::Other(
@@ -1624,7 +1849,10 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
));
}
let reset_paths =
reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch).await?;
reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch, || {
!guard.is_lock_lost() && !ctx.is_cancelled()
})
.await?;
if guard.is_lock_lost() {
return Err(ScannerError::Other(
"scanner leader lock was lost after publishing usage reset marker".to_string(),
@@ -2135,6 +2363,7 @@ async fn recover_legacy_incomplete_usage_floor(
expected_publication_epoch,
Some(primary.epoch),
ScannerUsageBootstrapPublishContext::Recovery,
|| true,
)
.await?;
warn!(
crates/scanner/src/scanner/leadership.rs
+7 -1
View File
@@ -191,6 +191,7 @@ pub(super) async fn initialize_usage_baseline_bootstrap(
expected_epoch,
None,
ScannerUsageBootstrapPublishContext::Initial,
|| true,
)
.await
}
@@ -201,9 +202,10 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
claimed_epoch: u64,
expected_publication_epoch: Option<u64>,
allow_bootstrap_pending: bool,
owns_fence: impl Fn() -> bool,
) -> Result<(), ScannerError> {
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() {
if ctx.is_cancelled() || !owns_fence() {
return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string()));
}
@@ -264,6 +266,9 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
"scanner usage epoch fence changed while preparing its conditional write".to_string(),
));
};
if ctx.is_cancelled() || !owns_fence() {
return Err(ScannerError::Other("scanner leadership was lost before usage fencing".to_string()));
}
save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions())
.await
};
@@ -319,6 +324,7 @@ pub(super) async fn complete_scanner_leadership_claim(
claimed_epoch,
expected_publication_epoch,
allow_bootstrap_pending,
|| true,
)
.await
{
crates/scanner/src/scanner/tests.rs
+415 -3
View File
@@ -634,6 +634,8 @@ struct MemoryConfigStore {
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
error_after_commit_deletes: Mutex<HashSet<String>>,
cancel_after_deletes: Mutex<HashMap<String, CancellationToken>>,
pause_next_publication_admission: Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>,
put_counts: Mutex<HashMap<String, usize>>,
publication_admission_blocked: AtomicBool,
block_publication_after_admissions: AtomicUsize,
@@ -4081,6 +4083,9 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
revisions.remove(&key);
drop(revisions);
drop(objects);
if let Some(token) = self.cancel_after_deletes.lock().await.remove(&key) {
token.cancel();
}
if self.error_after_commit_deletes.lock().await.remove(&key) {
return Err(EcstoreError::other("injected delete error after commit"));
}
@@ -4088,6 +4093,11 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
let pause = self.pause_next_publication_admission.lock().await.take();
if let Some((entered, resume)) = pause {
entered.notify_one();
resume.notified().await;
}
if self.publication_admission_blocked.load(Ordering::Acquire) {
return None;
}
@@ -4589,7 +4599,7 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met
.expect("publication must also read the intact backup");
assert_eq!(baseline.data.as_deref(), Some(data.as_slice()));
assert_eq!(baseline.revision, DataUsageCacheRevision::Missing);
fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false)
fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false, || true)
.await
.expect("legacy backup must be fenced into v2");
let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
@@ -4818,6 +4828,28 @@ async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() {
);
}
let cycle_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("cycle should remain before retry");
let marker_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("bootstrap should remain before retry");
let retry = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
.await
.expect("completed cleanup should be reentrant");
assert_eq!(retry.leader_epoch, result.leader_epoch);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("cycle should remain"),
cycle_before_retry
);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("bootstrap should remain"),
marker_before_retry
);
let (floor, state) = persisted_usage_floor_for_startup(store, false)
.await
.expect("reset marker should be resumable");
@@ -4973,7 +5005,7 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() {
store.objects.lock().await.insert(key.clone(), b"newer-json".to_vec());
store.revisions.lock().await.insert(key, 2);
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3)
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true)
.await
.expect_err("stale primary revision must not be overwritten");
assert!(
@@ -4983,6 +5015,348 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() {
);
}
#[tokio::test]
async fn scanner_usage_state_reset_resumes_every_cleanup_boundary_without_rewriting_intent() {
for completed in 0..=4 {
let store = Arc::new(MemoryConfigStore::default());
let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str();
let cleanup_paths = [
format!("{primary_path}.bkp"),
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str().to_string(),
];
for path in std::iter::once(primary_path).chain(cleanup_paths.iter().map(String::as_str)) {
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(1);
save_config(store.clone(), path, serde_json::to_vec(&usage).expect("fixture should encode"))
.await
.expect("fixture should persist");
}
// These objects belong to other owners, even when reset cleanup resumes.
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
save_config(store.clone(), path, b"retain".to_vec())
.await
.expect("unrelated state should persist");
}
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
let cancelled = CancellationToken::new();
if completed == 0 {
store
.cancel_after_successful_puts
.lock()
.await
.insert(memory_config_key(RUSTFS_META_BUCKET, primary_path), (2, cancelled.clone()));
} else {
store
.cancel_after_deletes
.lock()
.await
.insert(memory_config_key(RUSTFS_META_BUCKET, &cleanup_paths[completed - 1]), cancelled.clone());
}
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled())
.await
.expect_err("interruption should stop cleanup");
assert!(err.to_string().contains("ownership"), "boundary {completed}: {err}");
for (index, path) in cleanup_paths.iter().enumerate() {
assert_eq!(
store
.objects
.lock()
.await
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, path)),
index >= completed,
"boundary {completed}, slot {index}"
);
}
let intent = read_config_with_revision(store.clone(), primary_path)
.await
.expect("intent should persist");
let slots = read_usage_state_reset_slots(store.clone())
.await
.expect("restart should reload slots");
reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
.await
.expect("restart should complete the same intent");
assert_eq!(
read_config_with_revision(store.clone(), primary_path)
.await
.expect("intent should remain"),
intent
);
assert_eq!(store.put_counts.lock().await[&memory_config_key(RUSTFS_META_BUCKET, primary_path)], 2);
for path in cleanup_paths {
assert!(
!store
.objects
.lock()
.await
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, &path))
);
}
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
assert_eq!(read_config(store.clone(), path).await.expect("unrelated state should remain"), b"retain");
}
}
}
#[tokio::test]
async fn scanner_usage_state_reset_stops_usage_fence_after_owner_loss() {
let store = Arc::new(MemoryConfigStore::default());
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(1);
let bytes = serde_json::to_vec(&usage).expect("baseline should encode");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
.await
.expect("baseline should persist");
let checks = AtomicUsize::new(0);
let err = fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 3, Some(0), false, || {
checks.fetch_add(1, Ordering::SeqCst) == 0
})
.await
.expect_err("ownership lost during reads must prevent the write");
assert!(err.to_string().contains("leadership was lost"), "{err}");
assert_eq!(
read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("baseline should remain"),
bytes
);
}
#[tokio::test]
async fn scanner_usage_state_reset_cancels_during_publication_admission() {
for resuming in [false, true] {
let store = Arc::new(MemoryConfigStore::default());
let usage = if resuming {
scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3))
} else {
complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
};
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&usage).expect("primary should encode"),
)
.await
.expect("primary should persist");
save_config(store.clone(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), b"corrupt".to_vec())
.await
.expect("cleanup target should persist");
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
let before = store.objects.lock().await.clone();
let revisions_before = store.revisions.lock().await.clone();
let entered = Arc::new(tokio::sync::Notify::new());
let resume = Arc::new(tokio::sync::Notify::new());
*store.pause_next_publication_admission.lock().await = Some((entered.clone(), resume.clone()));
let cancelled = CancellationToken::new();
let (result, ()) = tokio::join!(
reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled()),
async {
entered.notified().await;
cancelled.cancel();
resume.notify_one();
}
);
let err = result.expect_err("losing ownership during admission must prevent mutation");
assert!(err.to_string().contains("ownership was lost"), "resuming={resuming}: {err}");
assert_eq!(*store.objects.lock().await, before);
assert_eq!(*store.revisions.lock().await, revisions_before);
}
}
#[tokio::test]
#[serial]
async fn scanner_usage_state_reset_rejects_corruption_without_a_trusted_floor() {
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), b"{corrupt".to_vec())
.await
.expect("corrupt primary should persist");
let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("evidence should load");
let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
.await
.expect_err("corruption must not become a zero floor");
assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("evidence should remain"),
before
);
assert!(matches!(
read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await,
Err(EcstoreError::ConfigNotFound)
));
let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
backup.scanner_epoch = Some(7);
backup.scanner_cycle = Some(40);
save_config(
store.clone(),
&format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
serde_json::to_vec(&backup).expect("backup should encode"),
)
.await
.expect("valid backup should persist");
let result = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store)
.await
.expect("valid backup should supply the recovery floor");
assert_eq!(result.leader_epoch, 8);
assert_eq!(result.next_cycle, 41);
}
#[tokio::test]
async fn scanner_usage_state_reset_rejects_replaced_intent_and_newer_cleanup_slot() {
let store = Arc::new(MemoryConfigStore::default());
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3));
let bytes = serde_json::to_vec(&marker).expect("marker should encode");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
.await
.expect("intent should persist");
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes)
.await
.expect("another intent should persist");
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
.await
.expect_err("same epoch cannot replace an intent revision");
assert!(err.to_string().contains("intent revision changed"), "{err}");
let mut newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
newer.scanner_epoch = Some(3);
let path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
let bytes = serde_json::to_vec(&newer).expect("newer snapshot should encode");
save_config(store.clone(), &path, bytes.clone())
.await
.expect("newer snapshot should persist");
let slots = read_usage_state_reset_slots(store.clone())
.await
.expect("slots should reload");
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
.await
.expect_err("cleanup cannot delete same-epoch progress");
assert!(err.to_string().contains("not older than its intent"), "{err}");
assert_eq!(read_config(store, &path).await.expect("newer snapshot should remain"), bytes);
}
#[tokio::test]
#[serial]
async fn scanner_usage_state_reset_rejects_decodable_untrusted_floor() {
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
let invalid_identity = DataUsageInfo {
usage_snapshot_complete: true,
buckets_count: 1,
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
..Default::default()
};
for usage in [DataUsageInfo::default(), invalid_identity] {
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&usage).expect("fixture should encode"),
)
.await
.expect("untrusted primary should persist");
let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("primary should load");
let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
.await
.expect_err("valid JSON alone cannot prove a usage floor");
assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("evidence should remain"),
before
);
assert!(matches!(
read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await,
Err(EcstoreError::ConfigNotFound)
));
}
}
#[test]
fn full_rescan_reset_rejects_unknown_marker_phase_even_with_invalid_compat_fields() {
for state in [serde_json::json!("rewrite-v2"), serde_json::json!(7), serde_json::Value::Null] {
let marker = serde_json::json!({"state": state, "retry_count": "future-type", "schema_version": 99});
let err = super::cycle_state::decode_recovery_marker_for_reset(
&serde_json::to_vec(&marker).expect("future marker should encode"),
&DataUsageCacheRevision::Etag("intent-1".to_string()),
)
.expect_err("unknown persistent phases must remain fenced");
assert!(err.to_string().contains("state is unsupported"), "{err}");
}
}
#[tokio::test]
#[serial]
async fn full_rescan_reset_preserves_unknown_phase_and_retries_completed_cleanup() {
let (_temp_dir, store) = setup_scanner_cycle_store().await;
save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt".to_vec())
.await
.expect("corrupt primary should persist");
save_config(
store.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
br#"{"state":"future-rewrite"}"#.to_vec(),
)
.await
.expect("future marker should persist");
let primary_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("primary should load");
let marker_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
.await
.expect("marker should load");
let err = reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
.await
.expect_err("unknown phase must block explicit reset");
assert!(err.to_string().contains("state is unsupported"), "{err}");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("primary should remain"),
primary_before
);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
.await
.expect("marker should remain"),
marker_before
);
save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{malformed".to_vec())
.await
.expect("recoverable marker should persist");
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
.await
.expect("reset should complete");
let primary = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("rebuilt primary should load");
let usage = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("fenced usage should load");
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
.await
.expect("retry after marker deletion should complete");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("rebuilt primary should remain"),
primary
);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("fenced usage should remain"),
usage
);
}
#[tokio::test]
async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
let store = Arc::new(MemoryConfigStore::default());
@@ -4993,7 +5367,7 @@ async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
.expect("usage reset slots should be inspected");
store.publication_admission_blocked.store(true, Ordering::Release);
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3)
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true)
.await
.expect_err("movement admission loss must defer reset");
assert!(
@@ -8169,6 +8543,44 @@ fn scanner_activity_snapshot_digest_fences_dirty_usage_state() {
assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending));
}
#[test]
fn scanner_activity_structural_digest_ignores_regular_bucket_writes() {
let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
let mut written = baseline.clone();
let activity = written.get_mut("node-2").expect("node should exist");
activity.namespace_generation = 8;
activity.dirty_usage_generation = 6;
activity.dirty_usage_pending = true;
assert_ne!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&written));
assert_eq!(
scanner_activity_structural_digest(&baseline),
scanner_activity_structural_digest(&written),
"bucket writes are refreshed through the dirty-bucket scope rather than invalidating every cache"
);
}
#[test]
fn scanner_activity_structural_digest_fences_restart_and_maintenance() {
let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
let mut restarted = baseline.clone();
restarted.get_mut("node-2").expect("node should exist").instance_id = "epoch-b".to_string();
let mut maintained = baseline.clone();
maintained
.get_mut("node-2")
.expect("node should exist")
.maintenance_generation = 4;
assert_ne!(
scanner_activity_structural_digest(&baseline),
scanner_activity_structural_digest(&restarted)
);
assert_ne!(
scanner_activity_structural_digest(&baseline),
scanner_activity_structural_digest(&maintained)
);
}
#[test]
fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() {
let snapshot = BTreeMap::from([
crates/scanner/src/scanner_folder/tests.rs
+2
View File
@@ -24,6 +24,8 @@ use std::io::Write;
use std::os::unix::fs::{PermissionsExt, symlink};
use std::sync::Mutex;
mod checkpoint_fixture;
/// Reset the process-global alert cooldown map; test-only.
fn reset_alert_cooldowns() {
*SCANNER_ALERT_EMISSION_COOLDOWN
crates/scanner/src/scanner_folder/tests/checkpoint_fixture.rs
+410
View File
@@ -0,0 +1,410 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::scanner_budget::ScannerCycleBudgetConfig;
use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk};
use crate::storage_api::scanner_io::ObjectIO;
use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
use std::io::Cursor;
use tokio::io::AsyncReadExt;
const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin";
const STATIC_OBJECTS: u64 = 24;
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0);
const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]);
/// Real cache persistence codec and CAS calls, backed by two bounded local files.
#[derive(Debug)]
struct FixtureStore {
root: tempfile::TempDir,
reject_save: AtomicBool,
}
impl FixtureStore {
fn new() -> Arc<Self> {
Arc::new(Self {
root: tempfile::tempdir().expect("checkpoint fixture storage directory"),
reject_save: AtomicBool::new(false),
})
}
fn path(&self, object: &str) -> std::path::PathBuf {
assert!(object.ends_with(CACHE_NAME) || object.ends_with(&format!("{CACHE_NAME}.bkp")));
self.root
.path()
.join(if object.ends_with(".bkp") { "backup" } else { "main" })
}
async fn strict_load(&self) -> DataUsageCache {
let bytes = tokio::fs::read(self.root.path().join("main"))
.await
.expect("saved checkpoint fixture must exist");
decode_fixture(&bytes).expect("saved checkpoint fixture must contain a valid bucket root")
}
}
#[async_trait::async_trait]
impl ObjectIO for FixtureStore {
type Error = crate::EcstoreError;
type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec;
type HeaderMap = http::HeaderMap;
type ObjectOptions = crate::ScannerObjectOptions;
type ObjectInfo = crate::ScannerObjectInfo;
type GetObjectReader = crate::ScannerGetObjectReader;
type PutObjectReader = crate::ScannerPutObjReader;
async fn get_object_reader(
&self,
_bucket: &str,
object: &str,
_range: Option<Self::RangeSpec>,
_headers: Self::HeaderMap,
_options: &Self::ObjectOptions,
) -> crate::EcstoreResult<Self::GetObjectReader> {
let bytes = tokio::fs::read(self.path(object)).await.map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
crate::EcstoreError::FileNotFound
} else {
crate::EcstoreError::from(error)
}
})?;
assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
Ok(crate::ScannerGetObjectReader {
stream: Box::new(Cursor::new(bytes)),
object_info: crate::ScannerObjectInfo {
etag: Some("fixture".into()),
..Default::default()
},
buffered_body: None,
body_source: Default::default(),
})
}
async fn put_object(
&self,
_bucket: &str,
object: &str,
data: &mut Self::PutObjectReader,
options: &Self::ObjectOptions,
) -> crate::EcstoreResult<Self::ObjectInfo> {
if self.reject_save.load(Ordering::SeqCst) {
return Err(crate::EcstoreError::PreconditionFailed);
}
let path = self.path(object);
let exists = tokio::fs::try_exists(&path).await?;
let preconditions = options.http_preconditions.as_ref().expect("checkpoint writes must use CAS");
if (exists && preconditions.if_none_match_value() == Some("*"))
|| (!exists && preconditions.if_match_value().is_some())
|| (exists && preconditions.if_match_value() != Some("fixture"))
{
return Err(crate::EcstoreError::PreconditionFailed);
}
let mut bytes = Vec::new();
(&mut data.stream).take(MAX_CACHE_BYTES + 1).read_to_end(&mut bytes).await?;
assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
tokio::fs::write(path, bytes).await?;
Ok(crate::ScannerObjectInfo {
etag: Some("fixture".into()),
..Default::default()
})
}
}
#[async_trait::async_trait]
impl crate::ScannerConfigObjectDelete for FixtureStore {
async fn delete_config_object(
&self,
_bucket: &str,
_object: &str,
_options: crate::ScannerObjectOptions,
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
Err(crate::EcstoreError::NotImplemented)
}
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
}
}
fn decode_fixture(bytes: &[u8]) -> Result<DataUsageCache, &'static str> {
if bytes.is_empty() || bytes.len() > usize::try_from(MAX_CACHE_BYTES).expect("fixture bound") {
return Err("missing or oversized checkpoint fixture");
}
let cache = DataUsageCache::unmarshal(bytes).map_err(|_| "corrupt checkpoint fixture")?;
if cache.info.name != "bucket" || cache.checked_flatten("bucket").is_none() {
return Err("checkpoint fixture has no valid bucket root");
}
Ok(cache)
}
fn retained(cache: &DataUsageCache) -> u64 {
assert!(
!cache.root().is_some_and(|root| root.compacted),
"a compacted bucket root cannot prove static-prefix coverage"
);
cache
.checked_flatten("bucket/static")
.map_or(0, |entry| u64::try_from(entry.objects).expect("fixture object count fits u64"))
}
#[derive(Debug, PartialEq, Eq)]
enum CoverageDiagnosis {
Progress,
NoNewWork,
LostAtPrepare,
LostAtReload,
WalkWithoutRetention,
}
fn diagnose(previous: u64, prepared: u64, walked: u64, scanned: u64, reloaded: u64) -> CoverageDiagnosis {
if reloaded < scanned {
CoverageDiagnosis::LostAtReload
} else if prepared < previous {
CoverageDiagnosis::LostAtPrepare
} else if walked > 0 && reloaded <= previous {
CoverageDiagnosis::WalkWithoutRetention
} else if reloaded > previous {
CoverageDiagnosis::Progress
} else {
CoverageDiagnosis::NoNewWork
}
}
#[test]
fn checkpoint_fixture_diagnosis_rejects_walk_without_retention() {
assert_eq!(diagnose(4, 4, 9, 8, 8), CoverageDiagnosis::Progress);
assert_eq!(diagnose(4, 4, 9, 4, 4), CoverageDiagnosis::WalkWithoutRetention);
assert_eq!(diagnose(4, 0, 9, 4, 4), CoverageDiagnosis::LostAtPrepare);
assert_eq!(diagnose(4, 4, 9, 8, 4), CoverageDiagnosis::LostAtReload);
assert_eq!(diagnose(4, 4, 0, 4, 4), CoverageDiagnosis::NoNewWork);
}
#[test]
fn checkpoint_fixture_missing_and_corrupt_inputs_fail() {
for bytes in [
vec![],
vec![0xc1],
DataUsageCache::default().marshal_msg().expect("empty cache encoding"),
vec![0; usize::try_from(MAX_CACHE_BYTES + 1).expect("oversized fixture")],
] {
assert!(decode_fixture(&bytes).is_err(), "invalid fixture must not become an empty complete root");
}
}
#[test]
fn checkpoint_fixture_compaction_preserves_aggregate_not_child_enumeration() {
let mut cache = DataUsageCache::default();
cache.info.name = "bucket".to_string();
cache.replace("bucket", "", DataUsageEntry::default());
cache.replace("bucket/static", "bucket", DataUsageEntry::default());
for index in 0..4 {
cache.replace(
&format!("bucket/static/{index}"),
"bucket/static",
DataUsageEntry {
objects: 1,
..Default::default()
},
);
}
cache.reduce_children_of(&hash_path("bucket/static"), 1, true);
let decoded = decode_fixture(&cache.marshal_msg().expect("encode compacted cache")).expect("decode compacted fixture");
let entry = decoded
.find("bucket/static")
.expect("compaction must retain the static subtree root");
assert!(entry.compacted);
assert!(entry.children.is_empty());
assert_eq!(
retained(&decoded),
4,
"compaction retains aggregate coverage even when leaf keys are absent"
);
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_save_reload_resume() {
run_checkpoint_fixture(false).await;
}
#[tokio::test]
#[serial]
async fn checkpoint_fixture_hot_digest_diagnostic() {
run_checkpoint_fixture(true).await;
}
async fn run_checkpoint_fixture(change_digest: bool) {
let (scanner, root) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(root.clone()),
};
for index in 0..STATIC_OBJECTS {
write_test_object_metadata(&root, "bucket", &format!("static/{index:04}")).await;
}
let store = FixtureStore::new();
let mut previous = 0;
let mut visited = 0;
for round in 0..3_u8 {
write_test_object_metadata(&root, "bucket", "hot/current").await;
let mut cache = DataUsageCache::default();
let revisions = cache
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("load checkpoint revisions");
if round > 0 {
assert_eq!(retained(&store.strict_load().await), previous);
}
let plan = crate::scanner_io::checkpoint_fixture_bucket_digest(PLAN, change_digest.then_some(u64::from(round)));
crate::scanner_io::current_cache_root_or_prepare_with_generation(
&mut cache,
"bucket",
SOURCE,
11,
7,
plan,
crate::scanner_io::DataUsageCacheReuseOptions {
require_source: true,
tier_registry_generation: None,
},
);
let prepared = retained(&cache);
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(4),
..Default::default()
},
);
let outcome = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget.clone(),
vec![scanner.local_disk.clone()],
cache,
None,
HealScanMode::Normal,
)
.await
.expect("budgeted local disk scan returns partial cache");
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
panic!("budgeted fixture must remain partial")
};
assert!(!cache.info.snapshot_complete, "partial must never publish a complete root");
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
let scanned = retained(&cache);
cache
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect("persist partial checkpoint");
let mut loaded = DataUsageCache::default();
loaded
.load(store.clone(), CACHE_NAME)
.await
.expect("reload persisted partial checkpoint");
let reloaded = retained(&loaded);
assert_eq!(reloaded, retained(&store.strict_load().await));
assert_eq!(scanned, reloaded, "save/load must retain static subtree coverage");
assert!(!loaded.info.snapshot_complete);
visited += budget.entries_visited();
let diagnosis = diagnose(previous, prepared, budget.entries_visited(), scanned, reloaded);
eprintln!(
"checkpoint_fixture round={round} hot_digest={change_digest} visited_total={visited} before={previous} prepared={prepared} scanned={scanned} reloaded={reloaded} diagnosis={diagnosis:?}"
);
if !change_digest || std::env::var_os("RUSTFS_CHECKPOINT_REQUIRE_PROGRESS").is_some() {
assert_eq!(
diagnosis,
CoverageDiagnosis::Progress,
"visited growth must produce durable static coverage"
);
}
crate::remote_scanner::checkpoint_fixture_partial_return(budget.progress(), budget.entries_visited()).await;
previous = reloaded;
}
assert!(visited > 0, "fixture must exercise the directory walk");
assert!(previous > 0, "fixture must retain and enumerate static subtree entries");
let mut loaded = DataUsageCache::default();
let revisions = loaded
.load_with_revisions(store.clone(), CACHE_NAME)
.await
.expect("load final checkpoint");
let before = tokio::fs::read(store.root.path().join("main"))
.await
.expect("read durable checkpoint bytes");
let epoch_error = loaded
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1)
.await
.expect_err("stale publication epoch must reject persistence");
assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED));
store.reject_save.store(true, Ordering::SeqCst);
loaded.info.next_cycle += 1;
loaded
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
.await
.expect_err("injected save failure must not report durable progress");
assert_eq!(
tokio::fs::read(store.root.path().join("main"))
.await
.expect("read unchanged checkpoint bytes"),
before
);
let parent = CancellationToken::new();
parent.cancel();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let result = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget.clone(),
vec![scanner.local_disk.clone()],
loaded.clone(),
None,
HealScanMode::Normal,
)
.await;
assert!(result.is_err(), "pre-scan cancellation must not produce a complete root");
assert_eq!(budget.reason(), None, "parent cancellation is not object budget exhaustion");
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let result = scanner
.local_disk
.clone()
.nsscanner_disk(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
loaded,
None,
HealScanMode::Normal,
)
.await
.expect("unbounded scan must complete after durable partial progress");
let ScannerDiskScanOutcome::Complete(cache) = result else {
panic!("unbounded fixture must produce a complete disk cache");
};
assert!(cache.info.snapshot_complete);
assert!(cache.info.scan_checkpoint.is_none());
assert_eq!(
cache.checked_flatten("bucket").expect("complete bucket root").objects,
usize::try_from(STATIC_OBJECTS + 1).expect("fixture object count fits usize")
);
}
crates/scanner/src/scanner_io.rs
+129 -3
View File
@@ -21,6 +21,7 @@ use crate::{
DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState,
ScannerError, SizeSummary, TierStats,
};
use bytes::Bytes;
use futures::future::join_all;
use metrics::counter;
use rand::seq::SliceRandom as _;
@@ -54,6 +55,7 @@ use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, error, warn};
use crate::ScannerObjectInfo as ObjectInfo;
use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
use crate::storage_api::ScannerStorage;
use crate::storage_api::scan::NamespaceLocking as _;
use crate::storage_api::scanner_io::{BucketInfo, BucketOptions};
@@ -111,6 +113,121 @@ pub(crate) struct ScannerBucketScanScope {
baseline_scan_plan_digest: Option<DataUsageScanPlanDigest>,
}
impl ScannerBucketScanScope {
fn is_default(&self) -> bool {
self.selected_buckets.is_none() && self.baseline_scan_plan_digest.is_none()
}
fn from_dirty_buckets(selected_buckets: HashSet<String>, baseline_scan_plan_digest: DataUsageScanPlanDigest) -> Self {
Self {
selected_buckets: Some(Arc::new(selected_buckets)),
baseline_scan_plan_digest: Some(baseline_scan_plan_digest),
}
}
}
#[derive(Clone, Copy)]
pub(super) struct ScannerCacheBaselineProof<'a> {
pub(super) data: Option<&'a Bytes>,
pub(super) expected_sources: &'a HashSet<DataUsageCacheSource>,
pub(super) leader_epoch: u64,
pub(super) want_cycle: u64,
pub(super) scan_plan_digest: DataUsageScanPlanDigest,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct ScannerPeerDirtyUsageExpectation {
instance_id: String,
generation: u64,
pending: bool,
}
fn verified_remote_dirty_usage_buckets(
expected_peers: &HashMap<String, ScannerPeerDirtyUsageExpectation>,
peer_snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>,
) -> Option<HashSet<String>> {
if expected_peers.is_empty() || peer_snapshots.len() != expected_peers.len() {
return None;
}
let mut received_peers = HashSet::with_capacity(peer_snapshots.len());
let mut dirty_buckets = HashSet::new();
for (host, snapshot) in peer_snapshots {
let expected = expected_peers.get(&host)?;
if !received_peers.insert(host)
|| snapshot.instance_id != expected.instance_id
|| snapshot.generation != expected.generation
|| snapshot.generation == u64::MAX
|| snapshot.protocol_version != crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION
|| !snapshot.complete
|| snapshot.pending_bucket_count != u64::try_from(snapshot.buckets.len()).unwrap_or(u64::MAX)
|| (expected.pending && snapshot.pending_bucket_count == 0)
{
return None;
}
dirty_buckets.extend(snapshot.buckets.into_keys());
}
(received_peers.len() == expected_peers.len()).then_some(dirty_buckets)
}
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
let data = proof.data?;
let baseline = serde_json::from_slice::<DataUsageInfo>(data).ok()?;
if !baseline.is_complete_bucket_usage_snapshot()
|| baseline.usage_snapshot_partial
|| baseline.usage_snapshot_converged != Some(true)
|| baseline.scanner_epoch != Some(proof.leader_epoch)
|| baseline.usage_snapshot_set_states.len() != proof.expected_sources.len()
{
return None;
}
let mut states = HashSet::with_capacity(baseline.usage_snapshot_set_states.len());
for state in &baseline.usage_snapshot_set_states {
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
if !proof.expected_sources.contains(&source)
|| !states.insert(source)
|| !state.complete
|| state.tombstone
|| state.scanner_epoch != Some(proof.leader_epoch)
|| state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle)
|| state.scan_plan_digest != Some(proof.scan_plan_digest.0)
{
return None;
}
}
(states == *proof.expected_sources).then_some(proof.scan_plan_digest)
}
fn scoped_scan_scope_from_dirty_buckets(
requested_scope: ScannerBucketScanScope,
dirty_buckets: HashSet<String>,
dirty_snapshot_complete: bool,
all_buckets: &[BucketInfo],
baseline_proof: ScannerCacheBaselineProof<'_>,
) -> ScannerBucketScanScope {
if !requested_scope.is_default() || !dirty_snapshot_complete {
return requested_scope;
}
let current_buckets = all_buckets.iter().map(|bucket| bucket.name.as_str()).collect::<HashSet<_>>();
let selected_buckets = dirty_buckets
.into_iter()
.filter(|bucket| current_buckets.contains(bucket.as_str()))
.collect::<HashSet<_>>();
if selected_buckets.is_empty() {
return requested_scope;
}
let Some(baseline_scan_plan_digest) = complete_scanner_cache_baseline_plan_digest(baseline_proof) else {
return requested_scope;
};
ScannerBucketScanScope::from_dirty_buckets(selected_buckets, baseline_scan_plan_digest)
}
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
}
@@ -209,6 +326,14 @@ fn scanner_bucket_cache_digest(
DataUsageScanPlanDigest(hasher.finalize().into())
}
#[cfg(test)]
pub(crate) fn checkpoint_fixture_bucket_digest(
scan_plan_digest: DataUsageScanPlanDigest,
dirty_generation: Option<u64>,
) -> DataUsageScanPlanDigest {
scanner_bucket_cache_digest(scan_plan_digest, dirty_generation)
}
fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error>) -> Result<()> {
if results.iter().any(|result| result.info.last_update.is_some()) {
return Ok(());
@@ -741,7 +866,7 @@ mod io_cache;
mod io_cycle;
#[cfg(test)]
use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan};
pub(crate) use io_cycle::nsscanner_with_storage_status;
pub(crate) use io_cycle::{ScannerCycleRequest, nsscanner_with_storage_status_scoped};
mod io_disk;
#[cfg(test)]
mod publish_gate_tests;
@@ -758,8 +883,9 @@ pub(crate) use cache::{
};
pub use dirty_usage::{
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
scanner_maintenance_generation,
};
#[cfg(test)]
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
crates/scanner/src/scanner_io/cache.rs
+18
View File
@@ -282,9 +282,26 @@ pub(super) fn completed_data_usage_info(
.iter()
.map(|(bucket, usage)| (bucket.clone(), usage.size))
.collect();
let mut usage_snapshot_set_states = results
.iter()
.map(|result| {
let source = result.info.source?;
Some(DataUsageSnapshotSetState {
pool_index: u64::try_from(source.pool_index).ok()?,
set_index: u64::try_from(source.set_index).ok()?,
scanner_cycle: Some(result.info.next_cycle),
scanner_epoch: Some(result.info.leader_epoch),
scan_plan_digest: Some(result.info.scan_plan_digest?.0),
complete: true,
tombstone: false,
})
})
.collect::<Option<Vec<_>>>()?;
usage_snapshot_set_states.sort_by_key(|state| (state.pool_index, state.set_index));
let data_usage_info = DataUsageInfo {
last_update: Some(merged_last_update),
scanner_cycle: Some(results.first()?.info.next_cycle),
scanner_epoch: Some(results.first()?.info.leader_epoch),
objects_total_count: u64::try_from(total.objects).ok()?,
versions_total_count: u64::try_from(total.versions).ok()?,
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
@@ -295,6 +312,7 @@ pub(super) fn completed_data_usage_info(
bucket_sizes,
buckets_usage,
usage_snapshot_complete: true,
usage_snapshot_set_states,
..Default::default()
};
Some((data_usage_info, merged_last_update))
crates/scanner/src/scanner_io/dirty_usage.rs
+106
View File
@@ -52,6 +52,112 @@ pub enum ScannerDirtyUsageAckError {
ProcessChanged,
#[error("scanner dirty usage generation cannot be acknowledged")]
InvalidGeneration,
#[error("scanner dirty usage bucket incarnation fence is unavailable")]
IncarnationUnavailable,
}
/// A scoped ACK requires storage-owned lifecycle and incarnation fences.
/// Callers must only send ACKs backed by durable per-bucket publication.
pub fn acknowledge_scoped_dirty_usage(
instance_id: &str,
entries: &[(&crate::storage_api::EcstoreBucketMetadataMutationGuard, u64)],
probe_only: bool,
) -> std::result::Result<u64, ScannerDirtyUsageAckError> {
// Lock order: sorted bucket lifecycle/metadata fences (caller), then dirty map.
// No await or storage operation occurs while the dirty map is locked.
let (cleared, pending) = {
let mut dirty = dirty_usage_buckets();
let checked = entries
.iter()
.map(|(guard, generation)| {
guard
.checked_bucket_incarnation()
.map(|(bucket, _)| (bucket, *generation))
.map_err(|_| ScannerDirtyUsageAckError::IncarnationUnavailable)
})
.collect::<std::result::Result<Vec<_>, _>>()?;
let cleared = apply_scoped_dirty_usage_ack(
instance_id,
scanner_activity_epoch(),
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire),
&mut dirty,
&checked,
probe_only,
)?;
if cleared > 0 {
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
}
(cleared, dirty.len())
};
if !probe_only {
global_metrics().record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared), usize_to_u64_saturated(pending));
}
Ok(usize_to_u64_saturated(cleared))
}
fn apply_scoped_dirty_usage_ack(
instance_id: &str,
current_instance: &str,
current_generation: u64,
dirty: &mut DirtyUsageBuckets,
entries: &[(&str, u64)],
probe_only: bool,
) -> std::result::Result<usize, ScannerDirtyUsageAckError> {
if instance_id != current_instance {
return Err(ScannerDirtyUsageAckError::ProcessChanged);
}
if current_generation == u64::MAX
|| entries
.iter()
.any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation)
{
return Err(ScannerDirtyUsageAckError::InvalidGeneration);
}
let mut cleared = 0;
if !probe_only {
for (bucket, generation) in entries {
if dirty.get(*bucket) == Some(generation) {
dirty.remove(*bucket);
cleared += 1;
}
}
}
Ok(cleared)
}
#[cfg(test)]
mod scoped_dirty_usage_tests {
use super::*;
#[test]
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
let mut dirty = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], true), Ok(0));
assert_eq!(dirty.len(), 2);
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(1));
assert_eq!(dirty.get("hot"), Some(&7));
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(0));
dirty.insert("cold".to_string(), 9);
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 9, &mut dirty, &[("cold", 8)], false), Ok(0));
assert_eq!(dirty.get("cold"), Some(&9));
}
#[test]
fn scoped_dirty_usage_rejects_restart_and_invalid_batch_before_clearing() {
let original = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
let mut dirty = original.clone();
assert_eq!(
apply_scoped_dirty_usage_ack("old", "new", 8, &mut dirty, &[("cold", 8)], false),
Err(ScannerDirtyUsageAckError::ProcessChanged)
);
for generation in [0, 9, u64::MAX] {
assert_eq!(
apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8), ("hot", generation)], false),
Err(ScannerDirtyUsageAckError::InvalidGeneration)
);
assert_eq!(dirty, original);
}
}
}
pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
crates/scanner/src/scanner_io/io_cycle.rs
+94 -1
View File
@@ -71,6 +71,7 @@ where
leader_epoch,
scan_mode,
scan_scope: ScannerBucketScanScope::default(),
persisted_usage_baseline: None,
};
nsscanner_with_storage_status_scoped(store, request).await
}
@@ -83,6 +84,79 @@ pub(crate) struct ScannerCycleRequest {
pub(crate) leader_epoch: u64,
pub(crate) scan_mode: HealScanMode,
pub(crate) scan_scope: ScannerBucketScanScope,
pub(crate) persisted_usage_baseline: Option<Bytes>,
}
struct ScannerBucketScopeResolution<'a> {
requested_scope: ScannerBucketScanScope,
baseline_proof: ScannerCacheBaselineProof<'a>,
activity_before: &'a crate::scanner::ScannerActivitySnapshot,
dirty_usage_snapshot: &'a DirtyUsageSnapshot,
all_buckets: &'a [BucketInfo],
}
async fn resolve_scanner_bucket_scan_scope<S>(
store: &S,
distributed: bool,
resolution: ScannerBucketScopeResolution<'_>,
) -> ScannerBucketScanScope
where
S: ScannerStorage,
{
if !resolution.requested_scope.is_default()
|| !resolution.dirty_usage_snapshot.covers_all_pending
|| resolution.dirty_usage_snapshot.generation == u64::MAX
|| resolution.dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES
{
return resolution.requested_scope;
}
let mut dirty_buckets = resolution
.dirty_usage_snapshot
.buckets
.keys()
.cloned()
.collect::<HashSet<_>>();
if distributed {
let Some(notification_system) = store.scanner_notification_system() else {
return resolution.requested_scope;
};
let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else {
return resolution.requested_scope;
};
let mut expected_peers = HashMap::new();
for (host, lease_instance_id, _) in crate::scanner::scanner_activity_publication_lease_targets(resolution.activity_before)
{
let Some((activity_instance_id, generation, pending)) =
crate::scanner::scanner_activity_dirty_usage_state_for_host(resolution.activity_before, &host)
else {
return resolution.requested_scope;
};
if activity_instance_id != lease_instance_id || expected_peers.contains_key(&host) {
return resolution.requested_scope;
}
expected_peers.insert(
host,
ScannerPeerDirtyUsageExpectation {
instance_id: activity_instance_id.to_string(),
generation,
pending,
},
);
}
let Some(remote_dirty_buckets) = verified_remote_dirty_usage_buckets(&expected_peers, peer_snapshots) else {
return resolution.requested_scope;
};
dirty_buckets.extend(remote_dirty_buckets);
}
scoped_scan_scope_from_dirty_buckets(
resolution.requested_scope,
dirty_buckets,
true,
resolution.all_buckets,
resolution.baseline_proof,
)
}
pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
@@ -97,6 +171,7 @@ where
leader_epoch,
scan_mode,
scan_scope,
persisted_usage_baseline,
} = request;
let child_token = ctx.child_token();
let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch);
@@ -186,8 +261,26 @@ where
}
bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
let scan_plan_digest =
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before));
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
let scan_scope = resolve_scanner_bucket_scan_scope(
store,
distributed,
ScannerBucketScopeResolution {
requested_scope: scan_scope,
baseline_proof: ScannerCacheBaselineProof {
data: persisted_usage_baseline.as_ref(),
expected_sources: &expected_sources,
leader_epoch,
want_cycle,
scan_plan_digest,
},
activity_before: &activity_before,
dirty_usage_snapshot: &dirty_usage_snapshot,
all_buckets: &all_buckets,
},
)
.await;
let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle));
let tier_registry = runtime_tier_registry_for_cycle(want_cycle, leader_epoch).await;
let tier_registry_generation = tier_registry.generation;
crates/scanner/src/scanner_io/publish_gate_tests.rs
+22
View File
@@ -655,9 +655,31 @@ fn completed_data_usage_info_requires_every_set_before_publish() {
.expect("all completed sets should produce a publishable data usage snapshot");
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
assert_eq!(data_usage_info.scanner_cycle, Some(0));
assert_eq!(data_usage_info.scanner_epoch, Some(0));
assert_eq!(data_usage_info.objects_total_count, 3);
assert_eq!(data_usage_info.buckets_usage.len(), 3);
assert!(data_usage_info.usage_snapshot_complete);
assert_eq!(
data_usage_info
.usage_snapshot_set_states
.iter()
.map(|state| {
(
state.pool_index,
state.set_index,
state.scanner_cycle,
state.scanner_epoch,
state.scan_plan_digest,
state.complete,
state.tombstone,
)
})
.collect::<Vec<_>>(),
vec![
(0, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false),
(1, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false),
]
);
assert_eq!(
data_usage_info
.buckets_usage
crates/scanner/src/scanner_io/tests.rs
+211
View File
@@ -17,6 +17,7 @@ use super::io_disk::tier_stats_template;
use super::*;
use crate::scanner_budget::ScannerCycleBudgetConfig;
use crate::scanner_folder::ScannerItem;
use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
use crate::storage_api::owner::{
EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
};
@@ -796,6 +797,195 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa
cache
}
fn complete_usage_baseline(
source: DataUsageCacheSource,
scan_plan_digest: DataUsageScanPlanDigest,
scanner_cycle: u64,
scanner_epoch: u64,
) -> bytes::Bytes {
let baseline = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
scanner_cycle: Some(scanner_cycle),
scanner_epoch: Some(scanner_epoch),
buckets_count: 1,
buckets_usage: HashMap::from([("photos".to_string(), Default::default())]),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(true),
usage_snapshot_set_states: vec![DataUsageSnapshotSetState {
pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"),
set_index: u64::try_from(source.set_index).expect("test set index should fit"),
scanner_cycle: Some(scanner_cycle),
scanner_epoch: Some(scanner_epoch),
scan_plan_digest: Some(scan_plan_digest.0),
complete: true,
tombstone: false,
}],
..Default::default()
};
bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode"))
}
#[test]
fn scoped_scan_requires_a_converged_complete_baseline_with_exact_set_provenance() {
let source = DataUsageCacheSource::new(1, 2);
let expected_sources = HashSet::from([source]);
let scan_plan_digest = DataUsageScanPlanDigest([9; 32]);
let baseline = complete_usage_baseline(source, scan_plan_digest, 7, 11);
assert_eq!(
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
data: Some(&baseline),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest,
}),
Some(scan_plan_digest)
);
let mut incomplete = serde_json::from_slice::<DataUsageInfo>(&baseline).expect("test baseline should decode");
incomplete.usage_snapshot_converged = Some(false);
let incomplete = bytes::Bytes::from(serde_json::to_vec(&incomplete).expect("test baseline should encode"));
assert_eq!(
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
data: Some(&incomplete),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest,
}),
None
);
let mut wrong_provenance = serde_json::from_slice::<DataUsageInfo>(&baseline).expect("test baseline should decode");
wrong_provenance.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32]);
let wrong_provenance = bytes::Bytes::from(serde_json::to_vec(&wrong_provenance).expect("test baseline should encode"));
assert_eq!(
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
data: Some(&wrong_provenance),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest,
}),
None
);
}
#[test]
fn scoped_scan_selects_only_current_dirty_buckets_after_baseline_validation() {
let source = DataUsageCacheSource::new(1, 2);
let expected_sources = HashSet::from([source]);
let baseline_scan_plan_digest = DataUsageScanPlanDigest([4; 32]);
let current_scan_plan_digest = DataUsageScanPlanDigest([5; 32]);
let baseline = complete_usage_baseline(source, current_scan_plan_digest, 7, 11);
let scope = scoped_scan_scope_from_dirty_buckets(
ScannerBucketScanScope::default(),
HashSet::from(["photos".to_string(), "deleted".to_string()]),
true,
&[bucket_info("photos")],
ScannerCacheBaselineProof {
data: Some(&baseline),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest: current_scan_plan_digest,
},
);
assert_eq!(scope.baseline_scan_plan_digest, Some(current_scan_plan_digest));
assert_eq!(
scope
.selected_buckets
.as_deref()
.expect("validated scope should select a bucket"),
&HashSet::from(["photos".to_string()])
);
assert_ne!(scope.baseline_scan_plan_digest, Some(baseline_scan_plan_digest));
}
fn peer_dirty_usage_snapshot(
instance_id: &str,
generation: u64,
complete: bool,
buckets: &[(&str, u64)],
) -> EcstoreScannerPeerDirtyUsageSnapshot {
EcstoreScannerPeerDirtyUsageSnapshot {
instance_id: instance_id.to_string(),
generation,
pending_bucket_count: u64::try_from(buckets.len()).expect("test bucket count should fit"),
protocol_version: crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
complete,
buckets: buckets
.iter()
.map(|(bucket, generation)| ((*bucket).to_string(), *generation))
.collect(),
}
}
#[test]
fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots() {
let expected_peers = HashMap::from([
(
"node-a:9000".to_string(),
ScannerPeerDirtyUsageExpectation {
instance_id: "instance-a".to_string(),
generation: 7,
pending: true,
},
),
(
"node-b:9000".to_string(),
ScannerPeerDirtyUsageExpectation {
instance_id: "instance-b".to_string(),
generation: 3,
pending: false,
},
),
]);
assert_eq!(
verified_remote_dirty_usage_buckets(
&expected_peers,
vec![
(
"node-a:9000".to_string(),
peer_dirty_usage_snapshot("instance-a", 7, true, &[("photos", 7)]),
),
(
"node-b:9000".to_string(),
peer_dirty_usage_snapshot("instance-b", 3, true, &[("archive", 3)]),
),
],
),
Some(HashSet::from(["photos".to_string(), "archive".to_string()]))
);
}
#[test]
fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state() {
let expected_peers = HashMap::from([(
"node-a:9000".to_string(),
ScannerPeerDirtyUsageExpectation {
instance_id: "instance-a".to_string(),
generation: 7,
pending: true,
},
)]);
for snapshot in [
peer_dirty_usage_snapshot("instance-a", 7, false, &[("photos", 7)]),
peer_dirty_usage_snapshot("instance-a", 6, true, &[("photos", 6)]),
peer_dirty_usage_snapshot("instance-b", 7, true, &[("photos", 7)]),
peer_dirty_usage_snapshot("instance-a", 7, true, &[]),
] {
assert!(
verified_remote_dirty_usage_buckets(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(),
"incomplete, stale, mismatched, or empty pending peer state must fall back to a full scan"
);
}
}
#[test]
fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
@@ -1048,6 +1238,27 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
}
}
#[test]
fn checkpoint_fixture_superseded_is_distinct_from_partial_and_cancel() {
for (budget, cancelled, bucket, expected) in [
(false, false, ScannerBucketScanStatus::Complete, ScannerCycleStatus::Superseded),
(true, false, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
(false, true, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
] {
assert_eq!(
classify_nsscanner_cycle(
true,
budget,
cancelled,
bucket,
DirtyUsageSnapshotStatus::Changed,
ScannerCycleActivityStatus::Unchanged
),
expected,
);
}
}
#[test]
fn unverified_activity_defers_partial_and_floor_cycles() {
let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
crates/scanner/src/storage_api.rs
+5 -3
View File
@@ -38,8 +38,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::object_opts_fr
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
get_replication_config as ecstore_get_replication_config,
BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard, get_lifecycle_config as ecstore_get_lifecycle_config,
get_object_lock_config as ecstore_get_object_lock_config, get_replication_config as ecstore_get_replication_config,
};
pub(crate) use rustfs_ecstore::api::bucket::replication::{
ReplicateObjectInfo, ReplicationConfig as EcstoreReplicationConfig,
@@ -103,7 +103,9 @@ pub(crate) use rustfs_ecstore::api::rebalance::{
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
RebalanceStats as EcstoreRebalanceStats,
};
pub(crate) use rustfs_ecstore::api::rpc::ScannerBucketListing as EcstoreScannerBucketListing;
pub(crate) use rustfs_ecstore::api::rpc::{
ScannerBucketListing as EcstoreScannerBucketListing, ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
pub(crate) use rustfs_ecstore::api::runtime::{
crates/zip/Cargo.toml
+9
View File
@@ -49,5 +49,14 @@ rustfs-rio.workspace = true
tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
thiserror = { workspace = true }
[dev-dependencies]
astral-tokio-tar = { workspace = true }
futures = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
sha2 = { workspace = true }
tar-codec = { workspace = true }
tar-framing = { workspace = true }
[lints]
workspace = true
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/README.md
Vendored
+24
View File
@@ -0,0 +1,24 @@
# minio-go Snowball fixtures
These request bodies are generated by
`github.com/minio/minio-go/v7.Client.PutObjectsSnowball` at the version pinned
in `generate/go.mod`. They cover the raw TAR and S2-compressed forms accepted by
RustFS Snowball extraction.
The decoded TAR intentionally ends immediately after the final padded member
body because minio-go flushes, rather than closes, its TAR writer. The
compatibility test permits that shape only when the authenticated request body
is complete at the exact member boundary; it does not make incomplete TAR
terminators generally valid.
Regenerate them from this directory with Go 1.25:
```console
cd generate
go mod download
go run . -out ..
```
`manifest.json` records the input objects and SHA-256 digest of each captured
request body. Review changes to the manifest and binary fixtures together when
updating minio-go.
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.mod
Vendored
+26
View File
@@ -0,0 +1,26 @@
module rustfs.local/snowball-fixture
go 1.25.0
require github.com/minio/minio-go/v7 v7.3.0
require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.19.2 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.41.0 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
)
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/go.sum
Vendored
+59
View File
@@ -0,0 +1,59 @@
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU=
github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/generate/main.go
Vendored
+193
View File
@@ -0,0 +1,193 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
const minioGoVersion = "v7.3.0"
type fixtureManifest struct {
Generator string `json:"generator"`
MinioGo string `json:"minio_go"`
GeneratedAt string `json:"generated_at"`
Objects []fixtureObject `json:"objects"`
Archives []fixtureArchive `json:"archives"`
}
type fixtureObject struct {
Key string `json:"key"`
Body string `json:"body"`
ModTime string `json:"mod_time"`
VersionID string `json:"version_id,omitempty"`
Headers map[string][]string `json:"headers,omitempty"`
}
type fixtureArchive struct {
File string `json:"file"`
Compressed bool `json:"compressed"`
Length int `json:"length"`
SHA256 string `json:"sha256"`
}
func objects() []fixtureObject {
return []fixtureObject{
{
Key: "alpha.txt",
Body: "alpha-body",
ModTime: "2024-01-02T03:04:05Z",
VersionID: "018cc251-f400-7c22-9e8d-8b1800000001",
Headers: map[string][]string{
"Content-Type": {"text/plain"},
"X-Amz-Meta-Owner": {"snowball-fixture"},
"X-Amz-Tagging": {"project=rustfs&source=minio-go"},
},
},
{
Key: "nested/世界.txt",
Body: "bravo-body",
ModTime: "2024-01-02T03:05:05Z",
Headers: map[string][]string{
"Content-Language": {"zh-CN"},
"X-Amz-Meta-Note": {"unicode-path"},
},
},
}
}
func captureSnowball(compressed bool, specs []fixtureObject) ([]byte, error) {
body := make(chan []byte, 1)
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
payload, err := io.ReadAll(request.Body)
if err != nil {
http.Error(writer, err.Error(), http.StatusInternalServerError)
return
}
body <- payload
writer.Header().Set("ETag", `"snowball-fixture"`)
writer.WriteHeader(http.StatusOK)
}))
defer server.Close()
client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{
// The S3 authentication layer removes AWS streaming-signature framing
// before Snowball extraction sees the request body. Anonymous signing
// captures those decoded archive bytes directly.
Creds: credentials.NewStatic("", "", "", credentials.SignatureAnonymous),
Secure: false,
Region: "us-east-1",
})
if err != nil {
return nil, fmt.Errorf("construct minio client: %w", err)
}
input := make(chan minio.SnowballObject, len(specs))
for _, spec := range specs {
modTime, err := time.Parse(time.RFC3339, spec.ModTime)
if err != nil {
return nil, fmt.Errorf("parse mod time for %q: %w", spec.Key, err)
}
headers := make(http.Header, len(spec.Headers))
for name, values := range spec.Headers {
headers[name] = append([]string(nil), values...)
}
input <- minio.SnowballObject{
Key: spec.Key,
Size: int64(len(spec.Body)),
ModTime: modTime,
Content: bytes.NewReader([]byte(spec.Body)),
VersionID: spec.VersionID,
Headers: headers,
}
}
close(input)
err = client.PutObjectsSnowball(context.Background(), "fixture-bucket", minio.SnowballOptions{
Opts: minio.PutObjectOptions{
ContentType: "application/octet-stream",
},
InMemory: true,
Compress: compressed,
}, input)
if err != nil {
return nil, fmt.Errorf("generate snowball request: %w", err)
}
return <-body, nil
}
func main() {
outDir := flag.String("out", "..", "fixture output directory")
flag.Parse()
specs := objects()
archives := make([]fixtureArchive, 0, 2)
for _, fixture := range []struct {
name string
compressed bool
}{
{name: "snowball.tar"},
{name: "snowball.tar.s2", compressed: true},
} {
payload, err := captureSnowball(fixture.compressed, specs)
if err != nil {
panic(err)
}
path := filepath.Join(*outDir, fixture.name)
if err := os.WriteFile(path, payload, 0o644); err != nil {
panic(fmt.Errorf("write %s: %w", path, err))
}
digest := sha256.Sum256(payload)
archives = append(archives, fixtureArchive{
File: fixture.name,
Compressed: fixture.compressed,
Length: len(payload),
SHA256: hex.EncodeToString(digest[:]),
})
}
manifest := fixtureManifest{
Generator: "github.com/minio/minio-go/v7.Client.PutObjectsSnowball",
MinioGo: minioGoVersion,
GeneratedAt: "2026-09-05T00:00:00Z",
Objects: specs,
Archives: archives,
}
payload, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
panic(err)
}
payload = append(payload, '\n')
path := filepath.Join(*outDir, "manifest.json")
if err := os.WriteFile(path, payload, 0o644); err != nil {
panic(fmt.Errorf("write %s: %w", path, err))
}
}
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/manifest.json
Vendored
+51
View File
@@ -0,0 +1,51 @@
{
"generator": "github.com/minio/minio-go/v7.Client.PutObjectsSnowball",
"minio_go": "v7.3.0",
"generated_at": "2026-09-05T00:00:00Z",
"objects": [
{
"key": "alpha.txt",
"body": "alpha-body",
"mod_time": "2024-01-02T03:04:05Z",
"version_id": "018cc251-f400-7c22-9e8d-8b1800000001",
"headers": {
"Content-Type": [
"text/plain"
],
"X-Amz-Meta-Owner": [
"snowball-fixture"
],
"X-Amz-Tagging": [
"project=rustfs\u0026source=minio-go"
]
}
},
{
"key": "nested/世界.txt",
"body": "bravo-body",
"mod_time": "2024-01-02T03:05:05Z",
"headers": {
"Content-Language": [
"zh-CN"
],
"X-Amz-Meta-Note": [
"unicode-path"
]
}
}
],
"archives": [
{
"file": "snowball.tar",
"compressed": false,
"length": 4096,
"sha256": "f00f2789dcb65b567f722f49cfdac9705e7bdac6c0badae75194327c32193d2e"
},
{
"file": "snowball.tar.s2",
"compressed": true,
"length": 528,
"sha256": "f8a9d9aa9b9ccdfae24ded1bff3741aacb935f1457a252efc9266674ff13c992"
}
]
}
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar
Vendored
BIN
View File
Binary file not shown.
crates/zip/tests/fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2
Vendored
BIN
View File
Binary file not shown.
crates/zip/tests/snowball_tar_codec_compat.rs
+548
View File
@@ -0,0 +1,548 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::io::Cursor;
use futures::StreamExt;
use rustfs_zip::CompressionFormat;
use serde::Deserialize;
use sha2::{Digest, Sha256};
use tar_codec::{Archive as _, DecodePolicy, Member, MemberPayload as _, PaxDecodePolicy, PaxVendorExtensionPolicy, TarArchive};
use tar_framing::{
FrameError, FrameErrorInner, PaxKeyword, PaxRecord, PaxValue, StreamPolicy, UstarKind,
logical::{MemberExtensions, PaxState, TarReader},
};
use tokio::io::AsyncReadExt;
const FIXTURE_ROOT: &str = "fixtures/snowball/minio-go-v7.3.0";
const RAW_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar");
const S2_FIXTURE: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/snowball.tar.s2");
const MANIFEST: &[u8] = include_bytes!("fixtures/snowball/minio-go-v7.3.0/manifest.json");
#[derive(Debug, Deserialize)]
struct FixtureManifest {
generator: String,
minio_go: String,
generated_at: String,
objects: Vec<FixtureObject>,
archives: Vec<FixtureArchive>,
}
#[derive(Debug, Deserialize)]
struct FixtureObject {
key: String,
body: String,
mod_time: String,
#[serde(default)]
version_id: String,
#[serde(default)]
headers: BTreeMap<String, Vec<String>>,
}
#[derive(Debug, Deserialize)]
struct FixtureArchive {
file: String,
compressed: bool,
length: usize,
sha256: String,
}
#[derive(Debug, Eq, PartialEq)]
struct ParsedMember {
path: String,
size: u64,
mtime: Option<u64>,
body: Vec<u8>,
minio_pax: BTreeMap<String, Option<Vec<u8>>>,
}
fn sha256_hex(bytes: &[u8]) -> String {
let mut encoded = String::with_capacity(64);
for byte in Sha256::digest(bytes) {
write!(&mut encoded, "{byte:02x}").expect("writing to a String should not fail");
}
encoded
}
async fn decode_s2(bytes: &[u8]) -> Vec<u8> {
let mut decoder = CompressionFormat::S2
.get_decoder(Cursor::new(bytes.to_vec()))
.expect("S2 fixture decoder should be available");
let mut decoded = Vec::new();
decoder.read_to_end(&mut decoded).await.expect("S2 fixture should decode");
decoded
}
async fn parse_with_tokio_tar(bytes: &[u8]) -> Vec<ParsedMember> {
let mut archive = tokio_tar::Archive::new(Cursor::new(bytes.to_vec()));
let mut entries = archive.entries().expect("tokio-tar should create an entry stream");
let mut parsed = Vec::new();
while let Some(entry) = entries.next().await {
let mut entry = entry.expect("tokio-tar should parse the fixture member");
let kind = entry.header().entry_type();
if kind == tokio_tar::EntryType::XGlobalHeader {
continue;
}
let path_bytes = entry.path_bytes().expect("tokio-tar should resolve the fixture path");
let path = std::str::from_utf8(path_bytes.as_ref())
.expect("fixture paths should be UTF-8")
.to_owned();
let size = entry.effective_size();
let mtime = entry.header().mtime().ok();
let mut minio_pax = BTreeMap::new();
if let Some(extensions) = entry
.pax_extensions()
.await
.expect("tokio-tar should parse local PAX records")
{
for extension in extensions {
let extension = extension.expect("fixture PAX record should be valid");
let key = extension.key().expect("fixture PAX keys should be UTF-8");
if key.starts_with("minio.") {
minio_pax.insert(key.to_owned(), Some(extension.value_bytes().to_vec()));
}
}
}
let mut body = Vec::new();
entry
.read_to_end(&mut body)
.await
.expect("tokio-tar should read the fixture body");
parsed.push(ParsedMember {
path,
size,
mtime,
body,
minio_pax,
});
}
parsed
}
fn effective_minio_pax(state: &PaxState<'_>, known_keywords: &mut Vec<PaxKeyword>) -> BTreeMap<String, Option<Vec<u8>>> {
for extension in state.extensions() {
for record in extension.records() {
let keyword = record.keyword();
if matches!(&keyword, PaxKeyword::Vendor { vendor, .. } if vendor.as_ref() == "minio")
&& !known_keywords.contains(&keyword)
{
known_keywords.push(keyword);
}
}
}
known_keywords
.iter()
.filter_map(|keyword| {
let record = state.effective_record(keyword)?;
let PaxRecord::Vendor { vendor, name, value } = record else {
return None;
};
let key = format!("{vendor}.{name}");
let value = match value {
PaxValue::Value(value) => Some(value.to_vec()),
PaxValue::Deleted => None,
};
Some((key, value))
})
.collect()
}
fn effective_mtime(header_mtime: Option<u64>, extensions: &MemberExtensions<'_>) -> Option<u64> {
let MemberExtensions::Pax(state) = extensions else {
return header_mtime;
};
match state.effective_record(&PaxKeyword::Mtime) {
Some(PaxRecord::Mtime(PaxValue::Value(value))) => Some(*value),
Some(PaxRecord::Mtime(PaxValue::Deleted)) => None,
_ => header_mtime,
}
}
fn padded_member_end(position: u64, size: u64) -> u64 {
let padded_size = size.checked_add(511).expect("fixture member size should not overflow") / 512 * 512;
position
.checked_add(512)
.and_then(|position| position.checked_add(padded_size))
.expect("fixture member end should not overflow")
}
fn is_authenticated_footerless_end(error: &FrameError, last_member_end: Option<u64>, request_body_complete: bool) -> bool {
// The production gate must source `request_body_complete` from RustFS's
// length, checksum, and trailing-header validation state.
request_body_complete && matches!(&error.inner, FrameErrorInner::MissingEndMarker) && last_member_end == Some(error.position)
}
fn candidate_snowball_decode_policy() -> DecodePolicy {
DecodePolicy::default()
.allow_gnu(true)
.allow_all_nul_numeric_fields(true)
.max_gnu_extension_size(1_048_576)
.pax_policy(
PaxDecodePolicy::default()
.max_extension_size(1_048_576)
.max_global_extensions_size(67_108_864)
.allow_global_pax_extensions(false)
.allow_non_utf8_pax_vendor_values(false)
.allow_duplicate_pax_records(false)
.allow_global_pax_member_metadata(false)
.vendor_extension_policy(PaxVendorExtensionPolicy::ignore(["minio"])),
)
}
async fn parse_with_tar_framing(bytes: &[u8]) -> (Vec<ParsedMember>, Option<FrameError>, Option<u64>) {
let policy = StreamPolicy::default()
.max_pax_extension_size(1024 * 1024)
.max_global_pax_extensions_size(4 * 1024 * 1024)
.max_gnu_extension_size(128 * 1024);
let mut reader = TarReader::new(Cursor::new(bytes.to_vec())).with_policy(policy);
let mut parsed = Vec::new();
let mut known_minio_keywords = Vec::new();
let mut last_member_end = None;
loop {
let mut frame = match reader.next_frame().await {
Ok(Some(frame)) => frame,
Ok(None) => return (parsed, None, last_member_end),
Err(error) => return (parsed, Some(error), last_member_end),
};
assert_eq!(frame.header.kind, UstarKind::Regular);
let path = String::from_utf8(
frame
.effective_path()
.expect("tar-framing should resolve the fixture path")
.into_owned(),
)
.expect("fixture paths should be UTF-8");
let size = frame.header.effective_size;
let mtime = effective_mtime(frame.header.mtime, &frame.extensions);
let minio_pax = match &frame.extensions {
MemberExtensions::Pax(state) => effective_minio_pax(state, &mut known_minio_keywords),
MemberExtensions::Gnu { .. } => BTreeMap::new(),
};
let mut body = Vec::new();
let mut chunk = Vec::new();
while frame
.payload
.next_chunk(&mut chunk, 64 * 1024)
.await
.expect("tar-framing should read the fixture body")
{
body.extend_from_slice(&chunk);
}
last_member_end = Some(padded_member_end(frame.header.position, size));
parsed.push(ParsedMember {
path,
size,
mtime,
body,
minio_pax,
});
}
}
#[test]
fn checked_in_fixtures_match_the_minio_go_manifest() {
let manifest: FixtureManifest = serde_json::from_slice(MANIFEST).expect("fixture manifest should be valid JSON");
assert_eq!(manifest.generator, "github.com/minio/minio-go/v7.Client.PutObjectsSnowball");
assert_eq!(manifest.minio_go, "v7.3.0");
assert_eq!(manifest.generated_at, "2026-09-05T00:00:00Z");
assert_eq!(manifest.objects.len(), 2);
assert_eq!(manifest.objects[0].key, "alpha.txt");
assert_eq!(manifest.objects[0].body, "alpha-body");
assert_eq!(manifest.objects[0].mod_time, "2024-01-02T03:04:05Z");
assert_eq!(manifest.objects[0].version_id, "018cc251-f400-7c22-9e8d-8b1800000001");
assert_eq!(
manifest.objects[0].headers.get("X-Amz-Meta-Owner"),
Some(&vec!["snowball-fixture".to_owned()])
);
for archive in &manifest.archives {
let bytes = match archive.file.as_str() {
"snowball.tar" => RAW_FIXTURE,
"snowball.tar.s2" => S2_FIXTURE,
file => panic!("unexpected archive in {FIXTURE_ROOT}/manifest.json: {file}"),
};
assert_eq!(bytes.len(), archive.length);
assert_eq!(sha256_hex(bytes), archive.sha256);
assert_eq!(archive.compressed, archive.file.ends_with(".s2"));
}
}
#[tokio::test]
async fn minio_go_raw_and_s2_fixtures_have_identical_footerless_tar_data() {
assert_eq!(decode_s2(S2_FIXTURE).await, RAW_FIXTURE);
assert_eq!(RAW_FIXTURE.len() % 512, 0);
assert!(RAW_FIXTURE.len() >= 1024);
assert!(
!RAW_FIXTURE[RAW_FIXTURE.len() - 1024..].iter().all(|byte| *byte == 0),
"minio-go Flush output should not contain the standard two-block terminator"
);
}
#[tokio::test]
async fn tar_framing_matches_tokio_tar_before_rejecting_the_missing_terminator() {
let expected = parse_with_tokio_tar(RAW_FIXTURE).await;
let (actual, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await;
let error = error.expect("footerless minio-go fixture should fail strict termination");
assert_eq!(actual, expected);
assert_eq!(
actual,
[
ParsedMember {
path: "alpha.txt".to_owned(),
size: 10,
mtime: Some(1_704_164_645),
body: b"alpha-body".to_vec(),
minio_pax: BTreeMap::from([
("minio.metadata.Content-Type".to_owned(), Some(b"text/plain".to_vec()),),
("minio.metadata.X-Amz-Meta-Owner".to_owned(), Some(b"snowball-fixture".to_vec()),),
(
"minio.metadata.X-Amz-Tagging".to_owned(),
Some(b"project=rustfs&source=minio-go".to_vec()),
),
("minio.versionId".to_owned(), Some(b"018cc251-f400-7c22-9e8d-8b1800000001".to_vec()),),
]),
},
ParsedMember {
path: "nested/世界.txt".to_owned(),
size: 10,
mtime: Some(1_704_164_705),
body: b"bravo-body".to_vec(),
minio_pax: BTreeMap::from([
("minio.metadata.Content-Language".to_owned(), Some(b"zh-CN".to_vec()),),
("minio.metadata.X-Amz-Meta-Note".to_owned(), Some(b"unicode-path".to_vec()),),
]),
},
]
);
assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker));
assert_eq!(
error.position,
u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64")
);
assert_eq!(last_member_end, Some(error.position));
}
#[tokio::test]
async fn footerless_compatibility_requires_authenticated_eof_at_the_member_boundary() {
let (_, error, last_member_end) = parse_with_tar_framing(RAW_FIXTURE).await;
let error = error.expect("the real fixture should be footerless");
assert!(is_authenticated_footerless_end(&error, last_member_end, true));
assert!(!is_authenticated_footerless_end(&error, last_member_end, false));
let mut one_zero_block = RAW_FIXTURE.to_vec();
one_zero_block.extend([0; 512]);
let (_, error, last_member_end) = parse_with_tar_framing(&one_zero_block).await;
let error = error.expect("one zero block is not a valid TAR terminator");
assert!(matches!(&error.inner, FrameErrorInner::MissingEndMarker));
assert_eq!(
last_member_end,
Some(u64::try_from(RAW_FIXTURE.len()).expect("fixture length should fit in u64"))
);
assert_eq!(
error.position,
u64::try_from(one_zero_block.len()).expect("fixture length should fit in u64")
);
assert!(!is_authenticated_footerless_end(&error, last_member_end, true));
}
#[tokio::test]
async fn tar_codec_policy_accepts_only_the_explicit_minio_vendor_namespace() {
let default_error = match TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec())).members().next().await {
Err(error) => error,
Ok(_) => panic!("the default policy should reject minio vendor records"),
};
assert!(default_error.to_string().contains("pax vendor extension minio."));
let mut members = TarArchive::new(Cursor::new(RAW_FIXTURE.to_vec()))
.with_policy(candidate_snowball_decode_policy())
.members();
let mut bodies = Vec::new();
loop {
let member = match members.next().await {
Ok(Some(member)) => member,
Ok(None) => panic!("footerless minio-go fixture should not report a valid archive end"),
Err(error) => {
assert!(error.to_string().contains("missing two-block end-of-archive marker"));
break;
}
};
let Member::File { mut payload, .. } = member else {
panic!("fixture should contain only regular files");
};
let mut body = Vec::new();
let mut chunk = Vec::new();
while payload
.next_chunk(&mut chunk, 64 * 1024)
.await
.expect("tar-codec should read the fixture body")
{
body.extend_from_slice(&chunk);
}
bodies.push(body);
}
assert_eq!(bodies, [b"alpha-body".to_vec(), b"bravo-body".to_vec()]);
assert!(
members
.next()
.await
.expect("the member cursor should be fused after an error")
.is_none()
);
}
fn pax_record(key: &str, value: &str) -> Vec<u8> {
let payload = format!("{key}={value}\n");
let mut len = payload.len() + 3;
loop {
let record = format!("{len} {payload}");
if record.len() == len {
return record.into_bytes();
}
len = record.len();
}
}
async fn append_pax_header(
builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
entry_type: tokio_tar::EntryType,
records: &[(&str, &str)],
) {
let mut payload = Vec::new();
for (key, value) in records {
payload.extend(pax_record(key, value));
}
let mut header = tokio_tar::Header::new_ustar();
header.set_entry_type(entry_type);
header.set_size(u64::try_from(payload.len()).expect("PAX test payload should fit in u64"));
header.set_mode(0o644);
header.set_cksum();
builder
.append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload))
.await
.expect("PAX test header should be written");
}
async fn append_regular(builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>, path: &str) {
let body = path.as_bytes();
let mut header = tokio_tar::Header::new_ustar();
header.set_entry_type(tokio_tar::EntryType::Regular);
header.set_size(u64::try_from(body.len()).expect("test member body should fit in u64"));
header.set_mode(0o644);
header.set_mtime(1_704_164_645);
header.set_cksum();
builder
.append_data(&mut header, path, Cursor::new(body))
.await
.expect("ordinary test member should be written");
}
async fn archive_with_local_pax(records: &[(&str, &str)]) -> Vec<u8> {
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, records).await;
append_regular(&mut builder, "member.txt").await;
builder.into_inner().await.expect("policy archive should finish").into_inner()
}
#[tokio::test]
async fn candidate_policy_rejects_unknown_vendor_and_duplicate_pax_records() {
let unknown_vendor = archive_with_local_pax(&[("acme.metadata.owner", "mallory")]).await;
let error = match TarArchive::new(Cursor::new(unknown_vendor))
.with_policy(candidate_snowball_decode_policy())
.members()
.next()
.await
{
Err(error) => error,
Ok(_) => panic!("the candidate Snowball policy should reject unknown vendors"),
};
assert!(
error
.to_string()
.contains("pax vendor extension acme.metadata.owner is not allowed")
);
let duplicate = archive_with_local_pax(&[
("minio.metadata.x-amz-meta-owner", "first"),
("minio.metadata.x-amz-meta-owner", "second"),
])
.await;
let error = match TarArchive::new(Cursor::new(duplicate))
.with_policy(candidate_snowball_decode_policy())
.members()
.next()
.await
{
Err(error) => error,
Ok(_) => panic!("the candidate Snowball policy should reject duplicate PAX records"),
};
assert!(
error
.to_string()
.contains("pax extended header contains duplicate record minio.metadata.x-amz-meta-owner")
);
}
#[tokio::test]
async fn global_minio_pax_inheritance_is_an_explicit_migration_difference() {
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
append_pax_header(
&mut builder,
tokio_tar::EntryType::XGlobalHeader,
&[("minio.metadata.x-amz-meta-owner", "global")],
)
.await;
append_pax_header(
&mut builder,
tokio_tar::EntryType::XHeader,
&[("minio.metadata.x-amz-meta-owner", "local")],
)
.await;
append_regular(&mut builder, "local.txt").await;
append_regular(&mut builder, "inherited.txt").await;
let archive = builder
.into_inner()
.await
.expect("precedence archive should finish")
.into_inner();
let legacy = parse_with_tokio_tar(&archive).await;
let (framing, error, _) = parse_with_tar_framing(&archive).await;
assert!(error.is_none());
assert_eq!(legacy.len(), 2);
assert_eq!(framing.len(), 2);
let owner_key = "minio.metadata.x-amz-meta-owner";
assert_eq!(legacy[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec())));
assert!(!legacy[1].minio_pax.contains_key(owner_key));
assert_eq!(framing[0].minio_pax.get(owner_key), Some(&Some(b"local".to_vec())));
assert_eq!(framing[1].minio_pax.get(owner_key), Some(&Some(b"global".to_vec())));
let error = match TarArchive::new(Cursor::new(archive))
.with_policy(candidate_snowball_decode_policy())
.members()
.next()
.await
{
Err(error) => error,
Ok(_) => panic!("the candidate Snowball policy should reject global PAX state"),
};
assert!(error.to_string().contains("global pax extended headers are not allowed"));
}
deny.toml
+2 -2
View File
@@ -37,8 +37,8 @@ unknown-git = "deny"
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
allow-git = [
# Temporary tokio-tar fork pinned to the reviewed parser limits,
# cancellation safety, and error-fusing change while
# astral-sh/tokio-tar#118 awaits an upstream release.
# cancellation safety, and error-fusing change while Snowball is
# prototyped against tar-codec and Swift retains its current reader.
# owner: cxymds review: 2026-10
"https://github.com/cxymds/tokio-tar.git",
# Official s3s repository. Temporarily pinned to the merged generic REST
docs/architecture/compat-cleanup-register.md
+1 -1
View File
@@ -13,7 +13,7 @@
- `backlog-1337` legacy restore orphan recovery: releases that predate the restore worker-lock marker can leave a valid operation-id and `ongoing-request="true"` after cancellation or process failure, with no durable liveness proof. New servers allow an exact, non-nil legacy generation to be superseded only when its consistently parsed request date is at least 24 hours old. Remove the clock-based legacy fallback after the minimum supported direct-upgrade release writes the v1 worker-lock marker on every restore and operators have resolved every retained pre-v1 ongoing generation.
- `backlog-2133-tier-delete-chunk-parent` bounded tier-delete dispatch compatibility: prefixes at or below the legacy manifest limit keep the byte-compatible v1 single-manifest protocol, while larger prefixes place a chunk-parent sentinel at the original deterministic root path and use operation-scoped child manifests. Older binaries reject the sentinel and child paths, preserving the v6 sole-owner downgrade fence instead of starting a competing local delete. Remove the v1 reader and fail-closed mixed-version sentinel only after every supported rollback release validates the parent/child protocol and migration tooling confirms that no retained v1 dispatch manifest remains.
- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release.
- `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on precedence-resolved MinIO PAX metadata; per-entry and cumulative extension limits; a physical-entry limit; cancellation-safe parsing and ownership of large streamed members; fused streams after errors; and compatibility with minio-go streams that omit the two-block terminator. Swift bulk extraction also uses the same fork. Keep the reviewed pin while the Snowball path is prototyped against tar-codec/tar-framing. Remove it only after a released API exposes the effective allowed vendor records, RustFS provides a cancellation-safe handoff for borrowed member payloads, footerless input is accepted solely when authenticated request framing proves EOF immediately after a complete member, the existing resource-limit, cancellation, error-fuse, and real minio-go fixtures pass against the replacement, and Swift no longer depends on the fork.
- `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources.
- `backlog-2122` rc.1-rc.3 non-empty scanner usage floor recovery: leadership fencing in those releases can stamp scanner_epoch onto a real bucket-usage snapshot before any scanner cycle completed, leaving a non-empty floor with no scanner_cycle and no authoritative baseline identity. New scanners recognize only this consistent incomplete fenced shape, preserve the epoch through the CAS-protected recovery marker, and rebuild namespace coverage without treating the old usage data as authoritative. Remove this recovery path after rc.1, rc.2, and rc.3 are no longer supported direct-upgrade sources.
- `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive.
docs/testing/README.md
+2
View File
@@ -21,6 +21,8 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
Every script named above is indexed with status and wiring in [`scripts/README.md`](../../scripts/README.md). Fixed GHSA advisories map to named regression tests in [security-regressions.md](security-regressions.md).
The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retained subtree coverage across budget interruption, persistence, reload, and plan invalidation.
## Naming conventions
### Reserved test-name substrings (migration gate)
docs/testing/ci-gates.md
+10
View File
@@ -109,3 +109,13 @@ Use an exact preview tag for an end-to-end release rehearsal. Manual dispatches
## Change checklist
Update this file in the same PR when a job or check name changes, a workflow gains or loses a `pull_request` or `schedule` trigger, required contexts or strict/merge-queue policy change, report-only vs gating semantics change, or `.github/scheduled-validations.json` membership changes. Do not copy timeouts, crons, or test counts here.
## ECStore invariant selection
The existing `ci.yml` test-and-lint job runs the ordinary ECStore and filemeta tests. After that run, `scripts/check_test_wiring.py --check-core` checks the same nextest profile and package selection against `.config/ecstore-required-tests.json`. Every named test must exist, match the filter, and be non-ignored; the job also requires a nonempty JUnit report. This checks membership without running the tests twice. `core-test-listing.json`, JUnit, and the run log are retained in the existing test-and-lint artifact.
The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, and corrupt part arrays. Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered.
The checked-in MinIO corpus is pinned by file SHA256 and its documented source release. The static wiring guard and the CI selection check both reject missing or changed fixtures. These are metadata fixtures, not a legacy shard-body corpus or proof of crash durability. Optional `legacy_bitrot_read_test` runs may still skip when their external corpus is absent; they do not satisfy a required compatibility lane. Real encrypted fixture reads remain in `minio-interop.yml`, and multi-node fault schedules remain in the existing nightly cluster lane. In-process reopen tests do not establish power-loss durability.
Run `python3 scripts/check_test_wiring.py --self-test` to exercise the negative cases: removed/ignored/filtered tests, malformed listing, absent fixtures, and wrong fixture hashes. Do not update hashes merely to silence the guard; a fixture change needs source/provenance and compatibility review.
docs/testing/ecstore-validation-suite-design.md
+16
View File
@@ -54,6 +54,22 @@ Fail-closed invariants every row enforces:
Fault injection is explicit and deterministic: local disk mocks for unit tests, process-level disk manipulation (`crates/e2e_test/src/chaos.rs`) for e2e tests. Property tests replay a fixed seed for payload, range, and missing-shard selection.
### PUT completion fixtures
`ObjectOptions::default()` uses `WriteCompletion::Quorum`: a namespace-lock-owning PUT may acknowledge write quorum while its rename tail retains the lock. A fixture that immediately inspects every disk or primes a metadata generation must set `write_completion: WriteCompletion::TailDrained` and keep normal locking. TailDrained waits for the existing rename fan-out; it does not require every disk to succeed or change fsync policy. Codec-only `no_lock` fixtures do not cover namespace locking.
The object tests reuse `rename_fanout_barrier::arm(object, disk_slot, phase)` and `observe_tasks(object)`. Wait for the barrier with a deadline, observe actual metadata quorum with `wait_for_paused_tail_metadata_quorum`, then release or cancel. The metadata check distinguishes a real quorum from disk tasks that have not started. Assert zero remaining rename tasks after the owned coordinator releases its lock; cancellation tests also wait for staging cleanup.
| Fixture | Completion boundary |
|---|---|
| `early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes` | Default PUT returns before the parked tail; a second writer remains blocked. |
| `tail_drained_put_*` | Explicit full-tail PUT retains its guard, preserves quorum success with a failed minority, rejects quorum-minus-one, and survives ACK waiter cancellation. |
| `transition_and_restore_reclaim_prior_metadata_generations` | Both source fixtures use TailDrained before cache priming, with normal namespace locks. |
| `object_transaction_fencing_persists_epoch_on_multipart_commit` | Multipart completion already always drains rename before inspecting all per-disk transaction UUIDs. |
| `decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page`, `dispatch_completion_cas_is_bounded_and_reaches_the_tail` | Durable receipt, journal, and manifest writers choose TailDrained; the pagination fixture also drains deliberate receipt replacement writes. |
Select these checks with `cargo nextest list -p rustfs-ecstore --features test-util -E 'test(tail_drained_put) | test(early_ack_tail_drain) | test(no_lock_put_waits_for_rename_tail) | test(object_transaction_fencing_persists_epoch_on_multipart_commit) | test(transition_and_restore_reclaim) | test(decommission_durable_ilm_receipt_pagination) | test(dispatch_completion_cas)'`, then run the same expression under the default and CI profiles without retries. Remaining crash, reopen, rollback, and lock-loss schedules use the existing domain tests; this completion fixture is not a replacement for those checks.
### Coverage gate
`full` and `destructive` run `cargo llvm-cov -p rustfs-ecstore --lib` and fail when line coverage of the gate scope is below `--unit-coverage-min`. The default minimum and the 100% target for EC read, write, decode, heal, metadata-quorum, and rollback paths are the `UNIT_COVERAGE_*` constants at the top of the runner. `cargo-llvm-cov` must be installed unless `--skip-coverage` is passed explicitly. The default scope `ec-critical` is:
docs/testing/scanner-checkpoint-fixture.md
+22
View File
@@ -0,0 +1,22 @@
# Scanner Checkpoint Fixture
The `checkpoint_fixture` tests exercise a bounded namespace of 24 static objects and one repeatedly updated hot object. Each of three rounds runs the production local disk scanner with an object budget, saves the returned partial cache through the production persistence codec and revision checks to a two-file test backend, and reloads it before preparing the next round. The fixture prints static-subtree coverage at each boundary and cumulative visited entries. This is a diagnostic of retained coverage, not a throughput benchmark.
Run the fixture and confirm the test filter selects a nonzero number of tests:
```sh
cargo test -p rustfs-scanner --lib checkpoint_fixture -- --list
RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib checkpoint_fixture -- --nocapture
```
The unchanged-plan case requires durable static coverage to increase each round. The hot-plan diagnostic changes the bucket plan digest between rounds and reports where coverage is lost without asserting that a particular defect must remain present. To require progress in this diagnostic as well:
```sh
RUST_MIN_STACK=4194304 RUSTFS_CHECKPOINT_REQUIRE_PROGRESS=1 cargo test -p rustfs-scanner --lib checkpoint_fixture_hot_digest_diagnostic -- --nocapture
```
A nonzero exit from the strict command means that walked work did not become additional retained static coverage. `LostAtPrepare` identifies invalidation before traversal; `LostAtReload` identifies loss between the returned cache and persisted data; `WalkWithoutRetention` identifies visited growth without durable coverage growth. Missing, corrupt, empty-root, and oversized checkpoint inputs are rejected by the strict fixture reader. Save failure and publication-epoch rejection must preserve the preceding file bytes. Parent cancellation is checked separately from object-budget exhaustion. Superseded classification is tested separately from either incomplete outcome.
For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store.
The synthetic namespace contains no customer data. Temporary files are removed with their owning fixture. Production scan semantics and persistent formats are unchanged, so rollback consists of removing these tests and this guide. A passing fixture alone does not establish that the field report in [issue #7108](https://github.com/rustfs/rustfs/issues/7108) has been independently reproduced or fixed. A field diagnosis must separately identify the source capture, cycle and leader identity, and decoded bucket/set caches.
rustfs/src/admin/handlers/heal.rs
+177 -20
View File
@@ -39,7 +39,7 @@ use rustfs_utils::path::path_join;
use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashSet};
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::future::Future;
use std::path::PathBuf;
use std::sync::Arc;
@@ -261,6 +261,7 @@ struct BackgroundHealStatus<'a> {
heal_active_tasks: u64,
heal_operations: rustfs_heal::HealOperationsSnapshot,
cluster_status_complete: bool,
coverage: &'a BackgroundHealCoverage,
#[serde(skip_serializing_if = "Option::is_none")]
progress: Option<BackgroundHealProgress>,
}
@@ -300,6 +301,23 @@ fn background_heal_runtime_state(
type BackgroundHealProgress = rustfs_heal::HealProgress;
#[derive(Debug, Serialize)]
struct BackgroundHealCoverage {
expected: usize,
responded: usize,
unknown: usize,
reasons: BTreeSet<BackgroundHealCoverageReason>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
#[serde(rename_all = "snake_case")]
enum BackgroundHealCoverageReason {
NotificationSystemUnavailable,
PeerTopologyIncomplete,
PeerStatusUnsupported,
PeerStatusUnavailable,
}
#[derive(Debug)]
struct ClusterHealStatusSnapshot {
info: BackgroundHealInfo,
@@ -307,6 +325,7 @@ struct ClusterHealStatusSnapshot {
operations: rustfs_heal::HealOperationsSnapshot,
progress: Option<BackgroundHealProgress>,
complete: bool,
coverage: BackgroundHealCoverage,
}
fn add_priority_counts(total: &mut rustfs_heal::HealPriorityCounts, next: rustfs_heal::HealPriorityCounts) {
@@ -338,6 +357,7 @@ fn add_operations(total: &mut rustfs_heal::HealOperationsSnapshot, next: rustfs_
}
fn aggregate_cluster_heal_status(snapshots: Vec<NodeHealStatusSnapshot>) -> ClusterHealStatusSnapshot {
let responded = snapshots.len();
let mut info = BackgroundHealInfo::default();
let mut operations = rustfs_heal::HealOperationsSnapshot::default();
let mut progress = Vec::new();
@@ -379,6 +399,12 @@ fn aggregate_cluster_heal_status(snapshots: Vec<NodeHealStatusSnapshot>) -> Clus
operations,
progress,
complete: true,
coverage: BackgroundHealCoverage {
expected: responded,
responded,
unknown: 0,
reasons: BTreeSet::new(),
},
}
}
@@ -413,12 +439,14 @@ fn merge_peer_heal_statuses(
mut snapshots: Vec<NodeHealStatusSnapshot>,
peer_statuses: Vec<Result<Option<NodeHealStatusSnapshot>, String>>,
expected_nodes: usize,
topology_complete: bool,
coverage_reason: Option<BackgroundHealCoverageReason>,
) -> S3Result<ClusterHealStatusSnapshot> {
let mut reasons: BTreeSet<_> = coverage_reason.into_iter().collect();
for peer_status in peer_statuses {
match peer_status {
Ok(Some(snapshot)) => snapshots.push(snapshot),
Ok(None) => {
reasons.insert(BackgroundHealCoverageReason::PeerStatusUnsupported);
warn!(
event = EVENT_ADMIN_REQUEST_FAILED,
component = LOG_COMPONENT_ADMIN_API,
@@ -430,6 +458,7 @@ fn merge_peer_heal_statuses(
);
}
Err(err) => {
reasons.insert(BackgroundHealCoverageReason::PeerStatusUnavailable);
warn!(
event = EVENT_ADMIN_REQUEST_FAILED,
component = LOG_COMPONENT_ADMIN_API,
@@ -452,9 +481,12 @@ fn merge_peer_heal_statuses(
// so during a reconfiguration the count can equal `expected_nodes` while
// the topology is known-incomplete. Counting alone would report a
// definitive answer precisely when the membership itself is in doubt.
let complete = topology_complete && snapshots.len() == expected_nodes;
let complete = reasons.is_empty() && snapshots.len() == expected_nodes;
let mut status = aggregate_cluster_heal_status(snapshots);
status.complete = complete;
status.coverage.expected = expected_nodes;
status.coverage.unknown = expected_nodes.saturating_sub(status.coverage.responded);
status.coverage.reasons = reasons;
// A partial answer must never be mistakable for a definitive verdict: an
// unreachable peer might be mid-heal, so reporting the reachable nodes'
// "idle" (or disabled/uninitialized) as the cluster state would falsely
@@ -497,7 +529,12 @@ async fn read_cluster_heal_status(
return Ok(aggregate_cluster_heal_status(snapshots));
}
let Some(notification_system) = notification_system else {
return Err(cluster_heal_status_unavailable("notification_system_unavailable"));
return merge_peer_heal_statuses(
snapshots,
Vec::new(),
expected_nodes,
Some(BackgroundHealCoverageReason::NotificationSystemUnavailable),
);
};
// An incomplete peer topology (a down member's client slot, a rolling
// upgrade) previously failed the whole endpoint here, before any peer was
@@ -540,7 +577,12 @@ async fn read_cluster_heal_status(
}))
.await;
merge_peer_heal_statuses(snapshots, peer_statuses, expected_nodes, topology_complete)
merge_peer_heal_statuses(
snapshots,
peer_statuses,
expected_nodes,
(!topology_complete).then_some(BackgroundHealCoverageReason::PeerTopologyIncomplete),
)
}
async fn query_peer_replacement_recovery_status<E>(
@@ -1164,6 +1206,7 @@ fn encode_background_heal_status(
heal_operations: rustfs_heal::HealOperationsSnapshot,
progress: Option<BackgroundHealProgress>,
cluster_status_complete: bool,
coverage: &BackgroundHealCoverage,
) -> S3Result<Vec<u8>> {
let status = BackgroundHealStatus {
info,
@@ -1172,6 +1215,7 @@ fn encode_background_heal_status(
heal_active_tasks: heal_operations.active_tasks,
heal_operations,
cluster_status_complete,
coverage,
progress,
};
serde_json::to_vec(&status).map_err(|e| {
@@ -1461,6 +1505,7 @@ impl Operation for BackgroundHealStatusHandler {
cluster_status.operations,
cluster_status.progress,
cluster_status.complete,
&cluster_status.coverage,
)?;
info!(
event = EVENT_ADMIN_RESPONSE_EMITTED,
@@ -1515,13 +1560,14 @@ impl Operation for ReplacementRecoveryStatusHandler {
mod tests {
use super::extract_heal_init_params;
use super::{
BackgroundHealProgress, HealInitParams, HealResp, HealRuntimeState, aggregate_cluster_heal_status,
aggregate_replacement_recovery_cluster_status, background_heal_runtime_state, build_heal_channel_request,
build_replacement_recovery_status_response, encode_background_heal_status, encode_heal_control_path,
encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability, heal_channel_response_items,
heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id, json_response,
map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status,
query_peer_replacement_recovery_status, reject_heal_admission, validate_heal_request_mode, validate_heal_target,
BackgroundHealCoverage, BackgroundHealCoverageReason, BackgroundHealProgress, HealInitParams, HealResp, HealRuntimeState,
aggregate_cluster_heal_status, aggregate_replacement_recovery_cluster_status, background_heal_runtime_state,
build_heal_channel_request, build_replacement_recovery_status_response, encode_background_heal_status,
encode_heal_control_path, encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability,
heal_channel_response_items, heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id,
json_response, map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status,
query_peer_replacement_recovery_status, read_cluster_heal_status, reject_heal_admission, validate_heal_request_mode,
validate_heal_target,
};
use crate::storage::rpc::node_service::heal::{
NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status,
@@ -2175,7 +2221,13 @@ mod tests {
..Default::default()
};
let encoded = encode_background_heal_status(&info, HealRuntimeState::Active, operations, None, true)
let coverage = BackgroundHealCoverage {
expected: 1,
responded: 1,
unknown: 0,
reasons: Default::default(),
};
let encoded = encode_background_heal_status(&info, HealRuntimeState::Active, operations, None, true, &coverage)
.expect("background heal info should serialize");
let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize");
@@ -2225,6 +2277,12 @@ mod tests {
rustfs_heal::HealOperationsSnapshot::default(),
Some(progress),
true,
&BackgroundHealCoverage {
expected: 1,
responded: 1,
unknown: 0,
reasons: Default::default(),
},
)
.expect("background heal info should serialize");
let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize");
@@ -2398,6 +2456,84 @@ mod tests {
assert!(peer_topology_complete(1, 0, 0, 1, 0));
}
#[tokio::test]
async fn test_background_heal_status_without_notification_preserves_local_snapshot() {
let initialized = rustfs_heal::heal_runtime_initialized();
let info = BackgroundHealInfo {
bitrot_start_cycle: 37,
current_scan_mode: HealScanMode::Deep,
..Default::default()
};
for expected in [1, 3] {
let status = tokio::time::timeout(Duration::from_secs(1), read_cluster_heal_status(info.clone(), None, expected))
.await
.expect("local status must not wait for remote peers")
.expect("missing notification must retain the local snapshot");
assert_eq!(status.info.bitrot_start_cycle, 37);
assert_eq!(status.info.current_scan_mode, HealScanMode::Deep);
assert_eq!(status.complete, expected == 1);
assert_eq!(status.coverage.expected, expected);
assert_eq!(status.coverage.responded, 1);
assert_eq!(status.coverage.unknown, expected - 1);
if expected == 1 {
assert!(status.coverage.reasons.is_empty());
} else {
assert!(matches!(status.state, HealRuntimeState::Degraded | HealRuntimeState::Active));
assert_eq!(
status.coverage.reasons,
[BackgroundHealCoverageReason::NotificationSystemUnavailable].into()
);
}
let encoded = encode_background_heal_status(
&status.info,
status.state,
status.operations,
status.progress,
status.complete,
&status.coverage,
)
.expect("fallback status must encode");
let decoded: rustfs_madmin::client::BackgroundHealStatus =
serde_json::from_slice(&encoded).expect("the actual madmin client must decode the server response");
assert_eq!(decoded.cluster_status_complete, expected == 1);
let coverage = decoded.coverage.expect("new server supplies coverage");
assert_eq!(coverage.expected, Some(expected));
assert_eq!(coverage.responded, Some(1));
assert_eq!(coverage.unknown, Some(expected - 1));
if expected > 1 {
assert_eq!(coverage.reasons, ["notification_system_unavailable"]);
}
}
assert_eq!(
rustfs_heal::heal_runtime_initialized(),
initialized,
"reading status must not initialize heal"
);
}
#[test]
fn test_background_heal_status_coverage_reasons_are_bounded() {
let local = NodeHealStatusSnapshot::for_test(true, true, BackgroundHealInfo::default(), Default::default(), None);
let peers = (0..100)
.map(|index| {
if index % 2 == 0 {
Ok(None)
} else {
Err("peer unavailable".to_owned())
}
})
.collect();
let status = merge_peer_heal_statuses(vec![local], peers, 101, None).expect("local status remains available");
assert_eq!(status.coverage.responded, 1);
assert_eq!(status.coverage.unknown, 100);
assert_eq!(status.coverage.reasons.len(), 2);
let encoded = serde_json::to_vec(&status.coverage).expect("coverage encodes");
assert!(encoded.len() < 256, "coverage must not grow with peer failures");
let decoded: rustfs_madmin::client::BackgroundHealCoverage =
serde_json::from_slice(&encoded).expect("client coverage decodes");
assert_eq!(decoded.reasons, ["peer_status_unsupported", "peer_status_unavailable"]);
}
#[test]
fn test_peer_status_merge_degrades_explicitly_and_never_claims_idle() {
let local = || {
@@ -2413,15 +2549,20 @@ mod tests {
// but the safety property of the previous fail-closed behaviour is
// preserved: the partial answer is labelled Degraded, never Idle, so
// unknown peer work cannot be mistaken for "nothing is running".
let partial = merge_peer_heal_statuses(vec![local()], vec![Err("peer timeout".to_string())], 2, true)
let partial = merge_peer_heal_statuses(vec![local()], vec![Err("peer timeout".to_string())], 2, None)
.expect("an unreachable peer degrades the answer instead of destroying it");
assert!(!partial.complete);
assert_eq!(partial.state, HealRuntimeState::Degraded);
assert_eq!(partial.coverage.expected, 2);
assert_eq!(partial.coverage.responded, 1);
assert_eq!(partial.coverage.unknown, 1);
assert_eq!(partial.coverage.reasons, [BackgroundHealCoverageReason::PeerStatusUnavailable].into());
let older_peer = merge_peer_heal_statuses(vec![local()], vec![Ok(None)], 2, true)
let older_peer = merge_peer_heal_statuses(vec![local()], vec![Ok(None)], 2, None)
.expect("an older peer degrades the answer instead of destroying it");
assert!(!older_peer.complete);
assert_eq!(older_peer.state, HealRuntimeState::Degraded);
assert_eq!(older_peer.coverage.reasons, [BackgroundHealCoverageReason::PeerStatusUnsupported].into());
let known_active = NodeHealStatusSnapshot::for_test(
true,
@@ -2433,12 +2574,12 @@ mod tests {
},
None,
);
let partial_active = merge_peer_heal_statuses(vec![known_active], vec![Ok(None)], 2, true)
let partial_active = merge_peer_heal_statuses(vec![known_active], vec![Ok(None)], 2, None)
.expect("known active work may be reported as an explicit partial status");
assert!(!partial_active.complete);
assert_eq!(partial_active.state, HealRuntimeState::Active);
merge_peer_heal_statuses(Vec::new(), vec![Err("peer timeout".to_string())], 2, true)
merge_peer_heal_statuses(Vec::new(), vec![Err("peer timeout".to_string())], 2, None)
.expect_err("no snapshot at all still fails closed");
}
@@ -2459,12 +2600,22 @@ mod tests {
None,
)
};
let full_count_incomplete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, false)
.expect("incomplete topology degrades the answer instead of destroying it");
let full_count_incomplete_topology = merge_peer_heal_statuses(
vec![snapshot()],
vec![Ok(Some(snapshot()))],
2,
Some(BackgroundHealCoverageReason::PeerTopologyIncomplete),
)
.expect("incomplete topology degrades the answer instead of destroying it");
assert!(!full_count_incomplete_topology.complete);
assert_eq!(full_count_incomplete_topology.state, HealRuntimeState::Degraded);
assert_eq!(full_count_incomplete_topology.coverage.unknown, 0);
assert_eq!(
full_count_incomplete_topology.coverage.reasons,
[BackgroundHealCoverageReason::PeerTopologyIncomplete].into()
);
let full_count_complete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, true)
let full_count_complete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, None)
.expect("complete topology and full count is a definitive answer");
assert!(full_count_complete_topology.complete);
assert_eq!(full_count_complete_topology.state, HealRuntimeState::Idle);
@@ -2478,6 +2629,12 @@ mod tests {
rustfs_heal::HealOperationsSnapshot::default(),
None,
false,
&BackgroundHealCoverage {
expected: 2,
responded: 1,
unknown: 1,
reasons: [BackgroundHealCoverageReason::PeerStatusUnavailable].into(),
},
)
.expect("degraded status must serialize");
let json: serde_json::Value = serde_json::from_slice(&encoded).expect("valid json");
rustfs/src/admin/handlers/site_replication.rs
+418 -244
View File
@@ -190,7 +190,8 @@ fn site_replicator_service_account_policy() -> S3Result<Policy> {
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("parse site replicator policy failed: {e}")))
}
// Lock order: lifecycle -> bucket operation -> repair admission -> state -> per-bucket metadata.
// Lock order: lifecycle -> bucket-mutation admission -> per-bucket mutation
// -> bucket operation -> repair admission -> state -> per-bucket metadata.
// "state" is the distributed state-object lock in
// crate::site_replication::state_lock, entered through
// update_site_replication_state (P1-15). There is no process-local state
@@ -434,6 +435,7 @@ pub fn register_site_replication_route(r: &mut S3Router<AdminOperation>) -> std:
// into this module: startup sits below this layer and must not depend upwards. The admin
// router is built before startup reconciles, so the hook is always installed in time.
crate::site_replication_reconcile::register_site_replication_reconciler(reconcile_site_replication_wiring);
crate::site_replication_reconcile::register_site_replication_retry_drainer(reconcile_site_replication_retry_drain);
for (method, path, operation) in [
(Method::PUT, "/v3/site-replication/add", AdminOperation(&SiteReplicationAddHandler {})),
@@ -1803,28 +1805,61 @@ async fn reconcile_site_replication_buckets() -> S3Result<()> {
/// (`SiteReplicationEditHandler`), so a tick landing between them would rewrite the targets
/// from the stale endpoint. The pending marker in the persisted state closes that window.
/// Skipping costs nothing — the timer comes back.
async fn site_replication_reconcile_prerequisites_ready() -> bool {
if current_iam_handle().is_none() || current_object_store_handle().is_none() {
return false;
}
if let Err(err) = migrate_collapsed_retry_queue_paths().await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "retry_queue_migration_failed",
error = ?err,
"admin site replication state"
);
return false;
}
true
}
fn reconcile_site_replication_retry_drain() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
Box::pin(async {
let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
return;
};
if !site_replication_reconcile_prerequisites_ready().await {
return;
}
match load_site_replication_state().await {
Ok(state) => {
if state.pending_endpoint_refresh.is_some() || state.pending_rotation.is_some() || state.pending_remove.is_some()
{
return;
}
}
Err(_) => return,
}
// Admission above observes a lifecycle-stable state. The lightweight
// drain itself handles only idempotent bucket setup, reloads state
// under the distributed repair lock, and shares that lock with bucket
// deletion. Do not hold this process-local guard across peer I/O: an
// outage recovery must not make admin add/edit/remove time out.
drop(lifecycle);
drain_site_replication_retry_queue_lightweight().await;
})
}
fn reconcile_site_replication_wiring() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
Box::pin(async {
// The scheduler starts before IAM and the object store are guaranteed ready (IAM
// bootstrap may still be recovering), so an early tick returns quietly instead of
// logging a failure for every reconciler.
if current_iam_handle().is_none() || current_object_store_handle().is_none() {
return;
}
let Some(_lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
return;
};
if let Err(err) = migrate_collapsed_retry_queue_paths().await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "retry_queue_migration_failed",
error = ?err,
"admin site replication state"
);
if !site_replication_reconcile_prerequisites_ready().await {
return;
}
@@ -1878,8 +1913,9 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin<Box<dyn std::future::Fut
"admin site replication state"
);
}
// Failed peer deliveries recorded in the retry queue; runs behind the
// same lifecycle guard and pending_* gates as the reconcilers above.
// The retry path re-checks membership from distributed state before
// each request; release lifecycle before a potentially large replay.
drop(lifecycle);
drain_site_replication_retry_queue().await;
})
}
@@ -3046,6 +3082,7 @@ fn set_pending_endpoint_refresh(state: &mut SiteReplicationState, pending: Pendi
last_error: "endpoint target refresh pending".to_string(),
updated_at: Some(OffsetDateTime::now_utc()),
edit_generation: None,
peer_unreachable: false,
deletions_recorded: false,
});
state.pending_endpoint_refresh = Some(pending);
@@ -3592,16 +3629,15 @@ const PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS: u64 = 24 * 60 * 60 * 1_000_000_000
/// must be a site this state currently replicates with — the same membership
/// rule the load-time mark pruning applies, so every mark recorded behind
/// this check is one a reload would keep — and not this site itself, which
/// never delivers edits to itself. The caller IGNORES an inadmissible fence
/// rather than failing the request: the delivery applies exactly as an
/// unstamped (pre-fence) delivery would, no high-water mark is read or
/// written, and the worst a forged fence achieves is forfeiting an ordering
/// guarantee its sender was never owed. The generation itself is NOT
/// bounded here: a genuine origin whose hybrid clock persisted a wall-clock
/// excursion allocates arbitrarily far in the future, and refusing to
/// record its marks would strip the ordering fence from exactly the
/// deliveries that still race — the staleness window on the read side is
/// what defuses forged marks instead.
/// never delivers edits to itself. The caller acknowledges an inadmissible
/// fenced request without applying it: after a remove commits, an older
/// in-flight retry from the departed origin must not recreate topology. Old
/// peers remain compatible because their unstamped edits still follow the
/// pre-fence path. The generation itself is NOT bounded here: a genuine
/// origin whose hybrid clock persisted a wall-clock excursion allocates
/// arbitrarily far in the future, and refusing to record its marks would
/// strip the ordering fence from exactly the deliveries that still race —
/// the staleness window on the read side is what defuses forged marks instead.
fn peer_edit_fence_is_admissible(state: &SiteReplicationState, local_deployment_id: &str, fence: &(String, u64)) -> bool {
let (origin, generation) = fence;
if origin != local_deployment_id && state.peers.contains_key(origin) {
@@ -4791,105 +4827,135 @@ async fn backfill_existing_buckets_after_add(
let resync_id = Uuid::new_v4().to_string();
for bucket in &buckets {
let name = &bucket.name;
let operation_name = bucket.name.clone();
let lock_bucket = operation_name.clone();
let operation_state = state.clone();
let operation_local_peer = local_peer.clone();
let operation_resync_id = resync_id.clone();
let operation_bootstrap_token = bootstrap_token.map(str::to_owned);
let bucket_errors = with_site_replication_bucket_mutation_lock(store.clone(), &lock_bucket, move || async move {
let mut errors = SiteReplicationErrorSummary::default();
let name = &operation_name;
if let Err(err) = ensure_site_replication_bucket_versioning(name).await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_versioning_setup_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name}: versioning setup failed: {err}"));
continue;
}
match ensure_site_replication_bucket_setup(name).await {
Ok(true) => {}
Ok(false) => {
// Runtime targets unavailable: the setup silently no-ops, which would make the
// downstream make-bucket broadcast and resync fail. Record it and skip so the
// operator sees this bucket was not propagated instead of an unqualified success.
if let Err(err) = ensure_site_replication_bucket_versioning(name).await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_setup_skipped",
"admin site replication state"
);
errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)"));
continue;
}
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_setup_failed",
result = "backfill_versioning_setup_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name}: bucket setup failed: {err}"));
errors.push(format!("{name}: versioning setup failed: {err}"));
return errors;
}
}
// Broadcast the bucket to peers so they create it too (idempotent on the peer side).
// Read the real lock_enabled flag so peers recreate the bucket with the same object-lock
// setting — object lock cannot be added after bucket creation.
let lock_enabled = match metadata_sys::get(name).await {
Ok(bm) => bm.lock_enabled,
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_metadata_read_failed",
fallback = "lock_enabled=false",
error = ?err,
"admin site replication state"
);
false
match ensure_site_replication_bucket_setup(name).await {
Ok(true) => {}
Ok(false) => {
// Runtime targets unavailable: the setup silently no-ops, which would make the
// downstream make-bucket broadcast and resync fail. Record it and skip so the
// operator sees this bucket was not propagated instead of an unqualified success.
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_setup_skipped",
"admin site replication state"
);
errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)"));
return errors;
}
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_setup_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name}: bucket setup failed: {err}"));
}
}
};
if let Err(err) = broadcast_site_replication_make_bucket(name, lock_enabled, None, bootstrap_token).await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_make_bucket_broadcast_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name}: make-bucket broadcast failed: {err}"));
}
// Kick a resync toward every remote peer so existing objects travel across.
for peer in state.peers.values() {
if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) {
continue;
}
let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await;
let result = if manifest.target_arn.is_empty() {
manifest
} else {
start_site_bucket_resync(name, &manifest.target_arn, &resync_id).await
// Broadcast the bucket to peers so they create it too (idempotent on the peer side).
// Read the real lock_enabled flag so peers recreate the bucket with the same object-lock
// setting — object lock cannot be added after bucket creation.
let lock_enabled = match metadata_sys::get(name).await {
Ok(bm) => bm.lock_enabled,
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_metadata_read_failed",
fallback = "lock_enabled=false",
error = ?err,
"admin site replication state"
);
false
}
};
if result.status == "failed" {
if let Err(err) =
broadcast_site_replication_make_bucket(name, lock_enabled, None, operation_bootstrap_token.as_deref()).await
{
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
peer = %peer.endpoint,
result = "backfill_resync_kick_failed",
detail = %result.err_detail,
result = "backfill_make_bucket_broadcast_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail));
errors.push(format!("{name}: make-bucket broadcast failed: {err}"));
}
// Kick a resync toward every remote peer so existing objects travel across.
for peer in operation_state.peers.values() {
if peer.deployment_id == operation_local_peer.deployment_id
|| same_identity_endpoint(&peer.endpoint, &operation_local_peer.endpoint)
{
continue;
}
let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await;
let result = if manifest.target_arn.is_empty() {
manifest
} else {
start_site_bucket_resync(name, &manifest.target_arn, &operation_resync_id).await
};
if result.status == "failed" {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
peer = %peer.endpoint,
result = "backfill_resync_kick_failed",
detail = %result.err_detail,
"admin site replication state"
);
errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail));
}
}
errors
})
.await;
match bucket_errors {
Ok(bucket_errors) => errors.extend(bucket_errors),
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %lock_bucket,
result = "backfill_bucket_mutation_lock_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{lock_bucket}: bucket mutation lock failed: {err}"));
}
}
}
@@ -6072,146 +6138,204 @@ fn parse_peer_join_response(body: &[u8], fallback_peer: PeerInfo) -> Result<SRPe
serde_json::from_slice(body)
}
fn ensure_add_bucket_set_matches_preflight(expected: &HashSet<String>, present: &HashSet<String>) -> S3Result<()> {
let mut missing = expected.difference(present).cloned().collect::<Vec<_>>();
if !missing.is_empty() {
missing.sort_unstable();
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!(
"bucket `{}` disappeared while site replication was being added; peers may already be joined — re-run replicate add",
missing[0]
),
));
}
let mut unexpected = present.difference(expected).cloned().collect::<Vec<_>>();
if !unexpected.is_empty() {
unexpected.sort_unstable();
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!(
"bucket `{}` appeared while site replication was being added; peers may already be joined — re-run replicate add",
unexpected[0]
),
));
}
Ok(())
}
#[async_trait::async_trait]
impl Operation for SiteReplicationAddHandler {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?;
reject_site_replicator_on_public_admin(&cred)?;
let replicate_ilm_expiry = sr_add_replicate_ilm_expiry(&req.uri);
let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers);
let lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?;
// Everything up to the commit below is preflight: peer probes, IAM
// work and the join fan-out all talk to the network, so none of it may
// run inside the state transaction. The snapshot read here is what the
// `updated_at` CAS in the commit validates.
let current_state = load_site_replication_state().await?;
if pending_endpoint_refresh(&current_state).is_some() {
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
}
let local_peer = current_local_peer(&req, &current_state);
let mut sites: Vec<PeerSite> = read_site_replication_json(req, &cred.secret_key, true).await?;
// The web console's "Set Up Site Replication" omits the local deployment from the payload;
// inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`.
ensure_local_site_present(&mut sites, &local_peer);
validate_add_sites(&sites, &local_peer)?;
let preflight_infos = add_preflight_infos(&sites, &current_state, &local_peer).await?;
validate_add_preflight_topology(&preflight_infos, &local_peer)?;
let expected_updated_at = current_state.updated_at;
require_add_peer_tls_capability(&sites, &local_peer).await?;
// Early exit on a state that moved under the preflight probes, BEFORE
// the IAM write and the join fan-out change anything remote. Advisory
// only — the binding check is the CAS inside the commit — but it fences
// the common race off the side-effect path and refreshes the merge
// base so the CAS window is only the join round trips.
let latest_state = load_site_replication_state().await?;
ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?;
let current_state = latest_state;
let (service_account_access_key, service_account_secret_key) =
ensure_site_replicator_service_account(&cred.access_key, false).await?;
let bootstrap_buckets = preflight_infos
.iter()
.filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint))
.flat_map(|info| info.buckets.keys().cloned())
.collect();
let add_in_progress_guard = SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets)?;
let mut state = merge_add_sites(
current_state,
local_peer.clone(),
sites.clone(),
service_account_access_key.clone(),
cred.access_key.clone(),
replicate_ilm_expiry,
);
state.sync_state_initialized = true;
let join_req = SRPeerJoinEnvelope {
request: SRPeerJoinReq {
svc_acct_access_key: service_account_access_key,
svc_acct_secret_key: service_account_secret_key.clone(),
svc_acct_parent: String::new(),
peers: state.peers.clone(),
updated_at: state.updated_at,
},
defer_sync_state_enable: true,
};
let peer_join_path =
with_site_replication_bootstrap_token(SITE_REPLICATION_PEER_JOIN_PATH, &add_in_progress_guard.token.to_string());
let admin_access_key = cred.access_key.clone();
let admission_store = current_object_store_handle()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
let list_store = admission_store.clone();
let (state, edit_generation, local_peer, service_account_secret_key, mut initial_sync_errors, _add_guard) =
with_site_replication_bucket_mutation_admission_lock(admission_store, move || async move {
// The writer starts before the local bucket snapshot and stays
// held through every peer join and the topology commit. A
// delete followed by a same-name create therefore cannot hide
// behind an unchanged final name set. Peer bootstrap callbacks
// use their internal path and do not acquire this public-
// mutation admission lock.
let current_state = load_site_replication_state().await?;
if pending_endpoint_refresh(&current_state).is_some() {
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
}
let local_peer = local_peer_at_endpoint(local_endpoint, &current_state);
// The web console's "Set Up Site Replication" omits the local deployment from the payload;
// inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`.
ensure_local_site_present(&mut sites, &local_peer);
validate_add_sites(&sites, &local_peer)?;
let preflight_infos = add_preflight_infos(&sites, &current_state, &local_peer).await?;
validate_add_preflight_topology(&preflight_infos, &local_peer)?;
let expected_updated_at = current_state.updated_at;
require_add_peer_tls_capability(&sites, &local_peer).await?;
// Early exit on a state that moved under the preflight probes, BEFORE
// the IAM write and the join fan-out change anything remote. Advisory
// only — the binding check is the CAS inside the commit — but it fences
// the common race off the side-effect path and refreshes the merge
// base so the CAS window is only the join round trips.
let latest_state = load_site_replication_state().await?;
ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?;
let current_state = latest_state;
let (service_account_access_key, service_account_secret_key) =
ensure_site_replicator_service_account(&admin_access_key, false).await?;
let expected_buckets: HashSet<String> =
preflight_infos.iter().flat_map(|info| info.buckets.keys().cloned()).collect();
let bootstrap_buckets: HashSet<String> = preflight_infos
.iter()
.filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint))
.flat_map(|info| info.buckets.keys().cloned())
.collect();
let add_in_progress_guard =
SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets.clone())?;
let mut state = merge_add_sites(
current_state,
local_peer.clone(),
sites.clone(),
service_account_access_key.clone(),
admin_access_key,
replicate_ilm_expiry,
);
state.sync_state_initialized = true;
let join_req = SRPeerJoinEnvelope {
request: SRPeerJoinReq {
svc_acct_access_key: service_account_access_key,
svc_acct_secret_key: service_account_secret_key.clone(),
svc_acct_parent: String::new(),
peers: state.peers.clone(),
updated_at: state.updated_at,
},
defer_sync_state_enable: true,
};
let peer_join_path = with_site_replication_bootstrap_token(
SITE_REPLICATION_PEER_JOIN_PATH,
&add_in_progress_guard.token.to_string(),
);
let mut joined_endpoints = HashSet::new();
let mut initial_sync_errors = SiteReplicationErrorSummary::default();
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
{
continue;
}
let mut joined_endpoints = HashSet::new();
let mut initial_sync_errors = SiteReplicationErrorSummary::default();
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|| !joined_endpoints.insert(site_identity_key(&site.endpoint))
{
continue;
}
let mut peer_join_req = join_req.clone();
peer_join_req.request.svc_acct_parent = site.access_key.clone();
let connection = PeerConnection::try_from(site)?;
let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key)
.send(&site.secret_key, &peer_join_req)
let mut peer_join_req = join_req.clone();
peer_join_req.request.svc_acct_parent = site.access_key.clone();
let connection = PeerConnection::try_from(site)?;
let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key)
.send(&site.secret_key, &peer_join_req)
.await?;
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
fallback_peer.deployment_id = preflight.deployment_id.clone();
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("parse peer join response from {} failed: {e}", site.endpoint),
)
})?;
if !join_response.initial_sync_error_message.is_empty() {
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
}
// An explicit no-op join. The peer answered 200 but wrote nothing —
// its persisted state is already newer than the snapshot it was
// sent — so the add is only PARTIALLY configured and saying
// "configured successfully" would be a lie (rustfs/rustfs#5963).
// `None` (a MinIO peer, or one older than the field) is not a
// no-op signal and is deliberately not reported.
if join_response.applied == Some(false) {
initial_sync_errors.push(format!(
"{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \
the site is not configured against this peer",
site.endpoint
));
}
state = reconcile_peer_with_actual_identity(state, join_response.peer);
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("peer join response from {} did not identify the requested site", site.endpoint),
)
})?;
validate_proposed_peer(&reconciled_peer).map_err(|err| {
S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!("invalid peer join response from {}: {err}", site.endpoint),
)
})?;
}
mark_unknown_peer_sync_enabled(&mut state.peers);
// Commit. The state transaction's CAS still fences topology
// writers that do not use bucket admission. By this point
// remote sites may already have accepted their joins, so a
// mismatch asks the operator to re-run add and reconverge.
let next_state = state;
let present = list_store
.list_bucket(&BucketOptions::default())
.await
.map_err(ApiError::from)?
.into_iter()
.map(|bucket| bucket.name)
.collect::<HashSet<_>>();
ensure_add_bucket_set_matches_preflight(&expected_buckets, &present)?;
let (state, edit_generation) = update_site_replication_state(move |state| {
if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() {
return Err(s3_error!(
InvalidRequest,
"site replication state changed during peer join; the peers may already be joined — re-run replicate add"
));
}
adopt_add_commit_state(state, next_state);
let edit_generation = next_peer_edit_generation(state);
Ok((state.clone(), edit_generation))
})
.await?;
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
fallback_peer.deployment_id = preflight.deployment_id.clone();
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("parse peer join response from {} failed: {e}", site.endpoint),
)
})?;
if !join_response.initial_sync_error_message.is_empty() {
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
}
// An explicit no-op join. The peer answered 200 but wrote nothing —
// its persisted state is already newer than the snapshot it was
// sent — so the add is only PARTIALLY configured and saying
// "configured successfully" would be a lie (rustfs/rustfs#5963).
// `None` (a MinIO peer, or one older than the field) is not a
// no-op signal and is deliberately not reported.
if join_response.applied == Some(false) {
initial_sync_errors.push(format!(
"{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \
the site is not configured against this peer",
site.endpoint
));
}
state = reconcile_peer_with_actual_identity(state, join_response.peer);
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("peer join response from {} did not identify the requested site", site.endpoint),
)
})?;
validate_proposed_peer(&reconciled_peer).map_err(|err| {
S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!("invalid peer join response from {}: {err}", site.endpoint),
)
})?;
}
mark_unknown_peer_sync_enabled(&mut state.peers);
// Commit. The CAS runs inside the transaction, against the state the
// transaction itself loaded — the peer round trips above took however
// long they took, and only this check can tell whether the topology
// this add was planned against is still the current one. The error
// says so: by this point the remote sites already accepted their
// joins, and re-running the add is what reconverges the local side.
let next_state = state;
let (state, edit_generation) = update_site_replication_state(move |state| {
if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() {
return Err(s3_error!(
InvalidRequest,
"site replication state changed during peer join; the peers may already be joined — re-run replicate add"
));
}
adopt_add_commit_state(state, next_state);
let edit_generation = next_peer_edit_generation(state);
Ok((state.clone(), edit_generation))
})
.await?;
Ok((
state,
edit_generation,
local_peer,
service_account_secret_key,
initial_sync_errors,
add_in_progress_guard,
))
})
.await?;
// The finalize fan-out delivers peer-edit payloads, so it carries the
// generation allocated in the commit above: the receiving site orders
@@ -7185,8 +7309,14 @@ impl Operation for SRPeerEditHandler {
// The fence is self-reported — the shared service account means
// the sender cannot be identified — so it is honoured only after
// the admissibility check, against the same state it will gate.
let commit_fence =
commit_fence.filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence));
let commit_fence = match commit_fence {
Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence),
// A fenced edit can only come from a current remote peer. If
// that origin left while the retry was in flight, applying
// its body here would resurrect the removed topology.
Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked)),
None => None,
};
// Ordering fence: the sending site allocates the generation under
// its state-object lock, so a delivery that lost the race carries
// a generation this site has already passed. Applying it would
@@ -8886,6 +9016,41 @@ mod tests {
);
}
#[test]
fn add_admission_starts_before_preflight_and_rejects_bucket_set_changes() {
let expected = HashSet::from(["remote-owned".to_string(), "shared".to_string()]);
let present = HashSet::from(["shared".to_string()]);
let err = ensure_add_bucket_set_matches_preflight(&expected, &present)
.expect_err("a missing bootstrap bucket must reject the topology commit");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
let present = HashSet::from([
"remote-owned".to_string(),
"shared".to_string(),
"created-during-add".to_string(),
]);
let err = ensure_add_bucket_set_matches_preflight(&expected, &present)
.expect_err("a bucket created during add must reject the topology commit");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
let src = include_str!("site_replication.rs");
let add = src
.split("impl Operation for SiteReplicationAddHandler")
.nth(1)
.and_then(|rest| rest.split("pub struct SiteReplicationRemoveHandler").next())
.expect("add handler block");
let admission = add
.find("with_site_replication_bucket_mutation_admission_lock")
.expect("distributed mutation admission");
let preflight = add.find("add_preflight_infos").expect("bucket preflight");
let validation = add
.find("ensure_add_bucket_set_matches_preflight")
.expect("bucket-set validation");
let commit = add.find("adopt_add_commit_state").expect("topology commit");
assert!(admission < preflight && preflight < validation && validation < commit);
}
#[test]
fn test_tls_capability_gates_run_before_add_or_edit_state_side_effects() {
let src = include_str!("site_replication.rs");
@@ -9182,13 +9347,19 @@ mod tests {
);
// Fence hardening: origin and generation are self-reported by a
// caller the shared service account cannot identify, so the handler
// must pass the fence through the admissibility check — against the
// same state the fence gates, i.e. inside the transaction — before
// reading or raising any high-water mark.
// must admit the fence against the same state it gates. An origin
// removed while a retry was in flight is acknowledged without
// applying the stale body; otherwise it could recreate topology.
assert!(
handler_block.contains(".filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence))"),
handler_block.contains(
"Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence)"
),
"SRPeerEditHandler must admit a fence only through peer_edit_fence_is_admissible inside the state transaction"
);
assert!(
handler_block.contains("Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked))"),
"SRPeerEditHandler must not apply a fenced edit after its origin leaves the current topology"
);
// P1-15 PR2: both halves of the fence and the edit they fence share
// ONE transaction. Checking the fence against a state read outside the
// lock would let the check pass on one snapshot and the write land on
@@ -10352,8 +10523,9 @@ mod tests {
/// A fence is self-reported: every site authenticates peer traffic with
/// the same site-replicator credential, so a compromised peer can stamp
/// ANY origin with ANY generation. An origin the receiver does not
/// replicate with — or the receiver itself — is ignored and plants no
/// mark; a mark a compromised peer plants for a CURRENT origin cannot
/// replicate with — or the receiver itself — is inadmissible and plants
/// no mark; the handler acknowledges such a request without applying its
/// body. A mark a compromised peer plants for a CURRENT origin cannot
/// silence that origin, because the staleness window refuses to fence on
/// a mark implausibly far above the genuine deliveries.
#[test]
@@ -12791,6 +12963,7 @@ mod tests {
last_error: "site replication is not enabled".to_string(),
updated_at: Some(OffsetDateTime::now_utc()),
edit_generation: None,
peer_unreachable: false,
deletions_recorded: false,
}],
..Default::default()
@@ -12989,6 +13162,7 @@ mod tests {
last_error: "peer offline".to_string(),
updated_at: Some(OffsetDateTime::now_utc()),
edit_generation: None,
peer_unreachable: false,
deletions_recorded: false,
}],
..Default::default()
rustfs/src/app/bucket_list_through.rs
+354 -5
View File
@@ -252,8 +252,16 @@ pub(crate) async fn merged_list_objects_v2(
.filter(|entry| merger.accepts(&entry.key().name))
.collect();
let keys: Vec<ListEntryKey> = kept.iter().map(SideEntry::key).collect();
if let Err(error) = merger.push_page(fetch.side, keys, is_truncated, next_token) {
match fetch.side {
MergeSide::Source => {
degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?;
continue;
}
MergeSide::Local => return Err(S3Error::with_message(S3ErrorCode::InternalError, error.to_string())),
}
}
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some));
merger.push_page(fetch.side, keys, is_truncated, next_token);
}
let outcome = merger.finish();
@@ -340,10 +348,12 @@ async fn fetch_source_page(
continuation_token: token,
max_keys: params.max_keys,
},
// Everything under `filter.prefix` rolls into one common prefix, so a
// single bounded listing settles whether it exists.
// Everything under `filter.prefix` rolls into one common prefix. An
// empty truncated probe must still follow its cursor before declaring
// that prefix absent.
SourceListPlan::Folded { probe_prefix, .. } => SourceListRequest {
prefix: Some(probe_prefix.as_str()),
continuation_token: token,
max_keys: 1,
..Default::default()
},
@@ -368,8 +378,8 @@ async fn fetch_source_page(
} else {
Vec::new()
},
false,
None,
!exists && page.is_truncated,
if exists { None } else { page.next_continuation_token },
))
}
_ => {
@@ -417,6 +427,17 @@ async fn local_delete_markers(store: &Arc<ECStore>, bucket: &str, keys: &[String
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{
FilterConfig, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, SourceCredentials, TlsConfig,
};
use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response};
use crate::app::storage_api::test::StoragePutObjReader;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use crate::app::storage_api::test::contract::object::ObjectIO as _;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
fn token(local: Option<&str>, local_done: bool) -> ListThroughToken {
ListThroughToken {
@@ -526,4 +547,332 @@ mod tests {
assert!(degraded);
assert_eq!(merger.next_fetch().map(|fetch| fetch.side), Some(MergeSide::Local));
}
/// Serves exactly the scripted S3 pages and joins every connection before
/// returning. A source retry or unexpected operation fails the test.
async fn scripted_list_source(pages: Vec<String>) -> (String, tokio_util::task::AbortOnDropHandle<Vec<String>>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind listing source");
let address = listener.local_addr().expect("listing source address");
let server = tokio::spawn(async move {
let mut requests = Vec::new();
for body in pages {
let (mut stream, _) = listener.accept().await.expect("accept source listing");
let mut request = Vec::new();
let mut chunk = [0; 4096];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let count = stream.read(&mut chunk).await.expect("read signed listing request");
assert!(count > 0, "source request must include complete headers");
request.extend_from_slice(&chunk[..count]);
assert!(request.len() <= 32 * 1024, "listing request headers must be bounded");
}
let first_line = String::from_utf8_lossy(&request)
.lines()
.next()
.expect("request line")
.to_string();
// The SDK joins the bucket endpoint with the LIST operation's `/` path.
assert!(
first_line.starts_with("GET /source-bucket/?"),
"expected a path-style bucket-root LIST request, got {first_line:?}"
);
assert!(first_line.contains("list-type=2"), "expected a ListObjectsV2 query, got {first_line:?}");
requests.push(first_line);
let response = format!(
"HTTP/1.1 200 OK\r\ncontent-type: application/xml\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
body.len()
);
stream.write_all(response.as_bytes()).await.expect("write source page");
stream.shutdown().await.expect("finish source response");
}
requests
});
(format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server))
}
fn source_xml(next: Option<&str>, truncated: bool, key: Option<&str>) -> String {
let next = next
.map(|token| format!("<NextContinuationToken>{token}</NextContinuationToken>"))
.unwrap_or_default();
let contents = key
.map(|key| format!("<Contents><Key>{key}</Key><Size>1</Size></Contents>"))
.unwrap_or_default();
format!(
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><IsTruncated>{truncated}</IsTruncated>{next}{contents}</ListBucketResult>"
)
}
struct ListThroughTestState {
bucket: String,
module_enabled: bool,
}
impl Drop for ListThroughTestState {
fn drop(&mut self) {
let sys = OnDemandMigrationSys::get();
sys.remove(&self.bucket);
sys.set_module_enabled(self.module_enabled);
}
}
async fn source_policy_request(
pages: Vec<String>,
policy: SourceErrorPolicy,
resume_source: Option<&str>,
filter_prefix: Option<&str>,
) -> (S3Result<S3Response<ListObjectsV2Output>>, Vec<String>) {
let store = shared_gating_ecstore().await;
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
let bucket = format!("odm-list-{}", uuid::Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create list-through bucket");
store
.put_object(
&bucket,
"z-local",
&mut StoragePutObjReader::from_vec(vec![1]),
&StorageObjectOptions::default(),
)
.await
.expect("seed real local listing");
let (endpoint, server) = scripted_list_source(pages).await;
let sys = OnDemandMigrationSys::get();
let _state_guard = ListThroughTestState {
bucket: bucket.clone(),
module_enabled: sys.is_module_enabled(),
};
sys.set_module_enabled(true);
let config = OnDemandMigrationConfig {
version: 1,
enabled: true,
source: SourceConfig {
provider: Provider::Minio,
endpoint: Some(endpoint),
region: "us-east-1".into(),
bucket: "source-bucket".into(),
path_style: PathStyle::Path,
credentials: Some(SourceCredentials {
access_key: "test-access".into(),
secret_key: "test-secret".into(),
session_token: None,
}),
tls: TlsConfig::default(),
},
filter: FilterConfig {
prefix: filter_prefix.map(str::to_string),
..Default::default()
},
policy: PolicyConfig {
list_through: true,
source_error: policy,
..Default::default()
},
};
sys.apply(&bucket, Some(&config)).await;
assert!(
sys.state(&bucket).expect("ODM state installed").client().is_ok(),
"fake source client must build"
);
let continuation_token = resume_source.map(|source| {
let token = ListThroughToken {
t: "odm-list".into(),
v: 1,
local: None,
local_done: false,
source: Some(source.into()),
source_done: false,
last_key: None,
};
base64_simd::STANDARD.encode_to_string(token.encode().as_bytes())
});
let input = ListObjectsV2Input {
bucket,
max_keys: Some(2),
continuation_token,
delimiter: filter_prefix.map(|_| "/".to_string()),
encoding_type: None,
expected_bucket_owner: None,
fetch_owner: None,
optional_object_attributes: None,
prefix: None,
request_payer: None,
start_after: None,
};
let request = S3Request {
input,
method: http::Method::GET,
uri: http::Uri::from_static("/?list-type=2"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let result = tokio::time::timeout(
Duration::from_secs(10),
DefaultBucketUsecase::from_global().execute_list_objects_v2(request),
)
.await
.expect("listing must complete within its bounded source budget");
let requests = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("source connections must finish")
.expect("source server must not panic");
(result, requests)
}
#[test]
#[serial_test::serial]
fn list_through_invalid_source_pagination_obeys_policy_on_the_handler_path() {
run_large_stack_test("list-through-source-policy", || async {
temp_env::async_with_vars(
[
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
("HTTP_PROXY", None),
("HTTPS_PROXY", None),
("ALL_PROXY", None),
("http_proxy", None),
("https_proxy", None),
("all_proxy", None),
("NO_PROXY", Some("*")),
("no_proxy", Some("*")),
],
async {
for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] {
for next in [None, Some(""), Some("stuck")] {
for key in [None, Some("a-source")] {
let (result, requests) =
source_policy_request(vec![source_xml(next, true, key)], policy, Some("stuck"), None).await;
assert_eq!(requests.len(), 1, "a malformed source page must not be retried");
assert!(requests[0].contains("continuation-token=stuck"));
assert_source_policy_result(result, policy);
}
}
let (result, requests) = source_policy_request(
vec![
source_xml(Some("stuck"), true, Some("a-source")),
source_xml(Some("stuck"), true, None),
],
policy,
None,
None,
)
.await;
assert_eq!(requests.len(), 2, "the failure must occur during a real refill");
assert!(!requests[0].contains("continuation-token="));
assert!(requests[1].contains("continuation-token=stuck"));
assert_source_policy_result(result, policy);
}
},
)
.await;
});
}
#[test]
#[serial_test::serial]
fn list_through_empty_advancing_source_pages_reach_eof_on_the_handler_path() {
run_large_stack_test("list-through-empty-source-pages", || async {
temp_env::async_with_vars(
[
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
("HTTP_PROXY", None),
("HTTPS_PROXY", None),
("ALL_PROXY", None),
("http_proxy", None),
("https_proxy", None),
("all_proxy", None),
("NO_PROXY", Some("*")),
("no_proxy", Some("*")),
],
async {
for filter_prefix in [None, Some("photos/2024/")] {
let source_key = if filter_prefix.is_some() {
"photos/2024/a-source"
} else {
"a-source"
};
let (result, requests) = source_policy_request(
vec![
source_xml(Some("opaque-next"), true, None),
source_xml(None, false, Some(source_key)),
],
SourceErrorPolicy::Propagate,
None,
filter_prefix,
)
.await;
assert_eq!(requests.len(), 2, "an empty truncated source page must reach its successor");
assert!(requests[1].contains("continuation-token=opaque-next"));
let response = result.expect("empty progressing source page is valid");
assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
let output = response.output;
let objects: Vec<_> = output
.contents
.unwrap_or_default()
.into_iter()
.map(|object| object.key.expect("listed object key"))
.collect();
if filter_prefix.is_some() {
assert_eq!(objects, vec!["z-local"]);
assert_eq!(
output
.common_prefixes
.unwrap_or_default()
.into_iter()
.map(|prefix| prefix.prefix.expect("rolled-up prefix"))
.collect::<Vec<_>>(),
vec!["photos/"]
);
} else {
assert_eq!(objects, vec!["a-source", "z-local"]);
assert!(output.common_prefixes.unwrap_or_default().is_empty());
}
assert_eq!(output.key_count, Some(2));
assert_eq!(output.is_truncated, Some(false));
assert!(output.next_continuation_token.is_none());
}
},
)
.await;
});
}
fn assert_source_policy_result(result: S3Result<S3Response<ListObjectsV2Output>>, policy: SourceErrorPolicy) {
match policy {
SourceErrorPolicy::Propagate => {
let error = result.expect_err("propagate must expose malformed pagination");
assert_eq!(error.status_code(), Some(http::StatusCode::FAILED_DEPENDENCY));
assert_eq!(error.code(), &S3ErrorCode::Custom("SourceUnavailable".into()));
assert_eq!(error.message(), Some("invalid_pagination"));
}
SourceErrorPolicy::NotFound => {
let response = result.expect("not_found must preserve the local listing");
assert_eq!(
response
.headers
.get("x-rustfs-on-demand-migration-list")
.expect("local_only header"),
"local_only"
);
let output = response.output;
assert_eq!(
output
.contents
.unwrap_or_default()
.into_iter()
.map(|object| object.key.expect("local key"))
.collect::<Vec<_>>(),
vec!["z-local"]
);
assert_eq!(output.is_truncated, Some(false));
assert_eq!(output.key_count, Some(1));
assert!(output.next_continuation_token.is_none());
}
}
}
}
rustfs/src/app/bucket_usecase.rs
+63 -37
View File
@@ -75,7 +75,8 @@ use crate::auth::get_condition_values_with_client_info;
use crate::error::ApiError;
use crate::shared_types::RemoteAddr;
use crate::site_replication::{
site_replication_bucket_meta_hook, site_replication_delete_bucket_hook, site_replication_make_bucket_hook,
cancel_site_replication_delete_bucket, commit_site_replication_delete_bucket, prepare_site_replication_delete_bucket,
site_replication_bucket_meta_hook, site_replication_make_bucket_hook, with_site_replication_bucket_mutation_lock,
};
use crate::storage::storage_api::lock_bucket_targets_metadata;
use http::StatusCode;
@@ -1331,23 +1332,34 @@ impl DefaultBucketUsecase {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let make_result = store
.make_bucket(
&bucket,
&MakeBucketOptions {
force_create: false,
lock_enabled,
..Default::default()
},
)
.await;
// Keep the local namespace mutation and its peer hook ordered across
// every node in this site. Otherwise a delete waiting for repair
// coordination can arrive after this create on remote sites.
let operation_bucket = bucket.clone();
let operation_store = store.clone();
let make_result = with_site_replication_bucket_mutation_lock(store, &bucket, move || async move {
let make_result = operation_store
.make_bucket(
&operation_bucket,
&MakeBucketOptions {
force_create: false,
lock_enabled,
..Default::default()
},
)
.await;
if make_result.is_ok() {
crate::storage::invalidate_bucket_validation_cache(&operation_bucket);
if let Err(err) = site_replication_make_bucket_hook(&operation_bucket, lock_enabled).await {
warn!(bucket = %operation_bucket, error = ?err, "site replication make bucket hook failed");
}
}
make_result
})
.await?;
match make_result {
Ok(()) => {
// Invalidate the bucket validation cache so subsequent GETs
// see the newly created bucket immediately.
crate::storage::invalidate_bucket_validation_cache(&bucket);
}
Ok(()) => {}
Err(StorageError::BucketExists(_)) => {
// Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK;
// non-owner gets 409 BucketAlreadyExists.
@@ -1358,10 +1370,6 @@ impl DefaultBucketUsecase {
Err(e) => return Err(ApiError::from(e).into()),
}
if let Err(err) = site_replication_make_bucket_hook(&bucket, lock_enabled).await {
warn!(bucket = %bucket, error = ?err, "site replication make bucket hook failed");
}
let output = CreateBucketOutput::default();
counter!("rustfs_create_bucket_total").increment(1);
let result = Ok(S3Response::new(output));
@@ -1397,16 +1405,41 @@ impl DefaultBucketUsecase {
authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?;
}
store
.delete_bucket(
&input.bucket,
&DeleteBucketOptions {
force,
..Default::default()
},
)
.await
.map_err(ApiError::from)?;
// Keep the local namespace mutation and its peer hook ordered across
// every node in this site so an older delete cannot overtake a new
// same-name make while it waits for repair coordination.
let operation_bucket = input.bucket.clone();
let operation_store = store.clone();
with_site_replication_bucket_mutation_lock(store, &input.bucket, move || async move {
let intent = prepare_site_replication_delete_bucket(&operation_bucket, force).await?;
let delete_result = operation_store
.delete_bucket(
&operation_bucket,
&DeleteBucketOptions {
force,
..Default::default()
},
)
.await;
match delete_result {
Ok(()) => {
crate::storage::invalidate_bucket_validation_cache(&operation_bucket);
if let Some(intent) = intent
&& let Err(err) = commit_site_replication_delete_bucket(&intent).await
{
warn!(bucket = %operation_bucket, error = ?err, "site replication delete bucket hook failed");
}
Ok::<(), S3Error>(())
}
Err(err) => {
if let Some(intent) = intent {
cancel_site_replication_delete_bucket(intent).await;
}
Err(S3Error::from(ApiError::from(err)))
}
}
})
.await??;
// Drop every cached object body for the now-deleted bucket so dead
// bytes do not sit resident until TTL. Covers both the normal and the
@@ -1415,16 +1448,9 @@ impl DefaultBucketUsecase {
let cache_adapter = current_object_data_cache_for_context(self.context.as_deref());
let _ = invalidate_object_data_cache_bucket_after_delete(&cache_adapter, &input.bucket).await;
// Invalidate bucket validation cache
crate::storage::invalidate_bucket_validation_cache(&input.bucket);
// Re-evaluate lifecycle and replication after bucket removal.
rustfs_scanner::record_scanner_maintenance_change(&input.bucket);
if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await {
warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed");
}
// Notify peers to drop their cached metadata for the now-deleted bucket.
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
notify_bucket_metadata_delete(input.bucket.clone(), request_context);
rustfs/src/app/object/shared.rs
+1 -1
View File
@@ -309,7 +309,7 @@ fn classify_bucket_default_sse_lookup(
) -> S3Result<Option<(ServerSideEncryptionConfiguration, OffsetDateTime)>> {
match lookup {
Ok(config) => Ok(Some(config)),
Err(err) if err == StorageError::ConfigNotFound => Ok(None),
Err(StorageError::ConfigNotFound) => Ok(None),
Err(err) => {
let api_error = ApiError::from(err);
error!(
rustfs/src/app/storage_api.rs
+5 -3
View File
@@ -29,11 +29,13 @@ pub(crate) fn EndpointServerPools(
pub(crate) mod s3 {
#[cfg(test)]
pub(crate) use s3s::dto::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ReplicationConfiguration,
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, ServerSideEncryptionByDefault,
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, Destination, ListObjectsV2Input,
ListObjectsV2Output, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus,
ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, VersioningConfiguration,
};
pub(crate) use s3s::{S3Error, S3ErrorCode, S3Result};
#[cfg(test)]
pub(crate) use s3s::{S3Request, S3Response};
}
pub(crate) mod admin {
rustfs/src/server/http.rs
+47
View File
@@ -208,6 +208,7 @@ const EVENT_PEER_ADDR_UNAVAILABLE: &str = "peer_addr_unavailable";
const EVENT_RPC_SIGNATURE_VERIFICATION_FAILED: &str = "rpc_signature_verification_failed";
const EVENT_GRPC_TRACE_CONTEXT_PROPAGATION_FAILED: &str = "grpc_trace_context_propagation_failed";
const HEAL_CONTROL_TONIC_RPC_PATH: &str = "/node_service.HealControlService/HealControl";
const SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH: &str = "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck";
const TIER_MUTATION_PREPARE_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/PrepareTierMutation";
const TIER_MUTATION_COMMIT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/CommitTierMutation";
const TIER_MUTATION_ABORT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/AbortTierMutation";
@@ -1856,6 +1857,7 @@ fn process_connection(
);
let rpc_service = RpcRequestPathService::new(
Routes::new(node_service)
.add_service(InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth))
.add_service(heal_control_service)
.add_service(tier_mutation_control_service)
.prepare(),
@@ -2259,6 +2261,7 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
.strip_prefix(TONIC_RPC_PREFIX)
.and_then(|suffix| suffix.strip_prefix('/'))
.or_else(|| (target.uri.path() == HEAL_CONTROL_TONIC_RPC_PATH).then_some("HealControl"))
.or_else(|| (target.uri.path() == SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH).then_some("ScannerScopedDirtyUsageAck"))
.or_else(|| (target.uri.path() == TIER_MUTATION_PREPARE_TONIC_RPC_PATH).then_some("PrepareTierMutation"))
.or_else(|| (target.uri.path() == TIER_MUTATION_COMMIT_TONIC_RPC_PATH).then_some("CommitTierMutation"))
.or_else(|| (target.uri.path() == TIER_MUTATION_ABORT_TONIC_RPC_PATH).then_some("AbortTierMutation"))
@@ -3427,6 +3430,50 @@ mod tests {
rustfs_common::set_global_local_node_name(&previous_node_name).await;
}
#[tokio::test]
#[serial_test::serial]
async fn scoped_dirty_usage_peer_probe_reaches_handler_through_production_auth() {
let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string());
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind scoped ACK auth test");
let addr = listener.local_addr().expect("listener address");
let previous_node_name = rustfs_common::get_global_local_node_name().await;
rustfs_common::set_global_local_node_name(&addr.to_string()).await;
let node = InterceptedService::new(NodeServiceServer::new(make_server()), check_auth);
let scanner = InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth);
let service = RpcRequestPathService::new(Routes::new(node).add_service(scanner).prepare());
let server = tokio::spawn(async move {
let (socket, _) = listener.accept().await.expect("accept test connection");
ConnBuilder::new(TokioExecutor::new())
.serve_connection(TokioIo::new(socket), TowerToHyperService::new(service))
.await
.expect("serve scoped ACK auth test");
});
let host = rustfs_utils::XHost::try_from(addr.to_string()).expect("peer address");
let client = storage::PeerRestClient::new(host, format!("http://{addr}"));
let result = client
.scanner_scoped_dirty_usage_capability(
"11111111-1111-1111-1111-111111111111".to_string(),
"a".repeat(32),
vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
bucket: "photos".into(),
bucket_incarnation: vec![1; 16].into(),
generation: 8,
}],
)
.await;
client.evict_connection().await;
server.abort();
let _ = server.await;
rustfs_common::set_global_local_node_name(&previous_node_name).await;
let error = result
.expect_err("probe must fail closed without the requested storage owner")
.to_string();
assert!(
error.contains("storage layer is not initialized") || error.contains("scoped dirty usage peer or process changed"),
"signed probe must pass production path authentication and reach owner validation: {error}"
);
}
#[tokio::test]
#[serial_test::serial]
async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() {
rustfs/src/site_replication/hooks.rs
+427 -35
View File
@@ -22,6 +22,57 @@ pub(crate) const SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION: &str = "confi
pub(crate) static SITE_REPLICATION_BUCKET_OP_LOCK: LazyLock<RwLock<()>> = LazyLock::new(|| RwLock::new(()));
const SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX: &str = "config/site-replication/bucket-mutation";
pub(crate) const SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH: &str =
"config/site-replication/bucket-mutation-admission.lock";
pub(crate) fn site_replication_bucket_mutation_lock_path(bucket: &str) -> String {
format!("{SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX}/{bucket}.lock")
}
pub(crate) async fn with_site_replication_bucket_mutation_lock<F, Fut, T>(
store: Arc<ECStore>,
bucket: &str,
operation: F,
) -> S3Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = T> + Send + 'static,
T: Send + 'static,
{
let mutation_store = store.clone();
let mutation_path = site_replication_bucket_mutation_lock_path(bucket);
with_config_object_read_lock(
store,
SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(),
move || async move {
with_config_object_write_lock(mutation_store, mutation_path, operation)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))
},
)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))?
}
/// Exclude every local bucket namespace mutation from an add's local preflight
/// snapshot until its topology commit. Peer bootstrap callbacks do not enter
/// this public-mutation admission path, so they can finish while the writer is
/// held; post-commit fan-out and backfill must run after it is released.
pub(crate) async fn with_site_replication_bucket_mutation_admission_lock<F, Fut, T>(
store: Arc<ECStore>,
operation: F,
) -> S3Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
T: Send + 'static,
{
with_config_object_write_lock(store, SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), operation)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))?
}
#[derive(Debug, Default)]
pub(crate) struct SiteReplicationBootstrapPlan {
pub(crate) iam_items: Vec<SRIAMItem>,
@@ -329,6 +380,91 @@ pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteRep
Ok(plan)
}
/// Build only the two bucket operations needed by the lightweight retry
/// drain. The full bootstrap plan scans every bucket and IAM record; doing
/// that on a 30-second recovery cadence would make lifecycle admission scale
/// with the whole site instead of the one queued bucket.
pub(crate) fn site_replication_bucket_retry_plan_for(
bucket: &SRBucketInfo,
replicate_ilm_expiry: bool,
) -> S3Result<SiteReplicationBootstrapPlan> {
let mut plan = SiteReplicationBootstrapPlan {
bucket_make_ops: vec![bootstrap_bucket_make_op_path(bucket)],
bucket_configure_ops: vec![bootstrap_bucket_op_path(
&bucket.bucket,
SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION,
)],
..Default::default()
};
append_bootstrap_bucket_items(&mut plan, bucket, replicate_ilm_expiry)?;
Ok(plan)
}
pub(crate) fn site_replication_bucket_retry_plan_from_info(
bucket: &SRBucketInfo,
replicate_ilm_expiry: bool,
) -> S3Result<SiteReplicationBootstrapPlan> {
let mut plan = site_replication_bucket_retry_plan_for(bucket, replicate_ilm_expiry)?;
// Omit only metadata the make/configure operations can reproduce exactly.
// Non-default versioning fields and operator-authored replication rules
// remain in the plan; their extra request cost intentionally defers the
// event to the complete drain when the lightweight budget is too small.
plan.bucket_items.retain(|item| !retry_bucket_metadata_is_redundant(item));
Ok(plan)
}
fn retry_bucket_metadata_is_redundant(item: &SRBucketMeta) -> bool {
match item.r#type.as_str() {
"version-config" => item.versioning.as_deref().is_some_and(|raw| {
deserialize::<VersioningConfiguration>(&decode_bucket_meta_wire_value(raw)).is_ok_and(|config| {
config
== VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
}
})
}),
"replication-config" => item.replication_config.as_deref().is_some_and(|raw| {
deserialize::<ReplicationConfiguration>(&decode_bucket_meta_wire_value(raw))
.is_ok_and(|config| config.role.trim().is_empty() && config.rules.iter().all(is_derived_site_replication_rule))
}),
// `Some("")` is the in-memory sentinel used when the bucket is lock
// enabled but has no object-lock configuration body. The make query
// carries lockEnabled=true; sending an empty metadata body is neither
// useful nor parseable.
"object-lock-config" => item.object_lock_config.as_deref() == Some(""),
_ => false,
}
}
pub(crate) async fn site_replication_bucket_retry_plan(
bucket: &str,
replicate_ilm_expiry: bool,
) -> S3Result<SiteReplicationBootstrapPlan> {
let Some(store) = current_object_store_handle() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let bucket_info = match store.get_bucket_info(bucket, &BucketOptions::default()).await {
Ok(bucket_info) => bucket_info,
Err(err) if is_err_bucket_not_found(&err) => return Ok(SiteReplicationBootstrapPlan::default()),
Err(err) => return Err(ApiError::from(err).into()),
};
let lock_enabled = bucket_info.object_locking;
let metadata = metadata_sys::get(bucket).await.map_err(ApiError::from)?;
let mut bucket_info = SRBucketInfo {
bucket: bucket.to_string(),
created_at: bucket_info.created,
location: current_region().map(|region| region.to_string()).unwrap_or_default(),
api_version: Some(SITE_REPL_API_VERSION.to_string()),
..Default::default()
};
populate_sr_bucket_info_from_metadata(&mut bucket_info, &metadata).await;
if lock_enabled && bucket_info.object_lock_config.is_none() {
bucket_info.object_lock_config = Some(String::new());
}
site_replication_bucket_retry_plan_from_info(&bucket_info, replicate_ilm_expiry)
}
pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> {
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await;
let runtime = {
@@ -393,20 +529,273 @@ pub(crate) async fn broadcast_site_replication_make_bucket(
broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await
}
pub async fn site_replication_delete_bucket_hook(bucket: &str, force_delete: bool) -> S3Result<()> {
const SITE_REPLICATION_DELETE_INTENT_PENDING: &str =
"bucket deletion reserved; local completion and peer delivery are not yet known";
#[derive(Clone)]
struct SiteReplicationDeleteBucketReservation {
peer: PeerInfo,
previous: Option<SiteReplicationRetryEvent>,
observed: SiteReplicationRetryEvent,
}
pub(crate) struct SiteReplicationDeleteBucketIntent {
path: String,
reservations: Vec<SiteReplicationDeleteBucketReservation>,
displaced: Vec<SiteReplicationRetryEvent>,
}
fn site_replication_delete_bucket_path(bucket: &str, force_delete: bool) -> String {
let operation = if force_delete {
"force-delete-bucket"
} else {
"delete-bucket"
};
let path = format!(
format!(
"/rustfs/admin/v3/site-replication/peer/bucket-ops?{}",
form_urlencoded::Serializer::new(String::new())
.append_pair("bucket", bucket)
.append_pair("operation", operation)
.finish()
);
broadcast_site_replication_json(&path, &serde_json::json!({})).await
)
}
/// Reserve every destructive peer delivery before the local namespace is
/// changed. The state transaction either persists the complete set or writes
/// nothing, so a full/unreadable queue fails the S3 delete closed.
pub(crate) async fn prepare_site_replication_delete_bucket(
bucket: &str,
force_delete: bool,
) -> S3Result<Option<SiteReplicationDeleteBucketIntent>> {
let path = site_replication_delete_bucket_path(bucket, force_delete);
let reservation_path = path.clone();
update_site_replication_state_when_changed(move |state| {
if !state.enabled() {
return Ok(StateCommit::Unchanged(None));
}
let local_peer = current_local_runtime_peer(state);
let peers = state
.peers
.values()
.filter(|peer| {
peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
})
.cloned()
.collect::<Vec<_>>();
if peers.is_empty() {
return Ok(StateCommit::Unchanged(None));
}
let mut reservations = Vec::with_capacity(peers.len());
let mut displaced = Vec::new();
for peer in peers {
let previous = state
.retry_queue
.iter()
.find(|event| retry_event_matches(event, &peer, &reservation_path))
.cloned();
displaced.extend(upsert_site_replication_retry_event(
&mut state.retry_queue,
&peer,
&reservation_path,
SITE_REPLICATION_DELETE_INTENT_PENDING,
None,
)?);
let observed = state
.retry_queue
.iter()
.find(|event| retry_event_matches(event, &peer, &reservation_path))
.cloned()
.ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InternalError,
"site replication delete reservation disappeared before commit".to_string(),
)
})?;
reservations.push(SiteReplicationDeleteBucketReservation {
peer,
previous,
observed,
});
}
Ok(StateCommit::Changed(Some(SiteReplicationDeleteBucketIntent {
path: reservation_path,
reservations,
displaced,
})))
})
.await
}
/// Roll back a reservation when the local storage delete definitively failed.
/// A concurrently revised reservation is preserved; it belongs to a newer
/// observation and this operation has no authority to settle it.
pub(crate) async fn cancel_site_replication_delete_bucket(intent: SiteReplicationDeleteBucketIntent) {
let path = intent.path.clone();
let result = update_site_replication_state_when_changed(move |state| {
let mut changed = false;
for reservation in intent.reservations {
let Some(index) = state.retry_queue.iter().position(|event| {
retry_event_matches(event, &reservation.peer, &reservation.observed.path)
&& event.id == reservation.observed.id
&& event.updated_at == reservation.observed.updated_at
}) else {
continue;
};
if let Some(previous) = reservation.previous {
state.retry_queue[index] = previous;
} else {
state.retry_queue.remove(index);
}
changed = true;
}
let mut restored_all = true;
for displaced in intent.displaced {
let duplicate = state.retry_queue.iter().any(|event| {
event.id == displaced.id
|| (event.peer_deployment_id == displaced.peer_deployment_id && event.path == displaced.path)
});
if duplicate {
continue;
}
if state.retry_queue.len() >= SITE_REPLICATION_RETRY_QUEUE_LIMIT {
restored_all = false;
continue;
}
state.retry_queue.push(displaced);
changed = true;
}
Ok(if changed {
StateCommit::Changed(restored_all)
} else {
StateCommit::Unchanged(restored_all)
})
})
.await;
match result {
Ok(true) => {}
Ok(false) => warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
path,
result = "delete_intent_cancel_incomplete",
"admin site replication state"
),
Err(err) => warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
path,
result = "delete_intent_cancel_failed",
error = ?err,
"admin site replication state"
),
}
}
async fn broadcast_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> {
let sends = intent.reservations.iter().cloned().map(|reservation| {
let request_path = intent.path.clone();
async move {
let fallback_peer = reservation.peer.clone();
let observed = reservation.observed.clone();
let delivery_path = request_path.clone();
let delivery = with_site_replication_state_read_lock(move |state| async move {
let Some(current_peer) = state.peers.get(&fallback_peer.deployment_id).cloned() else {
return Ok(None);
};
let service_account_secret_key =
match site_replicator_service_account_secret(&state.service_account_access_key).await {
Ok(secret) => secret,
Err(err) => {
let Some(secret) = legacy_site_replicator_state_secret(&state) else {
return Err(err);
};
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "legacy_state_service_account_secret_fallback",
error = ?err,
"admin site replication state"
);
secret
}
};
let result = async {
let transport = PeerTransport::for_runtime_peer(&current_peer).await?;
PeerAdminRequest::put(&transport.connection, &delivery_path, &state.service_account_access_key)
.with_client(&transport.client)
.send(&service_account_secret_key, &serde_json::json!({}))
.await
}
.await;
Ok(Some((current_peer, result)))
})
.await;
match delivery {
Ok(Some((current_peer, Ok(_)))) => {
dequeue_observed_site_replication_retry_event(&current_peer, &observed).await;
None
}
Ok(Some((current_peer, Err(err)))) => {
// Keep the failed deletion operator-visible, but never
// replay it automatically: without a bucket-incarnation
// fence, a delayed delete could erase a recreated bucket.
enqueue_site_replication_retry_event(&current_peer, &request_path, &err).await;
Some(err)
}
Ok(None) => {
dequeue_observed_site_replication_retry_event(&reservation.peer, &observed).await;
None
}
Err(err) => {
enqueue_site_replication_retry_event(&reservation.peer, &request_path, &err).await;
Some(err)
}
}
}
});
futures::future::join_all(sends)
.await
.into_iter()
.flatten()
.next()
.map_or(Ok(()), Err)
}
pub(crate) async fn commit_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> {
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await;
let store =
current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
let retry_peers = intent
.reservations
.iter()
.map(|reservation| reservation.peer.clone())
.collect::<Vec<_>>();
let retry_path = intent.path.clone();
let delivery_intent = SiteReplicationDeleteBucketIntent {
path: intent.path.clone(),
reservations: intent.reservations.clone(),
displaced: Vec::new(),
};
match with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move {
broadcast_site_replication_delete_bucket(&delivery_intent).await
})
.await
{
Ok(result) => result,
Err(err) => {
let err: S3Error = ApiError::from(err).into();
for peer in &retry_peers {
enqueue_site_replication_retry_event(peer, &retry_path, &err).await;
}
Err(err)
}
}
}
pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Result<()> {
@@ -515,6 +904,39 @@ pub(crate) fn maybe_time(value: OffsetDateTime) -> Option<OffsetDateTime> {
(value != OffsetDateTime::UNIX_EPOCH).then_some(value)
}
async fn populate_sr_bucket_info_from_metadata(entry: &mut SRBucketInfo, metadata: &BucketMetadata) {
entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok());
entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml);
entry.tags = raw_config_to_base64(&metadata.tagging_config_xml);
entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml);
entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml);
entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml);
entry.quota_config = raw_config_to_base64(&metadata.quota_config_json);
// Expiry subset only: this entry feeds both the bootstrap/repair plan
// (peers must not receive transition rules) and cross-site consistency
// views (transition rules are site-local and would read as false
// mismatches). A deleted expiry state is a `None` value with the
// deletion's axis so repair can converge peers that missed the live
// delete.
let expiry_statement = lifecycle_expiry_statement(metadata);
entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone());
entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml);
entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at);
entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at);
entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at);
entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at);
entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at);
entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at);
entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at);
// The expiry axis, not the whole-config write time: local transition-only
// edits inflate the latter, and a repair item stamped with it could
// out-rank a newer real expiry edit on a third site.
entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis);
entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at);
entry.replication_targets_online =
Some(site_replication_targets_online(&entry.bucket, &metadata.replication_config_xml).await);
}
pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S3Result<SRInfo> {
let Some(store) = current_object_store_handle() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
@@ -546,37 +968,7 @@ pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &Pee
};
if let Some(metadata) = metadata {
entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok());
entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml);
entry.tags = raw_config_to_base64(&metadata.tagging_config_xml);
entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml);
entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml);
entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml);
entry.quota_config = raw_config_to_base64(&metadata.quota_config_json);
// Expiry subset only: this entry feeds both the bootstrap/repair
// plan (peers must not receive transition rules) and cross-site
// consistency views (transition rules are site-local and would
// read as false mismatches). A deleted expiry state is a `None`
// value with the deletion's axis so repair can converge peers
// that missed the live delete.
let expiry_statement = lifecycle_expiry_statement(&metadata);
entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone());
entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml);
entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at);
entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at);
entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at);
entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at);
entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at);
entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at);
entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at);
// The expiry axis, not the whole-config write time: local
// transition-only edits inflate the latter, and a repair item
// stamped with it could out-rank a newer real expiry edit on a
// third site.
entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis);
entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at);
entry.replication_targets_online =
Some(site_replication_targets_online(&bucket.name, &metadata.replication_config_xml).await);
populate_sr_bucket_info_from_metadata(&mut entry, &metadata).await;
}
info.buckets.insert(bucket.name, entry);

Some files were not shown because too many files have changed in this diff Show More