mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-05 19:55:37 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 loverustfs
|
43a7f94f14 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
|
||||
sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
|
||||
sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
{
|
||||
"lane": "ci/test-and-lint",
|
||||
"tests": [
|
||||
{
|
||||
"invariant": "write-quorum",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::inline_put_commit_path_tests::inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one"
|
||||
},
|
||||
{
|
||||
"invariant": "metadata-rollback",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::core::io_primitives::tests::write_unique_file_info_reverts_metadata_when_write_quorum_fails"
|
||||
},
|
||||
{
|
||||
"invariant": "stale-writer",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::put_object_no_lock_aborts_after_outer_namespace_lock_loss"
|
||||
},
|
||||
{
|
||||
"invariant": "range-body",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::object::transition_upload_integrity_tests::transitioned_compressed_object_range_get_returns_plaintext_slice"
|
||||
},
|
||||
{
|
||||
"invariant": "multipart-cancellation",
|
||||
"suite": "rustfs-ecstore",
|
||||
"name": "set_disk::ops::multipart::tests::cancelled_complete_keeps_upload_lock_through_tail_cleanup"
|
||||
},
|
||||
{
|
||||
"invariant": "list-uncommitted-version",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge"
|
||||
},
|
||||
{
|
||||
"invariant": "minio-object-fixture",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "filemeta::test::parses_real_minio_object_xlmeta"
|
||||
},
|
||||
{
|
||||
"invariant": "corrupt-part-arrays",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
|
||||
}
|
||||
],
|
||||
"fixtures": [
|
||||
{
|
||||
"path": "crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex",
|
||||
"sha256": "e8093767806d701e639b48d023190e858fbc4cde69bcfd83c22af8cba8452ce5",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex",
|
||||
"sha256": "2a415ad3a3be5a9440035d4026ff880e0e8c1ec1701be9f4e077734e8dce03da",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex",
|
||||
"sha256": "7f21f50c326dd8b0228deb6dbdb7052b3d0a3f8ee6c85d43486f0e6bb7a97261",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex",
|
||||
"sha256": "f2b6e260aff106adf6039feb1c645686e84e75404ff725491fb18668be5db203",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
},
|
||||
{
|
||||
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex",
|
||||
"sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12",
|
||||
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -40,7 +40,6 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
|
||||
@@ -129,7 +129,6 @@ jobs:
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/test_security_workflow.py
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
|
||||
@@ -167,7 +167,6 @@ jobs:
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/test_security_workflow.py
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
@@ -270,7 +269,6 @@ jobs:
|
||||
CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '3' || '2' }}
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
set +e
|
||||
@@ -279,12 +277,6 @@ jobs:
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
if [[ "${status}" -eq 0 ]]; then
|
||||
cargo nextest list --profile ci --all --exclude e2e_test --message-format json \
|
||||
> artifacts/test-and-lint/core-test-listing.json \
|
||||
&& python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json \
|
||||
&& test -s target/nextest/ci/junit.xml || status=$?
|
||||
fi
|
||||
{
|
||||
echo "command=cargo nextest run --profile ci --all --exclude e2e_test"
|
||||
echo "exit_status=${status}"
|
||||
|
||||
@@ -74,23 +74,10 @@ env:
|
||||
jobs:
|
||||
security-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for the OIDC live gate script)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize security evidence
|
||||
id: evidence
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${SECURITY_ARTIFACTS_DIR}"
|
||||
printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
|
||||
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
@@ -111,6 +98,11 @@ jobs:
|
||||
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
|
||||
exit 1
|
||||
|
||||
- name: Checkout repository (for the OIDC live gate script)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
@@ -143,8 +135,7 @@ jobs:
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
|
||||
TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
|
||||
REPORT_FILE: /tmp/rustfs-security-report.md
|
||||
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -168,48 +159,29 @@ jobs:
|
||||
else
|
||||
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
|
||||
./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
id: report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
env:
|
||||
TEST_OUTCOME: ${{ steps.test.outcome }}
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
RESULT=failure
|
||||
if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
|
||||
RESULT=success
|
||||
if [ ! -f /tmp/rustfs-security-report.md ]; then
|
||||
{
|
||||
echo "# RustFS security test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Test Step Outcome: failure (suite did not produce a report)"
|
||||
} > /tmp/rustfs-security-report.md
|
||||
fi
|
||||
{
|
||||
echo "# RustFS security test report"
|
||||
echo ""
|
||||
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Test Step Outcome: ${RESULT}"
|
||||
echo "- Suite Step Outcome: ${TEST_OUTCOME}"
|
||||
echo ""
|
||||
# The dashboard prioritizes case rows over the step outcome.
|
||||
# Keep partial case results in the artifact when the suite fails.
|
||||
if [ "${RESULT}" = "success" ]; then
|
||||
cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
|
||||
elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
|
||||
echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics."
|
||||
else
|
||||
echo "The suite did not produce a non-empty report."
|
||||
fi
|
||||
} > "${SECURITY_ARTIFACTS_DIR}/report.md"
|
||||
cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
[ "${RESULT}" = "success" ]
|
||||
cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
|
||||
REPORT_FILE: /tmp/rustfs-security-report.md
|
||||
SUITE: security
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -238,9 +210,8 @@ jobs:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
SUITE: 'security'
|
||||
SUITE_LABEL: 'Security'
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
|
||||
REPORT_FILE: '/tmp/rustfs-security-report.md'
|
||||
LOG_FILE: ''
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -274,7 +245,7 @@ jobs:
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
if [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
@@ -292,12 +263,14 @@ jobs:
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
|
||||
if-no-files-found: error
|
||||
name: rustfs-security-test-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-security-report.md
|
||||
/tmp/rustfs-security.*/*
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
|
||||
@@ -18,7 +18,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
from_version:
|
||||
description: 'OLD RustFS release tag, e.g. 1.0.0-rc.3 (its release must ship a .deb asset). Leave empty for the default.'
|
||||
description: 'OLD RustFS release tag (must ship a .deb asset, e.g. 1.0.0-rc.3)'
|
||||
required: false
|
||||
default: '1.0.0-rc.3'
|
||||
from_url:
|
||||
@@ -26,7 +26,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
to_version:
|
||||
description: 'NEW RustFS release tag, e.g. 1.0.0-rc.5 (any version with a .deb asset). Leave empty for latest nightly.'
|
||||
description: 'NEW RustFS release tag (leave empty for latest nightly)'
|
||||
required: false
|
||||
to_url:
|
||||
description: 'NEW .deb URL. Overrides to_version / nightly default.'
|
||||
@@ -145,7 +145,6 @@ jobs:
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-upgrade.log
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-upgrade-test.sh
|
||||
@@ -176,29 +175,6 @@ jobs:
|
||||
else
|
||||
ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
# Fail fast with a clear message when a requested release tag has
|
||||
# no .deb asset (e.g. 1.0.0-rc.4 ships only zips), instead of
|
||||
# letting the suite die mid-run on a 404.
|
||||
check_release_asset() {
|
||||
local version="$1" tag asset url
|
||||
[ -n "${version}" ] && [ "${version}" != "null" ] || return 0
|
||||
tag="${version#v}"
|
||||
asset="rustfs_${tag//-/.}_amd64.deb"
|
||||
url="https://github.com/rustfs/rustfs/releases/download/${tag}/${asset}"
|
||||
if ! gh api "repos/rustfs/rustfs/releases/tags/${tag}" --jq '.assets[].name' 2>/dev/null | grep -qxF "${asset}"; then
|
||||
echo "ERROR: release ${tag} has no downloadable asset ${asset}:" >&2
|
||||
echo " ${url}" >&2
|
||||
echo "Pick a tag whose release ships a .deb (check its release assets)." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "resolved ${tag} -> ${url}"
|
||||
}
|
||||
if [ -z "${FROM_URL}" ]; then
|
||||
check_release_asset "${FROM_VERSION}"
|
||||
fi
|
||||
if [ -z "${TO_URL}" ]; then
|
||||
check_release_asset "${TO_VERSION}"
|
||||
fi
|
||||
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
|
||||
@@ -42,7 +42,6 @@ jobs:
|
||||
- name: Check latest scheduled runs
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
run: |
|
||||
set +e
|
||||
python3 scripts/check_scheduled_validation_freshness.py \
|
||||
|
||||
@@ -33,7 +33,6 @@ profile.json
|
||||
*.zst
|
||||
.secrets
|
||||
*.go
|
||||
!crates/zip/tests/fixtures/snowball/**/generate/*.go
|
||||
*.pb
|
||||
*.svg
|
||||
deploy/logs/*.log.*
|
||||
|
||||
Generated
+112
-324
File diff suppressed because it is too large
Load Diff
+6
-10
@@ -168,7 +168,7 @@ reqwest = "0.13.4"
|
||||
rustfs-kafka-async = { version = "1.3.1" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.53.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.5" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.4" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.5"
|
||||
tokio-util = { version = "0.7.19" }
|
||||
@@ -234,19 +234,15 @@ tokio-postgres-rustls = "0.14.0"
|
||||
# Utilities and Tools
|
||||
anyhow = "1.0.104"
|
||||
arc-swap = "1.9.2"
|
||||
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin while Snowball and Swift still depend on it. Remove after Snowball uses a released tar-codec/tar-framing API that exposes precedence-resolved MinIO vendor records, RustFS preserves cancellation-safe ownership of large streamed members, footerless minio-go input is accepted only at an authenticated complete request boundary, the existing resource-limit, cancellation, and error-fuse regressions pass, and Swift no longer needs this fork.
|
||||
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until every parser hardening used by Snowball is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published release includes extension, physical-entry, and sparse limits, cancellation-safe sparse parsing, and error-fused entry streams.
|
||||
astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
|
||||
# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
|
||||
tar-codec = "0.0.14"
|
||||
tar-framing = "0.0.14"
|
||||
atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.12.0" }
|
||||
aws-config = { version = "1.11.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.118.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.145.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.114.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.117.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.144.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.113.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
|
||||
aws-smithy-runtime-api = { version = "1.16.0" }
|
||||
aws-smithy-types = { version = "1.6.3" }
|
||||
|
||||
@@ -6743,99 +6743,6 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_site_replication_replays_bucket_created_during_peer_outage_real_dual_node() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
// Keep compilation outside the scenario timeout. Recovery itself waits
|
||||
// for the production 30-second lightweight retry tick.
|
||||
let _rustfs_binary = rustfs_binary_path();
|
||||
|
||||
match timeout(Duration::from_secs(150), async {
|
||||
let mut site_env = replication_fast_env();
|
||||
site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
|
||||
let mut site_a_env = RustFSTestEnvironment::new().await?;
|
||||
site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
|
||||
|
||||
let mut site_b_env = RustFSTestEnvironment::new().await?;
|
||||
site_b_env.start_rustfs_server_without_cleanup_with_env(&site_env).await?;
|
||||
|
||||
let site_a_client = site_a_env.create_s3_client();
|
||||
let site_b_client = site_b_env.create_s3_client();
|
||||
let bucket = "site-repl-peer-outage";
|
||||
let key = "after-recovery.txt";
|
||||
let payload = b"site replication recovered the missed bucket".to_vec();
|
||||
|
||||
let add_status = site_replication_add(
|
||||
&site_a_env,
|
||||
&[
|
||||
PeerSite {
|
||||
name: "outage-site-a".to_string(),
|
||||
endpoint: site_a_env.url.clone(),
|
||||
access_key: site_a_env.access_key.clone(),
|
||||
secret_key: site_a_env.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
PeerSite {
|
||||
name: "outage-site-b".to_string(),
|
||||
endpoint: site_b_env.url.clone(),
|
||||
access_key: site_b_env.access_key.clone(),
|
||||
secret_key: site_b_env.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
assert!(add_status.success, "unexpected site add result: {add_status:?}");
|
||||
wait_for_site_replication_enabled(&site_a_env, 2).await?;
|
||||
wait_for_site_replication_enabled(&site_b_env, 2).await?;
|
||||
|
||||
site_b_env.stop_server();
|
||||
site_a_client.create_bucket().bucket(bucket).send().await?;
|
||||
site_a_client.head_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let queued = site_replication_info(&site_a_env)
|
||||
.await?
|
||||
.retry_stats
|
||||
.ok_or("peer outage did not persist a site replication retry event")?;
|
||||
assert!(queued.pending + queued.failed > 0, "peer outage retry queue was unexpectedly empty");
|
||||
|
||||
site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
|
||||
let recovery_deadline = tokio::time::Instant::now() + Duration::from_secs(75);
|
||||
loop {
|
||||
let bucket_recovered = site_b_client.head_bucket().bucket(bucket).send().await.is_ok();
|
||||
let queue_empty = site_replication_info(&site_a_env).await?.retry_stats.is_none();
|
||||
if bucket_recovered && queue_empty {
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() >= recovery_deadline {
|
||||
return Err(format!(
|
||||
"site replication retry did not settle after peer recovery; bucket_recovered={bucket_recovered}, queue_empty={queue_empty}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
|
||||
site_a_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(wait_for_object_on_target(&site_b_client, bucket, key).await?, payload);
|
||||
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err("site replication peer-outage recovery timed out after 150 seconds".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-applying a site's own replication config must not disable the peer's reverse direction.
|
||||
///
|
||||
/// `PutBucketReplication` broadcasts the config to every peer — the console's replication
|
||||
|
||||
@@ -244,7 +244,6 @@ windows-sys = { workspace = true, features = [
|
||||
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
|
||||
|
||||
[dev-dependencies]
|
||||
aws-smithy-async.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
|
||||
@@ -196,16 +196,15 @@ pub mod bucket {
|
||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
|
||||
get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
|
||||
get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
|
||||
get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
|
||||
get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
|
||||
get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
|
||||
reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
|
||||
update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
|
||||
update_under_transaction_lock,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete,
|
||||
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
|
||||
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
|
||||
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
|
||||
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config,
|
||||
get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config,
|
||||
get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata,
|
||||
remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock,
|
||||
update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -480,11 +479,9 @@ pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr,
|
||||
NotificationSys, ScannerPublicationLeaseGrant, acquire_cross_pool_fence_fleet_proof,
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
legacy_transition_state_reconcile_fleet_proof_matches, new_global_notification_sys,
|
||||
scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
|
||||
acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ use rustfs_utils::http::{
|
||||
insert_header,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::fmt;
|
||||
use std::str::FromStr as _;
|
||||
@@ -376,11 +376,6 @@ pub struct BucketTargetSys {
|
||||
/// [`SsecPassthroughCapability`]; reset alongside `arn_remotes_map`.
|
||||
ssec_passthrough_map: Arc<RwLock<HashMap<String, SsecPassthroughRecord>>>,
|
||||
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
||||
/// Buckets whose persisted `bucket-targets.json` exists but cannot be
|
||||
/// decoded (rustfs/backlog#2282). Written under the bucket's update mutex
|
||||
/// alongside `targets_map`, and read before it so an unreadable
|
||||
/// configuration surfaces as a typed error instead of an empty target set.
|
||||
unreadable_targets: Arc<RwLock<HashSet<String>>>,
|
||||
pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
pub hc_client: Arc<HttpClient>,
|
||||
@@ -424,7 +419,6 @@ impl BucketTargetSys {
|
||||
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
ssec_passthrough_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
unreadable_targets: Arc::new(RwLock::new(HashSet::new())),
|
||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
hc_client: Arc::new(build_health_check_client()),
|
||||
@@ -634,40 +628,30 @@ impl BucketTargetSys {
|
||||
health_map.clone()
|
||||
}
|
||||
|
||||
/// Targets of one bucket, or of every bucket when `bucket` is empty.
|
||||
///
|
||||
/// A bucket that simply has no targets yields an empty list; a bucket
|
||||
/// whose persisted configuration cannot be decoded is an error, so an
|
||||
/// admin listing reports the fault instead of an empty list that reads as
|
||||
/// "replication is not configured" (rustfs/backlog#2282).
|
||||
pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Result<Vec<BucketTarget>, BucketTargetError> {
|
||||
pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Vec<BucketTarget> {
|
||||
let health_stats = self.target_health_stats().await;
|
||||
let mut targets = Vec::new();
|
||||
|
||||
if !bucket.is_empty() {
|
||||
match self.list_bucket_targets(bucket).await {
|
||||
Ok(bucket_targets) => {
|
||||
for mut target in bucket_targets.targets {
|
||||
if arn_type.is_empty() || target.target_type.to_string() == arn_type {
|
||||
if let Some(health) = health_stats.get(&target.arn) {
|
||||
target.total_downtime = health.offline_duration;
|
||||
target.online = health.online;
|
||||
target.last_online = health.last_online;
|
||||
target.latency = target::LatencyStat {
|
||||
curr: health.latency.curr,
|
||||
avg: health.latency.avg,
|
||||
max: health.latency.peak,
|
||||
};
|
||||
target.offline_count = health.offline_count;
|
||||
}
|
||||
targets.push(target);
|
||||
if let Ok(bucket_targets) = self.list_bucket_targets(bucket).await {
|
||||
for mut target in bucket_targets.targets {
|
||||
if arn_type.is_empty() || target.target_type.to_string() == arn_type {
|
||||
if let Some(health) = health_stats.get(&target.arn) {
|
||||
target.total_downtime = health.offline_duration;
|
||||
target.online = health.online;
|
||||
target.last_online = health.last_online;
|
||||
target.latency = target::LatencyStat {
|
||||
curr: health.latency.curr,
|
||||
avg: health.latency.avg,
|
||||
max: health.latency.peak,
|
||||
};
|
||||
target.offline_count = health.offline_count;
|
||||
}
|
||||
targets.push(target);
|
||||
}
|
||||
}
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
return Ok(targets);
|
||||
return targets;
|
||||
}
|
||||
|
||||
let targets_map = self.targets_map.read().await;
|
||||
@@ -690,16 +674,10 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
Ok(targets)
|
||||
targets
|
||||
}
|
||||
|
||||
pub async fn list_bucket_targets(&self, bucket: &str) -> Result<BucketTargets, BucketTargetError> {
|
||||
if self.unreadable_targets.read().await.contains(bucket) {
|
||||
return Err(BucketTargetError::BucketRemoteTargetsUnreadable {
|
||||
bucket: bucket.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let targets_map = self.targets_map.read().await;
|
||||
if let Some(targets) = targets_map.get(bucket) {
|
||||
Ok(BucketTargets {
|
||||
@@ -712,30 +690,13 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
/// Record that this bucket's persisted targets configuration exists but
|
||||
/// cannot be decoded (rustfs/backlog#2282).
|
||||
///
|
||||
/// Any snapshot published from an earlier readable load is deliberately
|
||||
/// left in place: withdrawing it would produce exactly the silent "no
|
||||
/// targets configured" state this marker exists to prevent. The marker is
|
||||
/// cleared by the next successful publish, which is what makes a repaired
|
||||
/// configuration take effect without a restart.
|
||||
pub async fn mark_targets_unreadable(&self, bucket: &str) {
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let _update_guard = update_mutex.lock().await;
|
||||
|
||||
self.unreadable_targets.write().await.insert(bucket.to_string());
|
||||
}
|
||||
|
||||
pub async fn delete(&self, bucket: &str) {
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let _update_guard = update_mutex.lock().await;
|
||||
|
||||
// Lock order: unreadable_targets, then targets_map, then
|
||||
// arn_remotes_map, then target_h_mutex, then ssec_passthrough_map
|
||||
// (always last; also taken standalone by the capability accessors).
|
||||
self.unreadable_targets.write().await.remove(bucket);
|
||||
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||
// capability accessors).
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
@@ -1132,11 +1093,6 @@ impl BucketTargetSys {
|
||||
/// Keeping persisted-config reads under the same mutex prevents a stale
|
||||
/// reload from overwriting a concurrent credential rotation.
|
||||
async fn update_all_targets_locked(&self, bucket: &str, targets: Option<&BucketTargets>) {
|
||||
// Reaching here means the persisted configuration decoded, so the
|
||||
// unreadable marker (if any) is stale. Cleared before the maps below
|
||||
// so `unreadable_targets` stays the outermost of this module's locks.
|
||||
self.unreadable_targets.write().await.remove(bucket);
|
||||
|
||||
let mut clients = Vec::new();
|
||||
if let Some(new_targets) = targets {
|
||||
for target in &new_targets.targets {
|
||||
@@ -1144,9 +1100,9 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order: unreadable_targets (above), then targets_map, then
|
||||
// arn_remotes_map, then target_h_mutex, then ssec_passthrough_map
|
||||
// (always last; also taken standalone by the capability accessors).
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex,
|
||||
// then ssec_passthrough_map (always last; also taken standalone by the
|
||||
// capability accessors).
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
@@ -1205,11 +1161,6 @@ impl BucketTargetSys {
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: &str, meta: &BucketMetadata) {
|
||||
if meta.bucket_targets_unreadable() {
|
||||
self.mark_targets_unreadable(bucket).await;
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(config) = &meta.bucket_target_config else {
|
||||
return;
|
||||
};
|
||||
@@ -2325,13 +2276,6 @@ pub enum BucketTargetError {
|
||||
BucketRemoteTargetNotFound {
|
||||
bucket: String,
|
||||
},
|
||||
/// The bucket's persisted targets configuration exists but cannot be
|
||||
/// decoded. Distinct from `BucketRemoteTargetNotFound`, which means the
|
||||
/// bucket genuinely has no targets: callers must not degrade this one to
|
||||
/// an empty target set (rustfs/backlog#2282).
|
||||
BucketRemoteTargetsUnreadable {
|
||||
bucket: String,
|
||||
},
|
||||
BucketRemoteArnTypeInvalid {
|
||||
bucket: String,
|
||||
},
|
||||
@@ -2365,9 +2309,6 @@ impl fmt::Display for BucketTargetError {
|
||||
BucketTargetError::BucketRemoteTargetNotFound { bucket } => {
|
||||
write!(f, "Remote target not found for bucket: {bucket}")
|
||||
}
|
||||
BucketTargetError::BucketRemoteTargetsUnreadable { bucket } => {
|
||||
write!(f, "Persisted replication target configuration is unreadable for bucket: {bucket}")
|
||||
}
|
||||
BucketTargetError::BucketRemoteArnTypeInvalid { bucket } => {
|
||||
write!(f, "Invalid ARN type for bucket: {bucket}")
|
||||
}
|
||||
@@ -3315,7 +3256,7 @@ mod tests {
|
||||
}],
|
||||
);
|
||||
|
||||
let targets = sys.list_targets("", "").await.expect("listing every bucket's targets");
|
||||
let targets = sys.list_targets("", "").await;
|
||||
|
||||
assert_eq!(targets.len(), 1);
|
||||
assert!(!targets[0].online);
|
||||
|
||||
@@ -584,173 +584,33 @@ impl ExpiryOp for FreeVersionTask {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TransitionDeleteVersionPlan {
|
||||
Direct { version_id_exact: bool },
|
||||
ProbeLegacyUnknown,
|
||||
}
|
||||
|
||||
fn legacy_transition_version_state_missing(oi: &ObjectInfo) -> Result<bool, std::io::Error> {
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_str, get_consistent_str,
|
||||
};
|
||||
|
||||
if !contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE) {
|
||||
let version_key_present = contains_key_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID);
|
||||
if version_key_present {
|
||||
if oi.transitioned_object.version_id.is_empty() {
|
||||
let has_non_empty_version = oi.user_defined.iter().any(|(key, value)| {
|
||||
rustfs_utils::http::metadata_compat::strip_internal_prefix_preserving_case(key)
|
||||
.is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID))
|
||||
&& !value.is_empty()
|
||||
});
|
||||
if !has_non_empty_version {
|
||||
// MinIO writes the transitioned-versionID key with an empty value
|
||||
// for unversioned tier objects. The backend probe remains the proof.
|
||||
return Ok(true);
|
||||
}
|
||||
} else if get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_ID)
|
||||
== Some(oi.transitioned_object.version_id.as_str())
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"legacy remote tier version metadata is conflicting or malformed",
|
||||
));
|
||||
}
|
||||
if !oi.transitioned_object.version_id.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"legacy remote tier version metadata is missing or inconsistent",
|
||||
));
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
let persisted = get_consistent_str(&oi.user_defined, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object has conflicting transition version state metadata",
|
||||
)
|
||||
})?;
|
||||
if persisted != oi.transition_version_state.as_str() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object transition version state metadata changed during decoding",
|
||||
));
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn transition_remote_version_delete_plan(oi: &ObjectInfo) -> Result<TransitionDeleteVersionPlan, std::io::Error> {
|
||||
match oi.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => {
|
||||
if legacy_transition_version_state_missing(oi)? {
|
||||
Ok(TransitionDeleteVersionPlan::ProbeLegacyUnknown)
|
||||
} else {
|
||||
validate_transition_remote_version(oi)
|
||||
.map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact })
|
||||
}
|
||||
}
|
||||
_ => validate_transition_remote_version(oi)
|
||||
.map(|version_id_exact| TransitionDeleteVersionPlan::Direct { version_id_exact }),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct ResolvedTransitionDeleteVersion {
|
||||
version_id_exact: bool,
|
||||
remote_already_missing: bool,
|
||||
}
|
||||
|
||||
async fn acquire_free_version_tier_lease(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<(TierOperationLease, TransitionDeleteVersionPlan), std::io::Error> {
|
||||
let delete_plan = transition_remote_version_delete_plan(oi)?;
|
||||
) -> Result<(TierOperationLease, bool), std::io::Error> {
|
||||
let version_id_exact = validate_transition_remote_version(oi)?;
|
||||
let identity = tier_destination_id_from_metadata(&oi.user_defined)?
|
||||
.ok_or_else(|| std::io::Error::other("tier free-version has no durable backend identity"))?;
|
||||
let lease =
|
||||
TierConfigMgr::acquire_operation_lease_for_backend_identity(tier_config_mgr, &oi.transitioned_object.tier, identity)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
Ok((lease, delete_plan))
|
||||
}
|
||||
|
||||
async fn resolve_transition_delete_version_plan(
|
||||
oi: &ObjectInfo,
|
||||
lease: &TierOperationLease,
|
||||
delete_plan: TransitionDeleteVersionPlan,
|
||||
) -> Result<ResolvedTransitionDeleteVersion, std::io::Error> {
|
||||
match delete_plan {
|
||||
TransitionDeleteVersionPlan::Direct { version_id_exact } => Ok(ResolvedTransitionDeleteVersion {
|
||||
version_id_exact,
|
||||
remote_already_missing: false,
|
||||
}),
|
||||
TransitionDeleteVersionPlan::ProbeLegacyUnknown => {
|
||||
let expected_version = oi.transitioned_object.version_id.as_str();
|
||||
if expected_version.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"remote tier cannot safely delete a legacy object without an exact version ID",
|
||||
));
|
||||
}
|
||||
let probe = lease
|
||||
.probe_transition_version(&oi.transitioned_object.name, expected_version)
|
||||
.await?;
|
||||
match (expected_version, probe) {
|
||||
(expected, crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(actual))
|
||||
if expected == actual =>
|
||||
{
|
||||
lease.validate_remote_version_id(expected)?;
|
||||
Ok(ResolvedTransitionDeleteVersion {
|
||||
version_id_exact: true,
|
||||
remote_already_missing: false,
|
||||
})
|
||||
}
|
||||
(_, crate::services::tier::warm_backend::TransitionCandidateProbe::Missing) => {
|
||||
Ok(ResolvedTransitionDeleteVersion {
|
||||
version_id_exact: false,
|
||||
remote_already_missing: true,
|
||||
})
|
||||
}
|
||||
(_, crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"remote tier cannot prove legacy transition delete state",
|
||||
)),
|
||||
_ => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"remote tier object version state is unknown",
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_resolved_transition_delete(
|
||||
oi: &ObjectInfo,
|
||||
lease: &TierOperationLease,
|
||||
resolved: ResolvedTransitionDeleteVersion,
|
||||
) -> Result<(), std::io::Error> {
|
||||
if !resolved.remote_already_missing {
|
||||
delete_object_from_remote_tier_with_lease_idempotent(
|
||||
&oi.transitioned_object.name,
|
||||
&oi.transitioned_object.version_id,
|
||||
lease,
|
||||
resolved.version_id_exact,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
Ok((lease, version_id_exact))
|
||||
}
|
||||
|
||||
async fn delete_free_version_remote_object_with_lease(
|
||||
oi: &ObjectInfo,
|
||||
lease: &TierOperationLease,
|
||||
delete_plan: TransitionDeleteVersionPlan,
|
||||
version_id_exact: bool,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let resolved = resolve_transition_delete_version_plan(oi, lease, delete_plan).await?;
|
||||
execute_resolved_transition_delete(oi, lease, resolved).await
|
||||
delete_object_from_remote_tier_with_lease_idempotent(
|
||||
&oi.transitioned_object.name,
|
||||
&oi.transitioned_object.version_id,
|
||||
lease,
|
||||
version_id_exact,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn free_version_physical_topology_generation(api: &ECStore) -> String {
|
||||
@@ -781,16 +641,6 @@ fn free_version_remote_tuple_matches(candidate: &ObjectInfo, expected: &ObjectIn
|
||||
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
|| expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
{
|
||||
let candidate_legacy_missing = legacy_transition_version_state_missing(candidate)?;
|
||||
let expected_legacy_missing = legacy_transition_version_state_missing(expected)?;
|
||||
if candidate.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
&& expected.transition_version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
&& candidate_legacy_missing
|
||||
&& expected_legacy_missing
|
||||
&& candidate.transitioned_object.version_id == expected.transitioned_object.version_id
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier free-version remote version state is unknown",
|
||||
@@ -866,7 +716,7 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
|
||||
.acquire_bucket_lifecycle_read_lock(&oi.bucket)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
|
||||
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, &api.tier_config_mgr()).await?;
|
||||
let local_object = encode_dir_object(&oi.name);
|
||||
let object_guards = api
|
||||
.acquire_all_physical_object_write_locks("tier_free_version_cleanup", &oi.bucket, &local_object)
|
||||
@@ -884,30 +734,16 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
|
||||
"tier free-version cleanup fence is invalid before remote delete",
|
||||
));
|
||||
}
|
||||
let resolved = tokio::select! {
|
||||
_ = cancel.cancelled() => {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
|
||||
}
|
||||
result = tokio::time::timeout_at(deadline, resolve_transition_delete_version_plan(oi, &lease, delete_plan)) => {
|
||||
result.map_err(|_| {
|
||||
std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote probe timed out")
|
||||
})??
|
||||
}
|
||||
};
|
||||
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier free-version cleanup fence changed after remote probe",
|
||||
));
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "tier free-version cleanup was cancelled"));
|
||||
}
|
||||
result = tokio::time::timeout_at(deadline, execute_resolved_transition_delete(oi, &lease, resolved)) => {
|
||||
result.map_err(|_| {
|
||||
std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out")
|
||||
})??;
|
||||
result = tokio::time::timeout_at(
|
||||
deadline,
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact),
|
||||
) => {
|
||||
result
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "tier free-version remote delete timed out"))??;
|
||||
}
|
||||
}
|
||||
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
|
||||
@@ -955,8 +791,8 @@ async fn delete_free_version_remote_object(
|
||||
oi: &ObjectInfo,
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await
|
||||
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await
|
||||
}
|
||||
|
||||
#[allow(
|
||||
@@ -972,8 +808,8 @@ where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = T>,
|
||||
{
|
||||
let (lease, delete_plan) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, delete_plan).await?;
|
||||
let (lease, version_id_exact) = acquire_free_version_tier_lease(oi, tier_config_mgr).await?;
|
||||
delete_free_version_remote_object_with_lease(oi, &lease, version_id_exact).await?;
|
||||
let result = delete_local().await;
|
||||
drop(lease);
|
||||
Ok(result)
|
||||
@@ -4852,39 +4688,6 @@ fn validate_transition_remote_version(oi: &ObjectInfo) -> Result<bool, std::io::
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TransitionReadVersionPlan {
|
||||
Direct,
|
||||
ProbeLegacyUnversioned,
|
||||
}
|
||||
|
||||
const LEGACY_TRANSITION_READ_PROBE_TIMEOUT: StdDuration = StdDuration::from_secs(30);
|
||||
|
||||
fn transition_remote_version_read_plan(oi: &ObjectInfo) -> Result<TransitionReadVersionPlan, std::io::Error> {
|
||||
let version = oi.transitioned_object.version_id.as_str();
|
||||
match oi.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => {
|
||||
if !legacy_transition_version_state_missing(oi)? {
|
||||
return validate_transition_remote_version(oi).map(|_| TransitionReadVersionPlan::Direct);
|
||||
}
|
||||
if version.is_empty() {
|
||||
Ok(TransitionReadVersionPlan::ProbeLegacyUnversioned)
|
||||
} else {
|
||||
Ok(TransitionReadVersionPlan::Direct)
|
||||
}
|
||||
}
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled if version.is_empty() => Ok(TransitionReadVersionPlan::Direct),
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull if version == "null" => Ok(TransitionReadVersionPlan::Direct),
|
||||
rustfs_filemeta::TransitionVersionState::Exact if !version.is_empty() && version != "null" => {
|
||||
Ok(TransitionReadVersionPlan::Direct)
|
||||
}
|
||||
_ => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object version state conflicts with its version ID",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
// The resolver joins the tier manager as the second injected port this read
|
||||
// needs; grouping the request half into a struct would churn every call site of
|
||||
// a bug fix.
|
||||
@@ -4899,12 +4702,7 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
|
||||
resolver: Option<&dyn ObjectEncryptionResolver>,
|
||||
) -> Result<GetObjectReader, std::io::Error> {
|
||||
let read_plan = transition_remote_version_read_plan(oi)?;
|
||||
// Reject invalid ranges and encryption requests before a compatibility
|
||||
// probe can amplify them into remote listing work.
|
||||
let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver)
|
||||
.await
|
||||
.map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?;
|
||||
validate_transition_remote_version(oi)?;
|
||||
let expected_identity = tier_destination_id_from_metadata(&oi.user_defined)?;
|
||||
let lease = match expected_identity {
|
||||
Some(identity) => {
|
||||
@@ -4918,36 +4716,7 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
Err(err) => return Err(std::io::Error::other(err)),
|
||||
};
|
||||
|
||||
match read_plan {
|
||||
TransitionReadVersionPlan::Direct => {
|
||||
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
|
||||
}
|
||||
TransitionReadVersionPlan::ProbeLegacyUnversioned => {
|
||||
// RUSTFS_COMPAT_TODO(backlog#2203): remove operation-time probing
|
||||
// after an admin reconcile can persist every proven legacy state.
|
||||
let probe = tokio::time::timeout(
|
||||
LEGACY_TRANSITION_READ_PROBE_TIMEOUT,
|
||||
tgt_client.probe_transition_candidate(&oi.transitioned_object.name),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "legacy remote tier version probe timed out"))??;
|
||||
match probe {
|
||||
crate::services::tier::warm_backend::TransitionCandidateProbe::UnversionedPresent => {}
|
||||
crate::services::tier::warm_backend::TransitionCandidateProbe::Unsupported => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"remote tier cannot prove legacy unversioned transition state",
|
||||
));
|
||||
}
|
||||
_ => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"remote tier object version state is unknown",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
tgt_client.validate_remote_version_id(&oi.transitioned_object.version_id)?;
|
||||
|
||||
// The same read plan the local path uses, so the tier fetch is positioned in
|
||||
// the object's *stored* coordinate system and the stream is handed the same
|
||||
@@ -4955,6 +4724,9 @@ pub(crate) async fn get_transitioned_object_reader_with_tier_manager(
|
||||
// through a plaintext-coordinate range and skipping the transform is how a
|
||||
// transitioned SSE object used to come back as silently corrupt bytes of the
|
||||
// right length (rustfs/rustfs#6025).
|
||||
let plan = ReadPlan::build_for_request(rs.clone(), oi, opts, h, resolver)
|
||||
.await
|
||||
.map_err(|err| std::io::Error::other(format!("building the read plan for {bucket}/{object} failed: {err}")))?;
|
||||
let (off, length) = (plan.storage_offset() as i64, plan.storage_length());
|
||||
let mut gopts = WarmBackendGetOpts::default();
|
||||
|
||||
@@ -5827,13 +5599,11 @@ mod tests {
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::test_util::MockWarmOp;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::test_util::register_mock_tier;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _};
|
||||
use crate::services::tier::warm_backend::WarmBackend as _;
|
||||
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
@@ -6529,75 +6299,7 @@ mod tests {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_allows_legacy_unknown_exact_version_for_non_destructive_read() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
let body = Bytes::from_static(b"legacy transitioned object body");
|
||||
let remote_version = backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(body.clone()),
|
||||
i64::try_from(body.len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("mock remote object should be stored");
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_legacy_transition_version_id(&mut user_defined, &remote_version);
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: i64::try_from(body.len()).expect("body length should fit"),
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object,
|
||||
version_id: remote_version,
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier: tier.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let range = Some(crate::storage_api_contracts::range::HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 7,
|
||||
end: 18,
|
||||
});
|
||||
let mut reader = get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&range,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("legacy unknown state should still allow a non-destructive read");
|
||||
let mut got = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut got)
|
||||
.await
|
||||
.expect("transitioned reader should drain");
|
||||
|
||||
assert_eq!(got, &body.as_ref()[7..=18]);
|
||||
assert_eq!(backend.get_count().await, 1);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert_eq!(
|
||||
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
|
||||
0,
|
||||
"tier generation lease should release after EOF"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_rejects_explicit_unknown_version_state_before_backend_io() {
|
||||
async fn transitioned_get_rejects_unknown_version_state_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
@@ -6613,7 +6315,6 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined_with_transition_version_state(rustfs_filemeta::TransitionVersionState::Unknown).into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -6629,202 +6330,19 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("explicit unknown remote version state must fail before backend IO"),
|
||||
Ok(_) => panic!("unknown remote version state must fail before backend IO"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_rejects_present_but_invalid_legacy_version_metadata() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
|
||||
for persisted_version in [
|
||||
Uuid::nil().to_string(),
|
||||
"\u{fffd}".to_string(),
|
||||
"bad\u{0001}version".to_string(),
|
||||
] {
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_legacy_transition_version_id(&mut user_defined, &persisted_version);
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 1,
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: String::new(),
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier: tier.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("present but invalid legacy version metadata must fail before backend IO"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_probes_legacy_empty_unknown_state_before_unversioned_read() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
let body = Bytes::from_static(b"legacy unversioned transitioned object body");
|
||||
let remote_version = backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(body.clone()),
|
||||
i64::try_from(body.len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("mock remote object should be stored");
|
||||
assert!(remote_version.is_empty());
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: i64::try_from(body.len()).expect("body length should fit"),
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object.clone(),
|
||||
version_id: String::new(),
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier: tier.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: HashMap::from([("x-minio-internal-transitioned-versionID".to_string(), String::new())]).into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut reader = get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("probe-proven legacy unversioned state should allow a non-destructive read");
|
||||
let mut got = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut got)
|
||||
.await
|
||||
.expect("transitioned reader should drain");
|
||||
|
||||
assert_eq!(got, body.as_ref());
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert_eq!(
|
||||
backend.op_log().await,
|
||||
vec![
|
||||
MockWarmOp::Put {
|
||||
object: remote_object.clone()
|
||||
},
|
||||
MockWarmOp::Probe {
|
||||
object: remote_object.clone()
|
||||
},
|
||||
MockWarmOp::Get { object: remote_object },
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
|
||||
0,
|
||||
"tier generation lease should release after EOF"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn transitioned_get_rejects_ambiguous_empty_unknown_state_without_backend_get() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&manager, &tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::VersionedPresent(
|
||||
"versioned-candidate".to_string(),
|
||||
)))
|
||||
.await;
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 1,
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object.clone(),
|
||||
version_id: String::new(),
|
||||
status: crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string(),
|
||||
tier,
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = match get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("versioned legacy unknown state without stored version must fail before backend GET"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(backend.op_log().await, vec![MockWarmOp::Probe { object: remote_object }]);
|
||||
assert_eq!(backend.get_count().await, 0);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_rejects_explicit_unknown_before_backend_io() {
|
||||
async fn free_version_delete_rejects_unknown_version_state_before_backend_io() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let backend = register_mock_tier(&manager, "WARM").await;
|
||||
let identity = test_tier_destination_identity(&manager, "WARM").await;
|
||||
let mut user_defined = user_defined_with_tier_destination_identity(identity);
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(),
|
||||
);
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
@@ -6833,251 +6351,17 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect_err("explicit unknown cleanup must fail before backend IO");
|
||||
.expect_err("unknown remote version state must fail before backend IO");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("version state is unknown"));
|
||||
assert_eq!(backend.op_log().await, Vec::<MockWarmOp>::new());
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn test_tier_destination_identity(
|
||||
manager: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
tier: &str,
|
||||
) -> crate::services::tier::tier::TierDestinationId {
|
||||
TierConfigMgr::acquire_operation_lease(manager, tier)
|
||||
.await
|
||||
.expect("test tier lease should be available")
|
||||
.backend_identity()
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
fn user_defined_with_tier_destination_identity(
|
||||
identity: crate::services::tier::tier::TierDestinationId,
|
||||
) -> HashMap<String, String> {
|
||||
let mut user_defined = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(identity),
|
||||
);
|
||||
user_defined
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
fn user_defined_with_transition_version_state(state: rustfs_filemeta::TransitionVersionState) -> HashMap<String, String> {
|
||||
let mut user_defined = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
state.as_str().to_string(),
|
||||
);
|
||||
user_defined
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
fn insert_legacy_transition_version_id(user_defined: &mut HashMap<String, String>, version_id: &str) {
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
version_id.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_tuple_rejects_mixed_legacy_missing_and_explicit_unknown() {
|
||||
let manager = TierConfigMgr::new();
|
||||
register_mock_tier(&manager, "WARM").await;
|
||||
let identity = test_tier_destination_identity(&manager, "WARM").await;
|
||||
let mut legacy_metadata = user_defined_with_tier_destination_identity(identity);
|
||||
insert_legacy_transition_version_id(&mut legacy_metadata, "legacy-version");
|
||||
let mut explicit_metadata = legacy_metadata.clone();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut explicit_metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown.as_str().to_string(),
|
||||
);
|
||||
let make_info = |user_defined: HashMap<String, String>| ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = super::free_version_remote_tuple_matches(&make_info(legacy_metadata), &make_info(explicit_metadata))
|
||||
.expect_err("mixed legacy-missing and explicit unknown provenance must fail closed");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_probes_exact_version_hidden_by_current_delete_marker() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = "WARM";
|
||||
let backend = register_mock_tier(&manager, tier).await;
|
||||
let identity = test_tier_destination_identity(&manager, tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
let body = Bytes::from_static(b"legacy exact cleanup body");
|
||||
let remote_version = backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(body),
|
||||
i64::try_from(b"legacy exact cleanup body".len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("mock remote object should be stored");
|
||||
let mut user_defined = user_defined_with_tier_destination_identity(identity);
|
||||
insert_legacy_transition_version_id(&mut user_defined, &remote_version);
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Missing))
|
||||
.await;
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state(&remote_object)
|
||||
.await
|
||||
.expect("current remote view should be readable"),
|
||||
TransitionCandidateProbe::Missing,
|
||||
"a current delete marker must hide the historical data version from an unversioned probe"
|
||||
);
|
||||
backend.clear_op_log().await;
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object.clone(),
|
||||
version_id: remote_version,
|
||||
tier: tier.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect("probe-proven legacy exact cleanup should delete the remote version");
|
||||
super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect("a retry after the exact remote version is already missing should be idempotent");
|
||||
|
||||
assert_eq!(
|
||||
backend.op_log().await,
|
||||
vec![
|
||||
MockWarmOp::Get {
|
||||
object: remote_object.clone()
|
||||
},
|
||||
MockWarmOp::Remove {
|
||||
object: remote_object.clone()
|
||||
},
|
||||
MockWarmOp::Get {
|
||||
object: remote_object.clone()
|
||||
},
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
vec![(remote_object, object_info.transitioned_object.version_id)]
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_retains_legacy_unknown_unversioned_object() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = "WARM";
|
||||
let backend = register_mock_tier(&manager, tier).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
let identity = test_tier_destination_identity(&manager, tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
let body = Bytes::from_static(b"legacy unversioned cleanup body");
|
||||
let remote_version = backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(body),
|
||||
i64::try_from(b"legacy unversioned cleanup body".len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("mock remote object should be stored");
|
||||
assert!(remote_version.is_empty());
|
||||
backend.clear_op_log().await;
|
||||
let mut user_defined = user_defined_with_tier_destination_identity(identity);
|
||||
user_defined.insert("x-minio-internal-transitioned-versionID".to_string(), String::new());
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object.clone(),
|
||||
version_id: String::new(),
|
||||
tier: tier.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect_err("legacy unversioned cleanup cannot exclude a versioning-state race");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock);
|
||||
assert!(backend.op_log().await.is_empty());
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert!(backend.remove_versions().await.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_delete_does_not_remove_a_different_remote_version() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let tier = "WARM";
|
||||
let backend = register_mock_tier(&manager, tier).await;
|
||||
let identity = test_tier_destination_identity(&manager, tier).await;
|
||||
let remote_object = format!("remote/{}", Uuid::new_v4());
|
||||
backend.set_put_remote_version(Some("different-version".to_string())).await;
|
||||
backend
|
||||
.put(
|
||||
&remote_object,
|
||||
ReaderImpl::Body(Bytes::from_static(b"different remote version")),
|
||||
i64::try_from(b"different remote version".len()).expect("body length should fit"),
|
||||
)
|
||||
.await
|
||||
.expect("different remote version should be stored");
|
||||
backend.clear_op_log().await;
|
||||
let mut user_defined = user_defined_with_tier_destination_identity(identity);
|
||||
insert_legacy_transition_version_id(&mut user_defined, "legacy-version");
|
||||
let object_info = ObjectInfo {
|
||||
transitioned_object: TransitionedObject {
|
||||
name: remote_object.clone(),
|
||||
version_id: "legacy-version".to_string(),
|
||||
tier: tier.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
user_defined: user_defined.into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
super::delete_free_version_remote_object(&object_info, &manager)
|
||||
.await
|
||||
.expect("a missing exact legacy version should be an idempotent cleanup success");
|
||||
|
||||
assert_eq!(backend.op_log().await, vec![MockWarmOp::Get { object: remote_object }]);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert!(backend.remove_versions().await.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
async fn free_version_remote_delete_requires_persisted_destination_identity() {
|
||||
|
||||
@@ -1170,7 +1170,6 @@ pub async fn save_manual_transition_job_record_if_current(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(current_etag.to_string()),
|
||||
..Default::default()
|
||||
@@ -1243,7 +1242,6 @@ pub(crate) async fn save_manual_transition_worker_result_if_absent(
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -1272,7 +1270,6 @@ pub(crate) async fn save_manual_transition_task_if_absent(
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -1624,7 +1621,6 @@ pub async fn save_manual_transition_scope_admission_if_absent(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -1676,7 +1672,6 @@ pub async fn save_manual_transition_scope_admission_if_current(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(current_etag.to_string()),
|
||||
..Default::default()
|
||||
|
||||
@@ -1733,7 +1733,6 @@ async fn save_config_if_none_fenced(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -1833,7 +1832,6 @@ async fn save_decommission_manifest_checkpoint_if_match(
|
||||
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
no_lock: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(observed_etag),
|
||||
@@ -1962,7 +1960,6 @@ async fn save_config_if_match_fenced(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag.to_string()),
|
||||
..Default::default()
|
||||
@@ -3783,7 +3780,6 @@ where
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -3873,7 +3869,6 @@ where
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
@@ -3898,7 +3893,6 @@ where
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
|
||||
@@ -70,11 +72,9 @@ static REMOTE_DELETE_BREAKER: LazyLock<Mutex<RemoteDeleteBreaker>> = LazyLock::n
|
||||
});
|
||||
|
||||
#[cfg(test)]
|
||||
type RemoteTierDeleteTestHook = Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>;
|
||||
|
||||
#[cfg(test)]
|
||||
static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<std::sync::Mutex<Option<RemoteTierDeleteTestHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
|
||||
static REMOTE_TIER_DELETE_TEST_HOOK: std::sync::LazyLock<
|
||||
std::sync::Mutex<Option<Box<dyn Fn(&str, &str, &str) -> std::io::Result<()> + Send + Sync>>>,
|
||||
> = std::sync::LazyLock::new(|| std::sync::Mutex::new(None));
|
||||
|
||||
#[derive(Debug)]
|
||||
struct RemoteDeleteBreaker {
|
||||
@@ -107,7 +107,7 @@ impl RemoteDeleteBreaker {
|
||||
fn prune(&mut self, now: Instant) {
|
||||
while let Some(ts) = self.failures.front().copied() {
|
||||
if now.duration_since(ts) > self.window {
|
||||
let _ = self.failures.pop_front();
|
||||
self.failures.pop_front();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
@@ -137,10 +137,10 @@ fn is_signer_header_error(err: &std::io::Error) -> bool {
|
||||
return false;
|
||||
}
|
||||
|
||||
if let Some(source) = err.get_ref()
|
||||
&& error_chain_contains_signer_header_marker(source)
|
||||
{
|
||||
return true;
|
||||
if let Some(source) = err.get_ref() {
|
||||
if error_chain_contains_signer_header_marker(source) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
let message = err.to_string().to_ascii_lowercase();
|
||||
@@ -205,7 +205,7 @@ impl ObjSweeper {
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn with_version(&mut self, vid: Option<Uuid>) -> &Self {
|
||||
self.version_id = vid;
|
||||
self.version_id = vid.clone();
|
||||
self
|
||||
}
|
||||
|
||||
@@ -219,7 +219,7 @@ impl ObjSweeper {
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
pub fn get_opts(&self) -> lifecycle::ObjectOpts {
|
||||
let mut opts = ObjectOpts {
|
||||
version_id: self.version_id,
|
||||
version_id: self.version_id.clone(),
|
||||
versioned: self.versioned,
|
||||
version_suspended: self.suspended,
|
||||
..Default::default()
|
||||
@@ -388,8 +388,8 @@ impl Jentry {
|
||||
impl ExpiryOp for Jentry {
|
||||
fn op_hash(&self) -> u64 {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(self.tier_name.as_bytes());
|
||||
hasher.update(self.obj_name.as_bytes());
|
||||
hasher.update(format!("{}", self.tier_name).as_bytes());
|
||||
hasher.update(format!("{}", self.obj_name).as_bytes());
|
||||
xxh64::xxh64(hasher.finalize().as_slice(), XXHASH_SEED)
|
||||
}
|
||||
|
||||
@@ -436,7 +436,7 @@ async fn delete_object_from_remote_tier_raw_with_manager(
|
||||
tier_name: &str,
|
||||
tier_config_mgr: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let lease = TierConfigMgr::acquire_operation_lease(tier_config_mgr, tier_name)
|
||||
let lease = TierConfigMgr::acquire_operation_lease(&tier_config_mgr, tier_name)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
delete_object_from_remote_tier_raw_with_lease(obj_name, rv_id, &lease, false, true).await
|
||||
|
||||
@@ -612,7 +612,6 @@ pub(crate) async fn save_transition_transaction_record(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -659,7 +658,6 @@ pub(crate) async fn save_transition_transaction_record_if_current(
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
|
||||
@@ -477,18 +477,6 @@ impl BucketMetadata {
|
||||
!self.table_bucket_config_json.is_empty()
|
||||
}
|
||||
|
||||
/// `bucket-targets.json` is stored for this bucket but this build cannot
|
||||
/// decode it.
|
||||
///
|
||||
/// Keeps "no replication targets configured" and "the target
|
||||
/// configuration cannot be read" apart, the same distinction the
|
||||
/// `fabricated` marker draws for the bucket metadata as a whole. Only
|
||||
/// meaningful after [`Self::parse_all_configs`] has run; readers must fail
|
||||
/// closed on `true` instead of serving an empty target set.
|
||||
pub fn bucket_targets_unreadable(&self) -> bool {
|
||||
!self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none()
|
||||
}
|
||||
|
||||
/// Parsed per-bucket durability override, if a valid one is stored.
|
||||
///
|
||||
/// Absent/empty/unparsable payloads all mean "no override" (the bucket
|
||||
@@ -976,32 +964,7 @@ impl BucketMetadata {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decode every stored sub-configuration into its typed field.
|
||||
///
|
||||
/// A decode failure never fails the whole load: this runs on every bucket
|
||||
/// metadata read, including startup and peer reload, so one bucket's
|
||||
/// corrupt sub-configuration must not make the bucket — or the node —
|
||||
/// unloadable. Instead the failure is *retained*: the raw bytes stay
|
||||
/// untouched and the typed field stays `None`, so `!raw.is_empty() &&
|
||||
/// typed.is_none()` is the durable "exists but cannot be read" signal that
|
||||
/// each accessor keys off. Which accessors must fail closed on it:
|
||||
///
|
||||
/// | Config | Verdict |
|
||||
/// |---|---|
|
||||
/// | policy | Fails closed: `get_bucket_policy` re-parses the raw JSON and propagates the error; `get_bucket_policy_raw` returns the stored bytes. |
|
||||
/// | object lock | Fails closed in `object_lock_config_state_from_authoritative_metadata`; a retention decision may never be taken on a guess. |
|
||||
/// | versioning | Fails closed in `get_versioning_config`; guessing Unversioned would make delete markers and version ids diverge from what is on disk. |
|
||||
/// | replication | Fails closed in `get_replication_config`. |
|
||||
/// | bucket targets | Fails closed in `get_bucket_targets_config`, and `sync_bucket_target_sys` marks the bucket unreadable in `BucketTargetSys` instead of publishing an empty target set (rustfs/backlog#2282). |
|
||||
/// | encryption | Fails closed in `get_sse_config`: degrading to "no default encryption" stores plaintext objects the operator required to be encrypted. |
|
||||
/// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
|
||||
/// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
|
||||
/// | lifecycle | Safe to degrade: no rules means no expiration and no transition, so nothing is deleted or moved on the strength of an unreadable rule set. The bucket keeps serving reads and writes. |
|
||||
/// | notification | Safe to degrade: events are an outbound side channel; no consumer draws a durability or authorization conclusion from their absence. |
|
||||
/// | tagging | Safe to degrade: bucket tags are cost-allocation labels here; object-level tag conditions come from object metadata, not this blob. |
|
||||
/// | CORS | Safe to degrade: an absent CORS configuration rejects cross-origin browser requests, which is already the restrictive direction. |
|
||||
/// | logging, website, accelerate, request payment, bucket ACL | Safe to degrade: each only shapes an optional response or an optional side channel, and none of them authorizes an action or decides whether data is retained. |
|
||||
pub(super) fn parse_all_configs(&mut self) -> Result<()> {
|
||||
fn parse_all_configs(&mut self) -> Result<()> {
|
||||
if let Err(e) = self.parse_policy_config() {
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
@@ -1125,26 +1088,20 @@ impl BucketMetadata {
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
}
|
||||
// A stored targets blob that cannot be decoded must not collapse into
|
||||
// the empty target set: that is indistinguishable from "no replication
|
||||
// configured", so replication stops and no caller ever sees an error
|
||||
// (rustfs/backlog#2282). Leaving the typed field `None` while the raw
|
||||
// bytes stay non-empty is the retained parse failure every targets
|
||||
// reader keys off; the bytes are preserved so the configuration is
|
||||
// still recoverable.
|
||||
self.bucket_target_config = None;
|
||||
if !self.bucket_targets_config_json.is_empty() {
|
||||
match serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json) {
|
||||
Ok(targets) => self.bucket_target_config = Some(targets),
|
||||
Err(e) => tracing::error!(
|
||||
if let Err(e) = serde_json::from_slice::<BucketTargets>(&self.bucket_targets_config_json)
|
||||
.map(|t| self.bucket_target_config = Some(t))
|
||||
{
|
||||
tracing::warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %self.name,
|
||||
config = "bucket_targets",
|
||||
error = %e,
|
||||
"Bucket replication targets are unreadable; replication for this bucket fails closed"
|
||||
),
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
self.bucket_target_config = Some(BucketTargets::default());
|
||||
}
|
||||
} else {
|
||||
self.bucket_target_config = Some(BucketTargets::default());
|
||||
@@ -1578,117 +1535,6 @@ mod test {
|
||||
assert_eq!(bucket_targets.targets[0].target_bucket, "target-bucket");
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2282: a stored targets blob this build cannot decode
|
||||
/// must not become the empty target set, and must stay distinguishable
|
||||
/// from a bucket that never configured a target.
|
||||
#[test]
|
||||
fn unreadable_bucket_targets_never_degrade_to_an_empty_target_set() {
|
||||
let truncated = br#"{"targets":[{"endpoint":"s3.example.com","#.to_vec();
|
||||
let mut corrupt = BucketMetadata::new("corrupt-targets");
|
||||
corrupt.bucket_targets_config_json = truncated.clone();
|
||||
|
||||
corrupt
|
||||
.parse_all_configs()
|
||||
.expect("one unreadable sub-config must not fail the whole metadata load");
|
||||
|
||||
assert!(
|
||||
corrupt.bucket_target_config.is_none(),
|
||||
"an undecodable targets blob must not produce a target set at all"
|
||||
);
|
||||
assert!(corrupt.bucket_targets_unreadable());
|
||||
assert_eq!(
|
||||
corrupt.bucket_targets_config_json, truncated,
|
||||
"the raw bytes must survive so the configuration stays recoverable"
|
||||
);
|
||||
|
||||
// The genuinely-absent case is unchanged, and the two now diverge.
|
||||
let mut absent = BucketMetadata::new("no-targets");
|
||||
absent.parse_all_configs().expect("absent targets parse");
|
||||
assert!(
|
||||
absent.bucket_target_config.as_ref().is_some_and(BucketTargets::is_empty),
|
||||
"a bucket that configured no target still reads as an empty target set"
|
||||
);
|
||||
assert!(!absent.bucket_targets_unreadable());
|
||||
}
|
||||
|
||||
/// `Credentials` carries no struct-level `serde(default)`, so one target
|
||||
/// missing `secretKey` is a hard parse error for the whole document. That
|
||||
/// must surface as "unreadable", never as "no targets configured".
|
||||
#[test]
|
||||
fn bucket_targets_missing_secret_key_are_unreadable_not_empty() {
|
||||
let mut bm = BucketMetadata::new("missing-secret-key");
|
||||
bm.bucket_targets_config_json = br#"{"targets":[{"endpoint":"s3.example.com","targetbucket":"remote","arn":"arn:rustfs:replication:us-east-1:src:1","credentials":{"accessKey":"AKIAEXAMPLE"}}]}"#.to_vec();
|
||||
|
||||
bm.parse_all_configs()
|
||||
.expect("a rejected targets document must not fail the whole metadata load");
|
||||
|
||||
assert!(
|
||||
bm.bucket_targets_unreadable(),
|
||||
"a targets document rejected for a missing secretKey is unreadable, not empty"
|
||||
);
|
||||
assert!(bm.bucket_target_config.is_none());
|
||||
}
|
||||
|
||||
/// The invariant every branch of `parse_all_configs` shares: a stored but
|
||||
/// undecodable payload keeps its raw bytes and leaves the typed field
|
||||
/// `None`, so no branch fabricates a value. What a reader may then do with
|
||||
/// that state is decided per config; see the table on `parse_all_configs`.
|
||||
#[test]
|
||||
fn every_config_branch_retains_its_parse_failure_instead_of_defaulting() {
|
||||
let malformed_xml = b"<not-a-valid-document".to_vec();
|
||||
let malformed_json = b"{not-json".to_vec();
|
||||
|
||||
let mut bm = BucketMetadata::new("all-configs-malformed");
|
||||
bm.policy_config_json = malformed_json.clone();
|
||||
bm.quota_config_json = malformed_json.clone();
|
||||
bm.bucket_targets_config_json = malformed_json.clone();
|
||||
bm.notification_config_xml = malformed_xml.clone();
|
||||
bm.lifecycle_config_xml = malformed_xml.clone();
|
||||
bm.object_lock_config_xml = malformed_xml.clone();
|
||||
bm.versioning_config_xml = malformed_xml.clone();
|
||||
bm.encryption_config_xml = malformed_xml.clone();
|
||||
bm.tagging_config_xml = malformed_xml.clone();
|
||||
bm.replication_config_xml = malformed_xml.clone();
|
||||
bm.cors_config_xml = malformed_xml.clone();
|
||||
bm.logging_config_xml = malformed_xml.clone();
|
||||
bm.website_config_xml = malformed_xml.clone();
|
||||
bm.accelerate_config_xml = malformed_xml.clone();
|
||||
bm.request_payment_config_xml = malformed_xml.clone();
|
||||
bm.public_access_block_config_xml = malformed_xml.clone();
|
||||
// `bucket_acl_config_json` is only checked for UTF-8, so only invalid
|
||||
// UTF-8 exercises its failure branch.
|
||||
bm.bucket_acl_config_json = vec![0xff, 0xfe];
|
||||
|
||||
bm.parse_all_configs()
|
||||
.expect("a bucket whose every config is corrupt must still load its metadata");
|
||||
|
||||
let cleared: [(&str, bool); 17] = [
|
||||
("policy", bm.policy_config.is_none()),
|
||||
("quota", bm.quota_config.is_none()),
|
||||
("bucket_targets", bm.bucket_target_config.is_none()),
|
||||
("notification", bm.notification_config.is_none()),
|
||||
("lifecycle", bm.lifecycle_config.is_none()),
|
||||
("object_lock", bm.object_lock_config.is_none()),
|
||||
("versioning", bm.versioning_config.is_none()),
|
||||
("encryption", bm.sse_config.is_none()),
|
||||
("tagging", bm.tagging_config.is_none()),
|
||||
("replication", bm.replication_config.is_none()),
|
||||
("cors", bm.cors_config.is_none()),
|
||||
("logging", bm.logging_config.is_none()),
|
||||
("website", bm.website_config.is_none()),
|
||||
("accelerate", bm.accelerate_config.is_none()),
|
||||
("request_payment", bm.request_payment_config.is_none()),
|
||||
("public_access_block", bm.public_access_block_config.is_none()),
|
||||
("bucket_acl", bm.bucket_acl_config.is_none()),
|
||||
];
|
||||
for (config, is_cleared) in cleared {
|
||||
assert!(is_cleared, "{config}: a corrupt payload must not be replaced by a default");
|
||||
}
|
||||
|
||||
assert_eq!(bm.bucket_targets_config_json, malformed_json, "raw bytes are retained");
|
||||
assert_eq!(bm.lifecycle_config_xml, malformed_xml, "raw bytes are retained");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_update_config_clears_parsed_config_on_delete() {
|
||||
let mut bm = BucketMetadata::new("test-bucket");
|
||||
|
||||
@@ -360,16 +360,6 @@ async fn refresh_buckets_metadata_once(sys: Arc<RwLock<BucketMetadataSys>>) {
|
||||
}
|
||||
|
||||
async fn sync_bucket_target_sys(bucket: &str, bm: &BucketMetadata) {
|
||||
if bm.bucket_targets_unreadable() {
|
||||
// "The configuration cannot be read" is not "no targets configured".
|
||||
// Publishing an empty snapshot here is what silently stopped
|
||||
// replication (rustfs/backlog#2282): mark the bucket instead, so every
|
||||
// targets reader gets a typed error, and leave any snapshot from an
|
||||
// earlier readable load in place rather than withdrawing it.
|
||||
BucketTargetSys::get().mark_targets_unreadable(bucket).await;
|
||||
return;
|
||||
}
|
||||
|
||||
BucketTargetSys::get()
|
||||
.update_all_targets(bucket, bm.bucket_target_config.as_ref())
|
||||
.await;
|
||||
@@ -655,12 +645,6 @@ pub struct BucketMetadataMutationGuard {
|
||||
}
|
||||
|
||||
impl BucketMetadataMutationGuard {
|
||||
/// Returns the storage-verified identity while both incarnation fences remain valid.
|
||||
pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
|
||||
self.ensure_valid(&self.bucket)?;
|
||||
Ok((&self.bucket, self.incarnation_id))
|
||||
}
|
||||
|
||||
fn ensure_valid(&self, bucket: &str) -> Result<()> {
|
||||
if self.bucket != bucket {
|
||||
return Err(Error::other("bucket metadata mutation guard does not match bucket"));
|
||||
@@ -680,29 +664,6 @@ async fn acquire_config_write_guard_for_incarnation(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
acquire_config_write_guard_with_migration(sys, bucket, expected_incarnation_id, true).await
|
||||
}
|
||||
|
||||
/// Scanner probes must not create an incarnation to make a capability available.
|
||||
pub async fn acquire_scanner_bucket_incarnation_fence(
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
expected_owner_id: Uuid,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
super::utils::check_valid_bucket_name(bucket)?;
|
||||
let sys = get_bucket_metadata_sys()?;
|
||||
if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() {
|
||||
return Err(Error::other("scanner bucket incarnation owner does not match"));
|
||||
}
|
||||
acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
|
||||
}
|
||||
|
||||
async fn acquire_config_write_guard_with_migration(
|
||||
sys: Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
migrate: bool,
|
||||
) -> Result<BucketMetadataMutationGuard> {
|
||||
let metadata_sys = sys.read().await.clone();
|
||||
let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
@@ -710,15 +671,13 @@ async fn acquire_config_write_guard_with_migration(
|
||||
// Legacy buckets are migrated while the lifecycle fence prevents a
|
||||
// same-name replacement. The second read under the write transaction is
|
||||
// the CAS source of truth for the actual rewrite.
|
||||
if migrate {
|
||||
await_bucket_namespace_operation(
|
||||
Some(&lifecycle_guard),
|
||||
bucket,
|
||||
"bucket config incarnation migration",
|
||||
metadata_sys.get_bucket_incarnation_id(bucket),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
await_bucket_namespace_operation(
|
||||
Some(&lifecycle_guard),
|
||||
bucket,
|
||||
"bucket config incarnation migration",
|
||||
metadata_sys.get_bucket_incarnation_id(bucket),
|
||||
)
|
||||
.await?;
|
||||
let transaction_guard = await_bucket_namespace_operation(
|
||||
Some(&lifecycle_guard),
|
||||
bucket,
|
||||
@@ -2159,9 +2118,7 @@ impl BucketMetadataSys {
|
||||
pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if !bm.public_access_block_config_xml.is_empty() && bm.public_access_block_config.is_none() {
|
||||
Err(Error::other("persisted bucket public access block configuration is invalid"))
|
||||
} else if let Some(config) = &bm.public_access_block_config {
|
||||
if let Some(config) = &bm.public_access_block_config {
|
||||
Ok((config.clone(), bm.public_access_block_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
@@ -2472,9 +2429,7 @@ impl BucketMetadataSys {
|
||||
pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if !bm.encryption_config_xml.is_empty() && bm.sse_config.is_none() {
|
||||
Err(Error::other("persisted bucket encryption configuration is invalid"))
|
||||
} else if let Some(config) = &bm.sse_config {
|
||||
if let Some(config) = &bm.sse_config {
|
||||
Ok((config.clone(), bm.encryption_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
@@ -2545,9 +2500,7 @@ impl BucketMetadataSys {
|
||||
pub async fn get_quota_config(&self, bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if !bm.quota_config_json.is_empty() && bm.quota_config.is_none() {
|
||||
Err(Error::other("persisted bucket quota configuration is invalid"))
|
||||
} else if let Some(config) = &bm.quota_config {
|
||||
if let Some(config) = &bm.quota_config {
|
||||
Ok((config.clone(), bm.quota_config_updated_at))
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
@@ -2569,9 +2522,7 @@ impl BucketMetadataSys {
|
||||
pub async fn get_bucket_targets_config(&self, bucket: &str) -> Result<BucketTargets> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
if bm.bucket_targets_unreadable() {
|
||||
Err(Error::other("persisted bucket replication target configuration is invalid"))
|
||||
} else if let Some(config) = &bm.bucket_target_config {
|
||||
if let Some(config) = &bm.bucket_target_config {
|
||||
Ok(config.clone())
|
||||
} else {
|
||||
Err(Error::ConfigNotFound)
|
||||
@@ -2642,7 +2593,6 @@ pub(crate) mod test_support {
|
||||
mod tests {
|
||||
use super::test_support::isolated_store_over_temp_disks;
|
||||
use super::*;
|
||||
use crate::bucket::bucket_target_sys::BucketTargetError;
|
||||
use crate::bucket::metadata::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG,
|
||||
BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG,
|
||||
@@ -2838,36 +2788,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The `parse_all_configs` audit (rustfs/backlog#2282): every accessor
|
||||
/// whose configuration grants something — plaintext storage, anonymous
|
||||
/// access, capacity, replication targets — reports a corrupt payload as
|
||||
/// invalid rather than as absent, because "absent" is what grants it.
|
||||
#[tokio::test]
|
||||
async fn malformed_permissive_configs_are_not_reported_as_absent() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "malformed-permissive-config";
|
||||
let mut metadata = BucketMetadata::new(bucket);
|
||||
metadata.encryption_config_xml = b"<ServerSideEncryptionConfiguration".to_vec();
|
||||
metadata.public_access_block_config_xml = b"<PublicAccessBlockConfiguration".to_vec();
|
||||
metadata.quota_config_json = b"{not-json".to_vec();
|
||||
metadata.bucket_targets_config_json = b"{not-json".to_vec();
|
||||
metadata
|
||||
.parse_all_configs()
|
||||
.expect("a corrupt sub-config must not fail the load");
|
||||
sys.set(bucket.to_string(), Arc::new(metadata)).await;
|
||||
|
||||
for (config, result) in [
|
||||
("encryption", sys.get_sse_config(bucket).await.err()),
|
||||
("public access block", sys.get_public_access_block_config(bucket).await.err()),
|
||||
("quota", sys.get_quota_config(bucket).await.err()),
|
||||
("bucket targets", sys.get_bucket_targets_config(bucket).await.err()),
|
||||
] {
|
||||
let err = result.unwrap_or_else(|| panic!("malformed {config} metadata must not read as a value"));
|
||||
assert_ne!(err, Error::ConfigNotFound, "malformed {config} metadata must not be reported as absent");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn config_states_distinguish_authoritative_absence_from_fabricated_metadata() {
|
||||
use std::sync::atomic::Ordering;
|
||||
@@ -3207,82 +3127,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_dirty_usage_incarnation_probe_does_not_migrate_legacy_metadata() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
|
||||
let bucket = "scoped-ack-legacy";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("create legacy bucket");
|
||||
}
|
||||
let mut metadata = BucketMetadata::new(bucket);
|
||||
metadata.bucket_incarnation_id = Uuid::nil();
|
||||
sys.read()
|
||||
.await
|
||||
.persist_and_set(metadata)
|
||||
.await
|
||||
.expect("persist legacy metadata");
|
||||
assert!(
|
||||
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(Uuid::new_v4()), false)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
|
||||
assert!(
|
||||
sys.read()
|
||||
.await
|
||||
.get_config_from_disk(bucket)
|
||||
.await
|
||||
.expect("read metadata")
|
||||
.bucket_incarnation_id
|
||||
.is_nil()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn scoped_dirty_usage_incarnation_rejects_deleted_and_recreated_bucket() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
|
||||
let bucket = "scoped-ack-recreated";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let old = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
|
||||
let guard = acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
|
||||
.await
|
||||
.expect("trusted incarnation fence");
|
||||
assert_eq!(guard.checked_bucket_incarnation().expect("valid fences"), (bucket, old));
|
||||
drop(guard);
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete bucket");
|
||||
assert!(
|
||||
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation");
|
||||
assert_ne!(old, new);
|
||||
assert!(
|
||||
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
acquire_config_write_guard_with_migration(sys, bucket, Some(new), false)
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
|
||||
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
@@ -4222,114 +4066,6 @@ mod tests {
|
||||
target_sys.delete(bucket).await;
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2282: an unreadable `bucket-targets.json` reaches every
|
||||
/// targets reader as a typed error; it neither withdraws a snapshot a
|
||||
/// previous readable load published, nor collapses into the "no targets
|
||||
/// configured" state that a bucket with an absent configuration reports.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn unreadable_bucket_targets_fail_closed_and_stay_distinct_from_absent() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let target_sys = BucketTargetSys::get();
|
||||
let unreadable = "targets-unreadable";
|
||||
let absent = "targets-absent";
|
||||
target_sys.delete(unreadable).await;
|
||||
target_sys.delete(absent).await;
|
||||
|
||||
// A readable load publishes this bucket's targets.
|
||||
let mut readable = BucketMetadata::new(unreadable);
|
||||
readable.bucket_target_config = Some(BucketTargets {
|
||||
targets: vec![target(unreadable, "live")],
|
||||
});
|
||||
sync_bucket_target_sys(unreadable, &readable).await;
|
||||
assert_eq!(
|
||||
target_sys
|
||||
.list_bucket_targets(unreadable)
|
||||
.await
|
||||
.expect("readable targets publish")
|
||||
.targets
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
|
||||
// The same bucket reloaded with a blob that cannot be decoded.
|
||||
let mut corrupt = BucketMetadata::new(unreadable);
|
||||
corrupt.bucket_targets_config_json = br#"{"targets":[{"endpoint":"#.to_vec();
|
||||
corrupt
|
||||
.parse_all_configs()
|
||||
.expect("an unreadable targets blob must not fail the metadata load");
|
||||
sys.set(unreadable.to_string(), Arc::new(corrupt)).await;
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
target_sys.list_bucket_targets(unreadable).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
),
|
||||
"an unreadable configuration must not read as an empty or a missing target set"
|
||||
);
|
||||
assert!(
|
||||
target_sys.list_targets(unreadable, "").await.is_err(),
|
||||
"the admin listing must surface the fault instead of an empty list"
|
||||
);
|
||||
let err = sys
|
||||
.get_bucket_targets_config(unreadable)
|
||||
.await
|
||||
.expect_err("an unreadable targets configuration must not read as a value");
|
||||
assert_ne!(err, Error::ConfigNotFound, "unreadable must not be reported as absent");
|
||||
|
||||
// A bucket that never configured a target keeps its previous behavior.
|
||||
let mut no_targets = BucketMetadata::new(absent);
|
||||
no_targets.parse_all_configs().expect("absent targets parse");
|
||||
sys.set(absent.to_string(), Arc::new(no_targets)).await;
|
||||
assert!(
|
||||
matches!(
|
||||
target_sys.list_bucket_targets(absent).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. })
|
||||
),
|
||||
"an absent configuration must still report as a missing target set"
|
||||
);
|
||||
assert!(
|
||||
target_sys
|
||||
.list_targets(absent, "")
|
||||
.await
|
||||
.expect("an absent configuration lists no targets")
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
sys.get_bucket_targets_config(absent)
|
||||
.await
|
||||
.expect("an absent targets configuration still reads as an empty set")
|
||||
.is_empty(),
|
||||
"the absent path must keep returning an empty target set, exactly as before"
|
||||
);
|
||||
|
||||
// One bucket's unreadable configuration does not reach another bucket.
|
||||
assert!(!matches!(
|
||||
target_sys.list_bucket_targets(absent).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
));
|
||||
|
||||
// A repaired configuration takes effect on the next load, no restart.
|
||||
let mut repaired = BucketMetadata::new(unreadable);
|
||||
repaired.bucket_target_config = Some(BucketTargets {
|
||||
targets: vec![target(unreadable, "repaired")],
|
||||
});
|
||||
sync_bucket_target_sys(unreadable, &repaired).await;
|
||||
assert_eq!(
|
||||
target_sys
|
||||
.list_bucket_targets(unreadable)
|
||||
.await
|
||||
.expect("a repaired configuration clears the unreadable marker")
|
||||
.targets
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
|
||||
target_sys.delete(unreadable).await;
|
||||
target_sys.delete(absent).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn metadata_reload_clears_stale_bucket_targets_when_config_is_removed() {
|
||||
|
||||
@@ -684,7 +684,6 @@ async fn write_checkpoint(
|
||||
};
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(preconditions),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -86,12 +86,7 @@ impl BreakerVerdict {
|
||||
Some(SourceError::Throttled | SourceError::Timeout | SourceError::Connect(_) | SourceError::ServerError(_)) => {
|
||||
BreakerVerdict::Failure
|
||||
}
|
||||
Some(
|
||||
SourceError::AccessDenied
|
||||
| SourceError::Unsupported(_)
|
||||
| SourceError::InvalidPagination(_)
|
||||
| SourceError::Other(_),
|
||||
) => BreakerVerdict::Neutral,
|
||||
Some(SourceError::AccessDenied | SourceError::Unsupported(_) | SourceError::Other(_)) => BreakerVerdict::Neutral,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,8 +189,8 @@ pub enum SourceListPlan {
|
||||
/// delimiter — the source's own roll-up boundary matches the request's.
|
||||
Page { prefix: String },
|
||||
/// `filter.prefix` reaches past a delimiter, so every key the source could
|
||||
/// contribute rolls into this one common prefix. Bounded probes follow
|
||||
/// empty progressing pages until a key proves existence or the source ends.
|
||||
/// contribute rolls into this one common prefix. One bounded probe listing
|
||||
/// decides whether it exists; there is nothing to paginate.
|
||||
Folded { probe_prefix: String, common_prefix: String },
|
||||
}
|
||||
|
||||
@@ -279,29 +279,6 @@ pub struct FetchRequest {
|
||||
pub token: Option<String>,
|
||||
}
|
||||
|
||||
/// Invalid pagination metadata. Opaque cursor values are never included in errors.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ListPageError {
|
||||
#[error("truncated listing has no continuation token")]
|
||||
Missing,
|
||||
#[error("truncated listing has an empty continuation token")]
|
||||
Empty,
|
||||
#[error("truncated listing repeats a continuation token")]
|
||||
Repeated,
|
||||
}
|
||||
|
||||
pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> {
|
||||
if is_truncated {
|
||||
match next_token {
|
||||
None => return Err(ListPageError::Missing),
|
||||
Some("") => return Err(ListPageError::Empty),
|
||||
Some(next) if Some(next) == token => return Err(ListPageError::Repeated),
|
||||
Some(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct SideState {
|
||||
start: SideCursor,
|
||||
@@ -387,11 +364,6 @@ impl ListThroughMerger {
|
||||
/// or `filter.prefix` excludes it.
|
||||
pub fn disable_source(&mut self) {
|
||||
self.source.disabled = true;
|
||||
// A refill can fail after a valid first page. A local-only response
|
||||
// must discard both that source payload and its ordering horizon.
|
||||
self.source.entries.clear();
|
||||
self.source.pages.clear();
|
||||
self.source.more = false;
|
||||
}
|
||||
|
||||
pub fn next_fetch(&self) -> Option<FetchRequest> {
|
||||
@@ -406,13 +378,7 @@ impl ListThroughMerger {
|
||||
/// Records one fetched page. `entries` must be sorted by `name` and already
|
||||
/// filtered with [`Self::accepts`]; the caller keeps the matching payloads
|
||||
/// in the same order.
|
||||
pub fn push_page(
|
||||
&mut self,
|
||||
side: MergeSide,
|
||||
entries: Vec<ListEntryKey>,
|
||||
is_truncated: bool,
|
||||
next_token: Option<String>,
|
||||
) -> Result<(), ListPageError> {
|
||||
pub fn push_page(&mut self, side: MergeSide, entries: Vec<ListEntryKey>, is_truncated: bool, next_token: Option<String>) {
|
||||
let state = match side {
|
||||
MergeSide::Local => &mut self.local,
|
||||
MergeSide::Source => &mut self.source,
|
||||
@@ -421,19 +387,15 @@ impl ListThroughMerger {
|
||||
Some(last) => last.next_token.clone(),
|
||||
None => state.start.token.clone(),
|
||||
};
|
||||
validate_list_page(is_truncated, token.as_deref(), next_token.as_deref())?;
|
||||
// Also reject a cycle through an earlier page in this bounded fetch.
|
||||
if is_truncated && state.pages.iter().any(|page| page.token == next_token) {
|
||||
return Err(ListPageError::Repeated);
|
||||
}
|
||||
state.more = is_truncated;
|
||||
// A truncated page without a cursor cannot be continued; treating the
|
||||
// side as finished is the only alternative to looping on it forever.
|
||||
state.more = is_truncated && next_token.is_some();
|
||||
state.pages.push(FetchedPage {
|
||||
token,
|
||||
count: entries.len(),
|
||||
next_token: is_truncated.then_some(next_token).flatten(),
|
||||
});
|
||||
state.entries.extend(entries);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn finish(self) -> MergeOutcome {
|
||||
@@ -637,15 +599,9 @@ mod tests {
|
||||
let (entries, truncated, next) = reference_page(keys, prefix, delimiter, fetch.token.as_deref(), max_keys);
|
||||
let kept: Vec<ListEntryKey> = entries.into_iter().filter(|entry| merger.accepts(&entry.name)).collect();
|
||||
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.iter().cloned());
|
||||
merger
|
||||
.push_page(fetch.side, kept, truncated, next)
|
||||
.expect("reference provider pages must advance");
|
||||
merger.push_page(fetch.side, kept, truncated, next);
|
||||
}
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(outcome.is_truncated, outcome.next_token.is_some());
|
||||
if outcome.is_truncated {
|
||||
assert_ne!(outcome.next_token, token, "every truncated merged page must make progress");
|
||||
}
|
||||
page_sizes.push(outcome.picks.len());
|
||||
for pick in &outcome.picks {
|
||||
let entry = buffers[usize::from(pick.side == MergeSide::Source)][pick.index].clone();
|
||||
@@ -660,25 +616,11 @@ mod tests {
|
||||
}
|
||||
|
||||
fn expected(local: &[String], source: &[String], prefix: &str, delimiter: Option<&str>) -> Vec<ListEntryKey> {
|
||||
// This oracle builds the complete namespace independently of the
|
||||
// provider's page/marker helper and the production merger.
|
||||
let mut namespace = std::collections::BTreeMap::new();
|
||||
for key in local.iter().chain(source) {
|
||||
let Some(suffix) = key.strip_prefix(prefix) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(delimiter) = delimiter.filter(|delimiter| !delimiter.is_empty())
|
||||
&& let Some((directory, _)) = suffix.split_once(delimiter)
|
||||
{
|
||||
namespace.insert(format!("{prefix}{directory}{delimiter}"), true);
|
||||
continue;
|
||||
}
|
||||
namespace.insert(key.clone(), false);
|
||||
}
|
||||
namespace
|
||||
.into_iter()
|
||||
.map(|(name, is_prefix)| ListEntryKey { name, is_prefix })
|
||||
.collect()
|
||||
let mut all: Vec<String> = local.iter().chain(source.iter()).cloned().collect();
|
||||
all.sort();
|
||||
all.dedup();
|
||||
let (entries, _, _) = reference_page(&all, prefix, delimiter, None, usize::MAX);
|
||||
entries
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -720,9 +662,7 @@ mod tests {
|
||||
token: None
|
||||
})
|
||||
);
|
||||
merger
|
||||
.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None)
|
||||
.expect("local EOF is valid");
|
||||
merger.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None);
|
||||
assert_eq!(merger.next_fetch(), None);
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(outcome.picks.len(), 1);
|
||||
@@ -743,14 +683,12 @@ mod tests {
|
||||
};
|
||||
let mut merger = ListThroughMerger::new(1, Some(&resume));
|
||||
merger.disable_source();
|
||||
merger
|
||||
.push_page(
|
||||
MergeSide::Local,
|
||||
vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
|
||||
true,
|
||||
Some("local-2".to_string()),
|
||||
)
|
||||
.expect("local cursor advances");
|
||||
merger.push_page(
|
||||
MergeSide::Local,
|
||||
vec![ListEntryKey::object("b"), ListEntryKey::object("c")],
|
||||
true,
|
||||
Some("local-2".to_string()),
|
||||
);
|
||||
let outcome = merger.finish();
|
||||
assert!(outcome.is_truncated);
|
||||
let token = outcome.next_token.expect("truncated page carries a token");
|
||||
@@ -760,212 +698,6 @@ mod tests {
|
||||
assert_eq!(token.local.as_deref(), Some("local-1"), "a partly read page is re-listed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_pages_require_a_nonempty_advancing_cursor() {
|
||||
for side in [MergeSide::Local, MergeSide::Source] {
|
||||
for entries in [vec![], vec![ListEntryKey::object("a")]] {
|
||||
for (next, expected) in [
|
||||
(None, Err(ListPageError::Missing)),
|
||||
(Some(""), Err(ListPageError::Empty)),
|
||||
(Some("stuck"), Err(ListPageError::Repeated)),
|
||||
(Some("advances"), Ok(())),
|
||||
] {
|
||||
let resume = ListThroughToken::new(
|
||||
SideCursor {
|
||||
token: Some("stuck".into()),
|
||||
done: false,
|
||||
},
|
||||
SideCursor {
|
||||
token: Some("stuck".into()),
|
||||
done: false,
|
||||
},
|
||||
None,
|
||||
);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
let result = merger.push_page(side, entries.clone(), true, next.map(str::to_string));
|
||||
assert_eq!(result, expected, "{side:?}, {entries:?}, {next:?}");
|
||||
let state = if side == MergeSide::Local {
|
||||
&merger.local
|
||||
} else {
|
||||
&merger.source
|
||||
};
|
||||
assert_eq!(state.pages.len(), usize::from(result.is_ok()), "invalid page must not be accepted");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_empty_cursor_is_rejected_before_an_identical_page_can_escape() {
|
||||
let resume = ListThroughToken::new(
|
||||
SideCursor { token: None, done: true },
|
||||
SideCursor {
|
||||
token: Some("stuck".into()),
|
||||
done: false,
|
||||
},
|
||||
None,
|
||||
);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
assert_eq!(
|
||||
merger.next_fetch(),
|
||||
Some(FetchRequest {
|
||||
side: MergeSide::Source,
|
||||
token: Some("stuck".into())
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
|
||||
Err(ListPageError::Repeated)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_pages_may_advance_within_the_fetch_budget_until_eof() {
|
||||
let mut merger = ListThroughMerger::new(2, None);
|
||||
merger.push_page(MergeSide::Local, vec![], false, None).expect("local EOF");
|
||||
for next in ["opaque-z", "opaque-a"] {
|
||||
assert_eq!(merger.next_fetch().expect("bounded source fetch").side, MergeSide::Source);
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![], true, Some(next.into()))
|
||||
.expect("opaque cursor advances regardless of sort order");
|
||||
}
|
||||
assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget");
|
||||
let outcome = merger.finish();
|
||||
assert!(outcome.picks.is_empty());
|
||||
assert!(outcome.is_truncated);
|
||||
let token = outcome.next_token.expect("empty progressing page has a cursor");
|
||||
assert_eq!(token.source.as_deref(), Some("opaque-a"));
|
||||
let mut merger = ListThroughMerger::new(2, Some(&token));
|
||||
assert_eq!(merger.next_fetch().expect("source resumes").token.as_deref(), Some("opaque-a"));
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None)
|
||||
.expect("source EOF");
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(
|
||||
outcome.picks,
|
||||
vec![MergePick {
|
||||
side: MergeSide::Source,
|
||||
index: 0
|
||||
}]
|
||||
);
|
||||
assert!(!outcome.is_truncated);
|
||||
assert!(outcome.next_token.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cursor_cycle_inside_the_fetch_budget_is_rejected() {
|
||||
let resume = ListThroughToken::new(
|
||||
SideCursor { token: None, done: true },
|
||||
SideCursor {
|
||||
token: Some("first".into()),
|
||||
done: false,
|
||||
},
|
||||
None,
|
||||
);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![], true, Some("second".into()))
|
||||
.expect("first page advances");
|
||||
assert_eq!(
|
||||
merger.push_page(MergeSide::Source, vec![], true, Some("first".into())),
|
||||
Err(ListPageError::Repeated)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_refill_failure_discards_buffered_source_entries_and_horizon() {
|
||||
let mut merger = ListThroughMerger::new(2, None);
|
||||
merger
|
||||
.push_page(MergeSide::Local, vec![ListEntryKey::object("z")], false, None)
|
||||
.expect("local EOF");
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![ListEntryKey::object("a")], true, Some("stuck".into()))
|
||||
.expect("first source page advances");
|
||||
assert_eq!(merger.next_fetch().expect("source refill is required").token.as_deref(), Some("stuck"));
|
||||
assert_eq!(
|
||||
merger.push_page(MergeSide::Source, vec![], true, Some("stuck".into())),
|
||||
Err(ListPageError::Repeated)
|
||||
);
|
||||
merger.disable_source();
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(
|
||||
outcome.picks,
|
||||
vec![MergePick {
|
||||
side: MergeSide::Local,
|
||||
index: 0
|
||||
}]
|
||||
);
|
||||
assert!(!outcome.is_truncated);
|
||||
assert!(outcome.next_token.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_through_static_namespace_boundary_matrix() {
|
||||
let corpus = [
|
||||
"a",
|
||||
"a/",
|
||||
"a/b",
|
||||
"a/b/child",
|
||||
"a0",
|
||||
"b",
|
||||
"b/leaf",
|
||||
"quote\"&<",
|
||||
"space key",
|
||||
"z",
|
||||
"é",
|
||||
"中/文",
|
||||
];
|
||||
for count in [0, 1, 3, 4, corpus.len()] {
|
||||
let keys: Vec<String> = corpus[..count].iter().map(|key| (*key).to_string()).collect();
|
||||
for placement in 0..3 {
|
||||
let (local, source): (Vec<_>, Vec<_>) =
|
||||
keys.iter()
|
||||
.enumerate()
|
||||
.fold((vec![], vec![]), |(mut local, mut source), (index, key)| {
|
||||
if placement != 1 || index % 2 == 0 {
|
||||
local.push(key.clone());
|
||||
}
|
||||
if placement != 0 || index % 2 == 0 {
|
||||
source.push(key.clone());
|
||||
}
|
||||
(local, source)
|
||||
});
|
||||
for prefix in ["", "a", "a/", "中/"] {
|
||||
for delimiter in [None, Some("/")] {
|
||||
for max_keys in [1, 3, 4] {
|
||||
let oracle = expected(&local, &source, prefix, delimiter);
|
||||
let (emitted, sizes) = walk(&local, &source, prefix, delimiter, max_keys);
|
||||
assert_eq!(
|
||||
emitted.iter().map(|(entry, _)| entry.clone()).collect::<Vec<_>>(),
|
||||
oracle,
|
||||
"count={count}, placement={placement}, prefix={prefix}, delimiter={delimiter:?}, max={max_keys}"
|
||||
);
|
||||
let expected_sizes: Vec<_> = if oracle.is_empty() {
|
||||
vec![0]
|
||||
} else {
|
||||
oracle.chunks(max_keys).map(<[ListEntryKey]>::len).collect()
|
||||
};
|
||||
assert_eq!(sizes, expected_sizes, "exact max and max+1 boundaries must agree");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_through_large_overlap_walk_keeps_all_5300_keys() {
|
||||
let source: Vec<_> = (0..5000).map(|index| format!("k{index:05}")).collect();
|
||||
let local: Vec<_> = (4800..5300).map(|index| format!("k{index:05}")).collect();
|
||||
let (emitted, sizes) = walk(&local, &source, "", None, 333);
|
||||
assert_eq!(emitted.len(), 5300);
|
||||
for (index, (entry, side)) in emitted.iter().enumerate() {
|
||||
assert_eq!(entry.name, format!("k{index:05}"));
|
||||
assert_eq!(*side, if index >= 4800 { MergeSide::Local } else { MergeSide::Source });
|
||||
}
|
||||
assert_eq!(sizes, [vec![333; 15], vec![305]].concat());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_round_trips_and_rejects_tampering() {
|
||||
let token = ListThroughToken::new(
|
||||
@@ -1064,10 +796,7 @@ mod tests {
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig {
|
||||
rng_seed: proptest::test_runner::RngSeed::Fixed(0xec5706),
|
||||
..ProptestConfig::with_cases(256)
|
||||
})]
|
||||
#![proptest_config(ProptestConfig::with_cases(256))]
|
||||
|
||||
/// Full pagination of a merged listing equals the sorted, deduplicated
|
||||
/// union of both sides, with every shared key served by local, and no
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
//! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never
|
||||
//! forwarded: v1 rejects SSE-C source objects outright.
|
||||
|
||||
use super::list_through::{ListPageError, validate_list_page};
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config,
|
||||
};
|
||||
@@ -224,8 +223,6 @@ pub enum SourceError {
|
||||
ServerError(u16),
|
||||
#[error("unsupported source object: {0}")]
|
||||
Unsupported(String),
|
||||
#[error("invalid source listing: {0}")]
|
||||
InvalidPagination(#[from] ListPageError),
|
||||
#[error("source request failed: {0}")]
|
||||
Other(String),
|
||||
}
|
||||
@@ -248,7 +245,6 @@ impl SourceError {
|
||||
SourceError::Connect(_) => "connect",
|
||||
SourceError::ServerError(_) => "server_error",
|
||||
SourceError::Unsupported(_) => "unsupported",
|
||||
SourceError::InvalidPagination(_) => "invalid_pagination",
|
||||
SourceError::Other(_) => "other",
|
||||
}
|
||||
}
|
||||
@@ -718,7 +714,6 @@ impl SourceClient {
|
||||
..*request
|
||||
})
|
||||
.await?;
|
||||
validate_list_page(page.is_truncated, request.continuation_token, page.next_continuation_token.as_deref())?;
|
||||
page.objects = page
|
||||
.objects
|
||||
.into_iter()
|
||||
@@ -805,6 +800,11 @@ impl SourceBackend for S3SourceBackend {
|
||||
|
||||
let is_truncated = output.is_truncated.unwrap_or(false);
|
||||
let next_continuation_token = output.next_continuation_token;
|
||||
if is_truncated && next_continuation_token.is_none() {
|
||||
return Err(SourceError::Other(
|
||||
"source reported a truncated listing without a continuation token".to_string(),
|
||||
));
|
||||
}
|
||||
let objects = output
|
||||
.contents
|
||||
.unwrap_or_default()
|
||||
@@ -1274,9 +1274,7 @@ mod tests {
|
||||
<CommonPrefixes><Prefix>data/photos/</Prefix></CommonPrefixes>
|
||||
<CommonPrefixes><Prefix>outside/</Prefix></CommonPrefixes>
|
||||
</ListBucketResult>"#;
|
||||
let next_body = body.replace("data/opaque", "data/next");
|
||||
let (client, requests) =
|
||||
scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), &next_body)]).await;
|
||||
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), body), ok(Vec::new(), body)]).await;
|
||||
let first = client
|
||||
.list_page(&SourceListRequest {
|
||||
prefix: Some("photos/"),
|
||||
@@ -1338,104 +1336,7 @@ mod tests {
|
||||
.list_objects_v2(None, None, 10)
|
||||
.await
|
||||
.expect_err("truncated page without token is corrupt");
|
||||
assert!(matches!(err, SourceError::InvalidPagination(ListPageError::Missing)), "{err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_page_validates_s3_cursor_progress_before_mapping_entries() {
|
||||
for contents in ["", "<Contents><Key>data/a</Key><Size>1</Size></Contents>"] {
|
||||
for (truncated, next, expected) in [
|
||||
(true, None, Some(ListPageError::Missing)),
|
||||
(true, Some(""), Some(ListPageError::Empty)),
|
||||
(true, Some("stuck"), Some(ListPageError::Repeated)),
|
||||
(true, Some("opaque-next"), None),
|
||||
(false, None, None),
|
||||
(false, Some("stuck"), None),
|
||||
] {
|
||||
let next_xml = next
|
||||
.map(|next| format!("<NextContinuationToken>{next}</NextContinuationToken>"))
|
||||
.unwrap_or_default();
|
||||
let body = format!(
|
||||
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><IsTruncated>{truncated}</IsTruncated>{next_xml}{contents}</ListBucketResult>"
|
||||
);
|
||||
let (client, requests) = scripted_client(&spec(Some("data/")), vec![ok(Vec::new(), &body)]).await;
|
||||
let result = client
|
||||
.list_page(&SourceListRequest {
|
||||
continuation_token: Some("stuck"),
|
||||
max_keys: 2,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
match expected {
|
||||
Some(expected) => {
|
||||
let error = result.expect_err("malformed pagination must fail at the provider boundary");
|
||||
assert!(
|
||||
matches!(&error, SourceError::InvalidPagination(actual) if *actual == expected),
|
||||
"{error:?}"
|
||||
);
|
||||
assert_eq!(error.class_label(), "invalid_pagination");
|
||||
assert!(!error.is_retryable());
|
||||
assert!(!error.to_string().contains("stuck"), "errors must not echo opaque tokens");
|
||||
}
|
||||
None => {
|
||||
let page = result.expect("progressing empty/nonempty pages and EOF are valid");
|
||||
assert_eq!(page.is_truncated, truncated);
|
||||
assert_eq!(page.next_continuation_token.as_deref(), next);
|
||||
assert_eq!(page.objects.len(), usize::from(!contents.is_empty()));
|
||||
if let Some(object) = page.objects.first() {
|
||||
assert_eq!(object.key, "a");
|
||||
}
|
||||
}
|
||||
}
|
||||
let requests = recorded(&requests);
|
||||
assert_eq!(requests.len(), 1, "invalid pagination must not be retried");
|
||||
assert!(requests[0].uri.contains("continuation-token=stuck"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ListOnlyBackend(SourcePage);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl SourceBackend for ListOnlyBackend {
|
||||
async fn list(&self, request: &SourceListRequest<'_>) -> Result<SourcePage, SourceError> {
|
||||
assert_eq!(request.continuation_token, Some("stuck"), "opaque cursors reach every provider unchanged");
|
||||
Ok(self.0.clone())
|
||||
}
|
||||
|
||||
async fn head(&self, _key: &str) -> Result<SourceHead, SourceError> {
|
||||
panic!("unexpected HEAD in list test")
|
||||
}
|
||||
async fn get(&self, _key: &str, _range: Option<&HTTPRangeSpec>) -> Result<SourceGet, SourceError> {
|
||||
panic!("unexpected GET in list test")
|
||||
}
|
||||
async fn tagging(&self, _key: &str) -> Result<HashMap<String, String>, SourceError> {
|
||||
panic!("unexpected tagging in list test")
|
||||
}
|
||||
async fn probe(&self) -> Result<(), SourceError> {
|
||||
panic!("unexpected probe in list test")
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_page_validates_non_s3_provider_cursors_at_the_common_boundary() {
|
||||
for (next, expected) in [
|
||||
(None, ListPageError::Missing),
|
||||
(Some(""), ListPageError::Empty),
|
||||
(Some("stuck"), ListPageError::Repeated),
|
||||
] {
|
||||
let mut client = prefix_client(Some("data/".into()));
|
||||
client.backend = Box::new(ListOnlyBackend(SourcePage {
|
||||
is_truncated: true,
|
||||
next_continuation_token: next.map(str::to_string),
|
||||
..Default::default()
|
||||
}));
|
||||
let error = client
|
||||
.list_objects_v2(None, Some("stuck"), 2)
|
||||
.await
|
||||
.expect_err("all providers must advance pagination");
|
||||
assert!(matches!(error, SourceError::InvalidPagination(actual) if actual == expected));
|
||||
}
|
||||
assert!(matches!(err, SourceError::Other(_)), "{err:?}");
|
||||
}
|
||||
|
||||
const TAGGING_BODY: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
@@ -177,7 +177,7 @@ impl From<&SourceError> for PullFailureReason {
|
||||
SourceError::Connect(_) => PullFailureReason::SourceConnect,
|
||||
SourceError::ServerError(_) => PullFailureReason::SourceServerError,
|
||||
SourceError::Unsupported(_) => PullFailureReason::SourceUnsupported,
|
||||
SourceError::InvalidPagination(_) | SourceError::Other(_) => PullFailureReason::SourceOther,
|
||||
SourceError::Other(_) => PullFailureReason::SourceOther,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -652,10 +652,9 @@ async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_async::time::TimeSource;
|
||||
use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
|
||||
RemoteS3EndpointSpec {
|
||||
@@ -825,174 +824,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct ClockSkewTimeSource(Arc<AtomicU64>);
|
||||
|
||||
impl TimeSource for ClockSkewTimeSource {
|
||||
fn now(&self) -> SystemTime {
|
||||
SystemTime::UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct ClockSkewConnector {
|
||||
request_headers: RecordedHeaders,
|
||||
error_code: &'static str,
|
||||
skew_seconds: i64,
|
||||
clock: ClockSkewTimeSource,
|
||||
}
|
||||
|
||||
fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {
|
||||
headers
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.as_str())
|
||||
.unwrap_or_else(|| panic!("signed request must contain {name}"))
|
||||
}
|
||||
|
||||
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
|
||||
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
|
||||
.expect("SDK signing timestamp must use the SigV4 format")
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for ClockSkewConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let mut headers = self.request_headers.lock().expect("clock skew request capture lock");
|
||||
assert!(headers.len() < 3, "clock skew fixture must not exceed two GET attempts and one HEAD");
|
||||
headers.push(
|
||||
request
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
|
||||
+ chrono::Duration::seconds(self.skew_seconds);
|
||||
let (status, body) = if headers.len() == 1 {
|
||||
(
|
||||
403,
|
||||
format!("<Error><Code>{}</Code><Message>Clock skew fixture</Message></Error>", self.error_code),
|
||||
)
|
||||
} else {
|
||||
(200, String::new())
|
||||
};
|
||||
let response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
.expect("clock skew fixture response");
|
||||
HttpConnectorFuture::ready(Ok(HttpResponse::try_from(response).expect("Smithy fixture response")))
|
||||
}
|
||||
}
|
||||
|
||||
async fn clock_skew_client(
|
||||
error_code: &'static str,
|
||||
skew_seconds: i64,
|
||||
retry: RemoteS3RetryPolicy,
|
||||
) -> (S3Client, RecordedHeaders, ClockSkewTimeSource) {
|
||||
let headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
|
||||
let clock = ClockSkewTimeSource(Arc::new(AtomicU64::new(1_700_000_000)));
|
||||
let connector = SharedHttpConnector::new(ClockSkewConnector {
|
||||
request_headers: Arc::clone(&headers),
|
||||
error_code,
|
||||
skew_seconds,
|
||||
clock: clock.clone(),
|
||||
});
|
||||
let mut spec = spec("s3.example.com", true);
|
||||
spec.retry = retry;
|
||||
let config = build_remote_s3_config(&spec)
|
||||
.await
|
||||
.expect("clock skew fixture uses the production outbound configuration")
|
||||
.http_client(http_client_fn(move |_settings, _components| connector.clone()))
|
||||
.time_source(clock.clone())
|
||||
.build();
|
||||
(S3Client::from_conf(config), headers, clock)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn remote_s3_clock_skew_retries_resign_and_seed_next_operation() {
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for skew_seconds in [-600, 600] {
|
||||
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
|
||||
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
|
||||
client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.expect("clock skew GET must retry successfully");
|
||||
assert_eq!(
|
||||
headers.lock().expect("captured requests").len(),
|
||||
2,
|
||||
"{error_code}: GET needs exactly one retry"
|
||||
);
|
||||
clock.0.fetch_add(17, Ordering::SeqCst);
|
||||
// SDK signing time is independent of Tokio's retry/scheduler clock.
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
client
|
||||
.head_bucket()
|
||||
.bucket("bucket")
|
||||
.send()
|
||||
.await
|
||||
.expect("subsequent HEAD must use the client's cached skew");
|
||||
let headers = headers.lock().expect("captured signed requests");
|
||||
assert_eq!(headers.len(), 3, "subsequent operation must succeed on its first attempt");
|
||||
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
|
||||
assert_eq!(
|
||||
signing_time(&headers[1]),
|
||||
initial + chrono::Duration::seconds(skew_seconds),
|
||||
"{error_code}: retry must apply the measured offset exactly"
|
||||
);
|
||||
assert_eq!(
|
||||
signing_time(&headers[2]),
|
||||
initial + chrono::Duration::seconds(skew_seconds + 17),
|
||||
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
|
||||
);
|
||||
let signature = |index: usize| {
|
||||
recorded_header(&headers[index], "authorization")
|
||||
.rsplit_once("Signature=")
|
||||
.expect("SigV4 authorization contains a signature")
|
||||
.1
|
||||
};
|
||||
assert_ne!(
|
||||
signature(0),
|
||||
signature(1),
|
||||
"{error_code}: retry must be signed again after adjusting its date"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn remote_s3_clock_skew_respects_one_attempt_policy() {
|
||||
use aws_smithy_types::error::metadata::ProvideErrorMetadata;
|
||||
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for retry in [
|
||||
RemoteS3RetryPolicy::Disabled,
|
||||
RemoteS3RetryPolicy::Standard { max_attempts: 1 },
|
||||
] {
|
||||
let (client, headers, _clock) = clock_skew_client(error_code, 600, retry).await;
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("clock skew must not override the caller's one-attempt budget");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some(error_code));
|
||||
assert_eq!(
|
||||
headers.lock().expect("captured requests").len(),
|
||||
1,
|
||||
"{error_code}: {retry:?} must send exactly one request"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_style_auto_and_path_force_path_style() {
|
||||
assert!(PathStyle::Auto.force_path_style());
|
||||
|
||||
@@ -46,7 +46,7 @@ use super::replication_storage_boundary::{
|
||||
HTTPPreconditions, ObjectInfo, ObjectOptions, ObjectToDelete, ReplicationDeletedObject, ReplicationObjectIO,
|
||||
ReplicationStorage,
|
||||
};
|
||||
use super::replication_target_boundary::{BucketTargetError, ReplicationTargetStore, replication_object_is_ssec_encrypted};
|
||||
use super::replication_target_boundary::{ReplicationTargetStore, replication_object_is_ssec_encrypted};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use futures_util::stream::{self, StreamExt};
|
||||
@@ -3084,23 +3084,6 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
|
||||
|
||||
let tgts = match ReplicationTargetStore::list_bucket_targets(bucket).await {
|
||||
Ok(targets) => Some(targets),
|
||||
// A bucket whose persisted target configuration cannot be decoded has
|
||||
// an unknown target set, not an empty one: scheduling against `None`
|
||||
// here would drop every heal for it without a trace
|
||||
// (rustfs/backlog#2282). Report it missed so the object is retried
|
||||
// once the configuration is readable again.
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. }) => {
|
||||
warn!(
|
||||
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket,
|
||||
reason = "target_config_unreadable",
|
||||
"Bucket replication targets are unreadable; replication heal queue fails closed"
|
||||
);
|
||||
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED,
|
||||
|
||||
@@ -15,8 +15,7 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::BucketTargetError;
|
||||
use crate::bucket::bucket_target_sys::BucketTargetSys;
|
||||
use crate::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys};
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectOutput;
|
||||
use aws_sdk_s3::types::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
|
||||
use http::HeaderMap;
|
||||
|
||||
@@ -203,7 +203,7 @@ mod tests {
|
||||
use parking_lot::Mutex;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn encode_context(context: &BTreeMap<String, String>) -> String {
|
||||
fn encode_context(context: &HashMap<String, String>) -> String {
|
||||
let ordered = context.iter().collect::<BTreeMap<_, _>>();
|
||||
serde_json::to_string(&ordered).expect("context serializes")
|
||||
}
|
||||
|
||||
@@ -30,7 +30,6 @@ use rustfs_protos::{
|
||||
ChannelClass, create_new_channel, get_channel_for_class,
|
||||
proto_gen::node_service::{
|
||||
heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient,
|
||||
scanner_control_service_client::ScannerControlServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
},
|
||||
};
|
||||
@@ -61,24 +60,6 @@ pub async fn node_service_time_out_client(
|
||||
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
|
||||
}
|
||||
|
||||
pub(crate) async fn scanner_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
) -> crate::error::Result<ScannerControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = match runtime_sources::cached_node_channel(addr).await {
|
||||
Some(channel) => channel,
|
||||
None => create_new_channel(addr)
|
||||
.await
|
||||
.map_err(|err| crate::error::Error::other(err.to_string()))?,
|
||||
};
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
|
||||
Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(limit)
|
||||
.max_encoding_message_size(limit))
|
||||
}
|
||||
|
||||
pub async fn heal_control_time_out_client(
|
||||
addr: &str,
|
||||
interceptor: TonicInterceptor,
|
||||
|
||||
@@ -2050,53 +2050,6 @@ impl PeerRestClient {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Probe only: scoped ACK production requires a durable per-bucket proof.
|
||||
pub async fn scanner_scoped_dirty_usage_capability(
|
||||
&self,
|
||||
owner_id: String,
|
||||
instance_id: String,
|
||||
entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
|
||||
) -> Result<bool> {
|
||||
use rustfs_protos::scoped_dirty_usage::*;
|
||||
let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id,
|
||||
instance_id,
|
||||
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
|
||||
probe_only: true,
|
||||
entries,
|
||||
};
|
||||
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
&self.grid_host,
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| response.cleared != 0
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
}
|
||||
Ok(response.supported)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
|
||||
let result = self
|
||||
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
|
||||
|
||||
@@ -5493,7 +5493,6 @@ where
|
||||
fence.ensure_held()?;
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
no_lock: true,
|
||||
http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
|
||||
..Default::default()
|
||||
@@ -14413,7 +14412,6 @@ impl ECStore {
|
||||
encoded.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -14568,7 +14566,6 @@ impl ECStore {
|
||||
encoded,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(http_preconditions),
|
||||
..Default::default()
|
||||
},
|
||||
@@ -14960,7 +14957,6 @@ impl ECStore {
|
||||
encoded,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
|
||||
@@ -317,22 +317,6 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
|
||||
dst_path: &str,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
|
||||
.await
|
||||
.result
|
||||
}
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
pub(in crate::disk) async fn rename_data_observed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::RenameDataObservation {
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -349,35 +333,22 @@ impl LocalDiskWrapper {
|
||||
} else {
|
||||
get_max_timeout_duration()
|
||||
};
|
||||
let observed = run_owned_mutation(external_guard, move || async move {
|
||||
let mut preflight_rejection = None;
|
||||
let result = operation
|
||||
run_owned_mutation(external_guard, move || async move {
|
||||
operation
|
||||
.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed =
|
||||
Box::pin(
|
||||
operation
|
||||
.disk
|
||||
.rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
|
||||
)
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
operation
|
||||
.disk
|
||||
.rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path)
|
||||
.await
|
||||
},
|
||||
timeout_duration,
|
||||
)
|
||||
.await;
|
||||
// Health tracking must observe the real disk error, not an Ok tuple.
|
||||
Ok(super::RenameDataObservation {
|
||||
result,
|
||||
preflight_rejection,
|
||||
})
|
||||
.await
|
||||
})
|
||||
.await;
|
||||
observed.unwrap_or_else(|error| super::RenameDataObservation::unknown(Err(error)))
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2617,46 +2588,6 @@ mod tests {
|
||||
assert_eq!(wrapper.metrics_snapshot().api_calls.get("unknown"), Some(&1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_preflight_evidence_preserves_health_errors_and_owned_reply() {
|
||||
for source_exists in [false, true] {
|
||||
for guarded in [false, true] {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8"))
|
||||
.expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
if source_exists {
|
||||
disk.make_volume("source").await.expect("source volume should exist");
|
||||
}
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let external_guard = guarded.then(|| Arc::new(DropProbe(Arc::clone(&drops))) as Arc<dyn Send + Sync>);
|
||||
let mut file_info = FileInfo::new("object", 1, 0);
|
||||
file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
|
||||
file_info.erasure.index = 1;
|
||||
let observed = wrapper
|
||||
.rename_data_observed("source", "object", &file_info, "missing-destination", "object", external_guard)
|
||||
.await;
|
||||
assert!(observed.rejected_before_publication(), "normal access rejection must carry proof");
|
||||
assert!(matches!(observed.result, Err(DiskError::VolumeNotFound)));
|
||||
let snapshot = wrapper.metrics_snapshot();
|
||||
assert_eq!(snapshot.api_calls.get("rename_data"), Some(&1));
|
||||
assert_eq!(snapshot.total_writes, 0, "health tracking must not observe the rejection as Ok");
|
||||
assert_eq!(drops.load(Ordering::SeqCst), usize::from(guarded));
|
||||
|
||||
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
let observed = wrapper
|
||||
.rename_data_observed("source", "object", &file_info, "missing-destination", "object", None)
|
||||
.await;
|
||||
assert!(!observed.rejected_before_publication(), "wrapper errors carry no local preflight proof");
|
||||
assert!(matches!(observed.result, Err(DiskError::FaultyDisk)));
|
||||
let snapshot = wrapper.metrics_snapshot();
|
||||
assert_eq!(snapshot.total_errors_availability, 1);
|
||||
assert_eq!(snapshot.total_writes, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_disk_health_wrapper_counts_returned_availability_errors() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
+1143
-202
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -75,25 +75,6 @@ use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
pub(crate) struct RenameDataObservation {
|
||||
pub(crate) result: Result<RenameDataResp>,
|
||||
preflight_rejection: Option<local::LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
impl RenameDataObservation {
|
||||
fn unknown(result: Result<RenameDataResp>) -> Self {
|
||||
Self {
|
||||
result,
|
||||
preflight_rejection: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rejected_before_publication(&self) -> bool {
|
||||
self.result.is_err() && self.preflight_rejection.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
|
||||
pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
|
||||
|
||||
@@ -730,36 +711,6 @@ impl Disk {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed_with_fence_observed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> RenameDataObservation {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
local_disk
|
||||
.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed_with_fence(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
|
||||
@@ -870,18 +870,6 @@ impl TierFreeVersionReceiptSink {
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal PUT completion boundary; this does not change fsync or write quorum.
|
||||
#[doc(hidden)]
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum WriteCompletion {
|
||||
/// Return at write quorum when the commit owner can retain its guards.
|
||||
#[default]
|
||||
Quorum,
|
||||
/// Drain the rename fan-out before returning. Minority failures still heal
|
||||
/// after a successful quorum commit; this does not require every disk to succeed.
|
||||
TailDrained,
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
@@ -908,10 +896,6 @@ pub struct ObjectOptions {
|
||||
/// Persisted bucket incarnation observed before authorization.
|
||||
pub expected_bucket_incarnation_id: Option<Uuid>,
|
||||
pub no_lock: bool,
|
||||
/// Control-plane writers that immediately read or CAS the same namespace
|
||||
/// key use TailDrained without changing namespace lock ownership.
|
||||
#[doc(hidden)]
|
||||
pub write_completion: WriteCompletion,
|
||||
/// True when an upper layer already holds the object read lock before
|
||||
/// forwarding a no_lock read to the set layer.
|
||||
pub metadata_cache_safe: bool,
|
||||
|
||||
@@ -62,27 +62,12 @@ const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
|
||||
const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
|
||||
const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
// Keep this synchronized with the version served by node_service. Including
|
||||
// the local member in the minimum prevents an older coordinator from
|
||||
// self-authorizing a policy implemented only by newer remote peers.
|
||||
const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
/// Version 5 is reserved for a fleet whose every metadata writer preserves
|
||||
/// explicit transition version state and destination identity, and implements
|
||||
/// conditional per-generation `xl.meta` writes with strong readback. The node
|
||||
/// service must not advertise this version until the conditional writer from
|
||||
/// rustfs/backlog#684 is available.
|
||||
const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
|
||||
type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
|
||||
|
||||
fn cross_pool_fence_policy_results(
|
||||
peer_epochs: BTreeMap<String, Uuid>,
|
||||
minimum_version: u32,
|
||||
) -> (
|
||||
CrossPoolFencePolicyResult,
|
||||
CrossPoolFencePolicyResult,
|
||||
CrossPoolFencePolicyResult,
|
||||
CrossPoolFencePolicyResult,
|
||||
) {
|
||||
) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
|
||||
let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
|
||||
Ok(peer_epochs.clone())
|
||||
} else {
|
||||
@@ -93,18 +78,7 @@ fn cross_pool_fence_policy_results(
|
||||
} else {
|
||||
Err(Error::other("decommission target fence policy capability version is unsupported"))
|
||||
};
|
||||
let legacy_transition_state_reconcile_result =
|
||||
if minimum_version >= LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION {
|
||||
Ok(peer_epochs.clone())
|
||||
} else {
|
||||
Err(Error::other("legacy transition state reconcile policy capability version is unsupported"))
|
||||
};
|
||||
(
|
||||
Ok(peer_epochs),
|
||||
journal_result,
|
||||
decommission_target_fence_result,
|
||||
legacy_transition_state_reconcile_result,
|
||||
)
|
||||
(Ok(peer_epochs), journal_result, decommission_target_fence_result)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -278,21 +252,10 @@ pub(crate) struct TierDeleteJournalFleetProofToken {
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
/// Effect-window authority for one legacy transition-state reconciliation.
|
||||
///
|
||||
/// The token intentionally cannot be cloned. Its permit keeps the admitted
|
||||
/// fleet generation alive until the caller finishes the final strong
|
||||
/// readback, while revocation makes every later validation fail immediately.
|
||||
pub struct LegacyTransitionStateReconcileFleetProofToken {
|
||||
token: FleetCapabilityProofToken,
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
|
||||
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
@@ -311,10 +274,6 @@ fn decommission_target_fence_fleet_proof_slot() -> &'static std::sync::RwLock<Fl
|
||||
DECOMMISSION_TARGET_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn legacy_transition_state_reconcile_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
|
||||
if let Some(proof) = state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
@@ -485,125 +444,6 @@ pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalF
|
||||
stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
|
||||
}
|
||||
|
||||
/// Acquire one non-cloneable authority that must span the complete reconcile
|
||||
/// effect window, including its final strong readback.
|
||||
pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<LegacyTransitionStateReconcileFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let proof = {
|
||||
let state = legacy_transition_state_reconcile_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, expected_topology, Instant::now())?
|
||||
};
|
||||
let observed_peer_epochs = observe_legacy_transition_state_reconcile_fleet(expected_topology).await?;
|
||||
let state = legacy_transition_state_reconcile_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
&state,
|
||||
&proof,
|
||||
expected_topology,
|
||||
&observed_peer_epochs,
|
||||
Instant::now(),
|
||||
)
|
||||
.then_some(proof)
|
||||
}
|
||||
|
||||
fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<LegacyTransitionStateReconcileFleetProofToken> {
|
||||
let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
|
||||
let permit = state.proof.as_ref()?.generation.try_acquire()?;
|
||||
Some(LegacyTransitionStateReconcileFleetProofToken { token, _permit: permit })
|
||||
}
|
||||
|
||||
async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
|
||||
let notification_sys = get_global_notification_sys()?;
|
||||
let (peer_epochs, minimum_version) = timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(expected_topology),
|
||||
)
|
||||
.await
|
||||
.ok()?
|
||||
.ok()?;
|
||||
let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peer_epochs, minimum_version);
|
||||
reconcile_result.ok()
|
||||
}
|
||||
|
||||
/// Revalidate the exact fleet generation captured by a reconcile token with a
|
||||
/// fresh synchronous observation. Callers must await this before each
|
||||
/// conditional metadata write and after the final strong readback.
|
||||
pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_with_observer(
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
proof,
|
||||
expected_topology,
|
||||
|| observe_legacy_transition_state_reconcile_fleet(expected_topology),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn legacy_transition_state_reconcile_fleet_proof_matches_with_observer<F, Fut>(
|
||||
slot: &std::sync::RwLock<FleetCapabilityProofState>,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
expected_topology: &str,
|
||||
observe: F,
|
||||
) -> bool
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Option<BTreeMap<String, Uuid>>>,
|
||||
{
|
||||
{
|
||||
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !legacy_transition_state_reconcile_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
let Some(observed_peer_epochs) = observe().await else {
|
||||
return false;
|
||||
};
|
||||
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
&state,
|
||||
proof,
|
||||
expected_topology,
|
||||
&observed_peer_epochs,
|
||||
Instant::now(),
|
||||
)
|
||||
}
|
||||
|
||||
fn legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof._permit.generation.is_accepting()
|
||||
&& fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
|
||||
&& state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation))
|
||||
}
|
||||
|
||||
fn legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
expected_topology: &str,
|
||||
observed_peer_epochs: &BTreeMap<String, Uuid>,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_at(state, proof, expected_topology, now)
|
||||
&& proof.token.peer_epochs.as_ref() == observed_peer_epochs
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
|
||||
let state = tier_delete_journal_fleet_proof_slot()
|
||||
@@ -926,7 +766,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
cross_pool_fence_fleet_proof_slot(),
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
decommission_target_fence_fleet_proof_slot(),
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
] {
|
||||
mark_fleet_capability_topology_conflict(slot);
|
||||
}
|
||||
@@ -959,12 +798,11 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
};
|
||||
let (fence_result, journal_result, decommission_target_fence_result, reconcile_result) = match fence_probe {
|
||||
let (fence_result, journal_result, decommission_target_fence_result) = match fence_probe {
|
||||
Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
|
||||
Err(err) => {
|
||||
let message = err.to_string();
|
||||
(
|
||||
Err(Error::other(message.clone())),
|
||||
Err(Error::other(message.clone())),
|
||||
Err(Error::other(message.clone())),
|
||||
Err(Error::other(message)),
|
||||
@@ -980,7 +818,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
|
||||
} else if let Some(err) = publish_fleet_capability_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
@@ -1043,24 +880,6 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
reconcile_result,
|
||||
Instant::now(),
|
||||
)
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "legacy_transition_state_reconcile_v1",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
|
||||
}
|
||||
});
|
||||
@@ -1140,7 +959,7 @@ impl NotificationSys {
|
||||
client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
let mut minimum_version = LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION;
|
||||
let mut minimum_version = u32::MAX;
|
||||
for result in join_all(probes).await {
|
||||
let (peer, version, epoch) = result?;
|
||||
if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
|
||||
@@ -1149,6 +968,11 @@ impl NotificationSys {
|
||||
minimum_version = minimum_version.min(version);
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
// A single-node deployment has no remote member to lower the local
|
||||
// policy version advertised by this binary.
|
||||
if minimum_version == u32::MAX {
|
||||
minimum_version = DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION;
|
||||
}
|
||||
Ok((peer_epochs, minimum_version))
|
||||
}
|
||||
}
|
||||
@@ -3366,36 +3190,20 @@ mod tests {
|
||||
#[test]
|
||||
fn cross_pool_policy_versions_authorize_only_their_supported_protocols() {
|
||||
let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
|
||||
let (generic_v2, journal_v2, decommission_v2, reconcile_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
|
||||
let (generic_v2, journal_v2, decommission_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
|
||||
assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
|
||||
assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
|
||||
assert!(decommission_v2.is_err(), "v2 cannot authorize the sticky per-target decommission fence");
|
||||
assert!(reconcile_v2.is_err(), "v2 cannot authorize legacy transition-state reconciliation");
|
||||
|
||||
let (generic_v3, journal_v3, decommission_v3, reconcile_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
|
||||
let (generic_v3, journal_v3, decommission_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
|
||||
assert!(generic_v3.is_ok());
|
||||
assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
|
||||
assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence");
|
||||
assert!(reconcile_v3.is_err());
|
||||
|
||||
let (generic_v4, journal_v4, decommission_v4, reconcile_v4) =
|
||||
cross_pool_fence_policy_results(peers.clone(), LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
|
||||
let (generic_v4, journal_v4, decommission_v4) = cross_pool_fence_policy_results(peers, 4);
|
||||
assert!(generic_v4.is_ok());
|
||||
assert!(journal_v4.is_ok());
|
||||
assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations");
|
||||
assert!(
|
||||
reconcile_v4.is_err(),
|
||||
"the current local policy lacks the conditional xl.meta writer required by reconcile"
|
||||
);
|
||||
|
||||
let (generic_v5, journal_v5, decommission_v5, reconcile_v5) = cross_pool_fence_policy_results(peers, 5);
|
||||
assert!(generic_v5.is_ok());
|
||||
assert!(journal_v5.is_ok());
|
||||
assert!(decommission_v5.is_ok());
|
||||
assert!(
|
||||
reconcile_v5.is_ok(),
|
||||
"only an all-v5 fleet preserves destination identity and conditional reconcile writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3650,234 +3458,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_admits_only_compatible_single_and_multi_node_fleets() {
|
||||
let now = Instant::now();
|
||||
for peers in [
|
||||
BTreeMap::new(),
|
||||
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4()), ("peer-b".to_string(), Uuid::new_v4())]),
|
||||
] {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let (_, _, _, result) =
|
||||
cross_pool_fence_policy_results(peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", result, now).is_none());
|
||||
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("an all-compatible fleet should admit reconciliation")
|
||||
};
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_restart_drains_concurrent_effect_windows() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, original_result) =
|
||||
cross_pool_fence_policy_results(original_peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", original_result, now).is_none());
|
||||
|
||||
let (first, second) = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
(
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the first reconcile writer should be admitted"),
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the second reconcile writer should be admitted"),
|
||||
)
|
||||
};
|
||||
|
||||
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, restarted_result) =
|
||||
cross_pool_fence_policy_results(restarted_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
let blocked =
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", restarted_result, now + Duration::from_millis(1))
|
||||
.expect("a restarted member must revoke the old generation and wait for both writers");
|
||||
assert!(blocked.to_string().contains("previous generation to drain"));
|
||||
{
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(state.proof.is_none());
|
||||
assert!(state.draining_generation.is_some());
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&first,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&second,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
}
|
||||
|
||||
drop(first);
|
||||
let (_, _, _, still_blocked_result) =
|
||||
cross_pool_fence_policy_results(restarted_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", still_blocked_result, now + Duration::from_millis(2),)
|
||||
.is_some(),
|
||||
"one remaining writer must keep the successor generation closed"
|
||||
);
|
||||
|
||||
drop(second);
|
||||
let (_, _, _, admitted_result) =
|
||||
cross_pool_fence_policy_results(restarted_peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", admitted_result, now + Duration::from_millis(3),)
|
||||
.is_none(),
|
||||
"the restarted generation may publish only after every old writer drains"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_fresh_observation_closes_the_polling_window() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, original_result) =
|
||||
cross_pool_fence_policy_results(original_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", original_result, now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original fleet should admit reconciliation")
|
||||
};
|
||||
|
||||
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
|
||||
"the periodic cache has not observed the restart yet"
|
||||
);
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
&restarted_peers,
|
||||
now,
|
||||
));
|
||||
|
||||
let (_, _, _, downgraded) = cross_pool_fence_policy_results(original_peers, 4);
|
||||
assert!(
|
||||
downgraded.is_err(),
|
||||
"a synchronous observation of a downgraded peer must fail before any cached proof can authorize a write"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_reconcile_invalid_token_skips_fleet_observation() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original fleet should admit reconciliation")
|
||||
};
|
||||
revoke_fleet_capability_proof(&slot);
|
||||
|
||||
assert!(
|
||||
!legacy_transition_state_reconcile_fleet_proof_matches_with_observer(&slot, &admitted, "topology-a", || async {
|
||||
panic!("an invalid local generation must not trigger a fleet observation");
|
||||
},)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_membership_and_topology_changes_revoke_authority() {
|
||||
let now = Instant::now();
|
||||
for replacement in [
|
||||
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4()), ("peer-b".to_string(), Uuid::new_v4())]),
|
||||
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]),
|
||||
] {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let original = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original fleet should admit reconciliation")
|
||||
};
|
||||
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(replacement), now + Duration::from_millis(1),)
|
||||
.is_some(),
|
||||
"membership or process-epoch replacement must wait for the admitted writer"
|
||||
);
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
}
|
||||
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(BTreeMap::new()), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original topology should admit reconciliation")
|
||||
};
|
||||
mark_fleet_capability_topology_conflict(&slot);
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(state.topology_conflict);
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_capability_downgrade_fails_closed() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, compatible_result) =
|
||||
cross_pool_fence_policy_results(peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", compatible_result, now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("v5 should admit reconciliation")
|
||||
};
|
||||
|
||||
let (_, _, _, downgraded_result) =
|
||||
cross_pool_fence_policy_results(peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION - 1);
|
||||
let err = publish_fleet_capability_probe_result(&slot, "topology-a", downgraded_result, now + Duration::from_millis(1))
|
||||
.expect("a v4 member must revoke reconcile authority");
|
||||
assert!(err.to_string().contains("reconcile policy capability version is unsupported"));
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(state.proof.is_none());
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
assert!(
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now + Duration::from_millis(1),)
|
||||
.is_none(),
|
||||
"a downgraded fleet must remain inspect-only"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
|
||||
let now = Instant::now();
|
||||
@@ -3959,57 +3539,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("incomplete"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_reconcile_probe_rejects_missing_or_unreachable_members() {
|
||||
let missing = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let missing_err = missing
|
||||
.probe_cross_pool_fence_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("a missing member slot must prevent reconcile capability proof");
|
||||
assert!(missing_err.to_string().contains("incomplete"));
|
||||
|
||||
let unreachable = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let unreachable_err = unreachable
|
||||
.probe_cross_pool_fence_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("an unreachable member must prevent reconcile capability proof");
|
||||
assert!(unreachable_err.to_string().contains("unreachable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_reconcile_single_node_stays_closed_before_local_cas_support() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: Vec::new(),
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let (peers, minimum_version) = notification_sys
|
||||
.probe_cross_pool_fence_fleet("topology-a")
|
||||
.await
|
||||
.expect("a single-node capability probe should complete");
|
||||
assert!(peers.is_empty());
|
||||
assert_eq!(minimum_version, LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
|
||||
let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peers, minimum_version);
|
||||
assert!(
|
||||
reconcile_result.is_err(),
|
||||
"the current node must not self-authorize reconcile before the conditional writer lands"
|
||||
);
|
||||
}
|
||||
|
||||
fn build_props(endpoint: &str) -> ServerProperties {
|
||||
ServerProperties {
|
||||
endpoint: endpoint.to_string(),
|
||||
|
||||
@@ -701,7 +701,7 @@ impl WarmBackend for MockWarmBackend {
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
async fn get(&self, object: &str, _rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
self.precondition().await?;
|
||||
let barrier = self.inner.get_barrier.lock().await.take();
|
||||
if let Some(barrier) = barrier {
|
||||
@@ -719,9 +719,6 @@ impl WarmBackend for MockWarmBackend {
|
||||
let Some(stored) = objects.get(object) else {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "mock object not found"));
|
||||
};
|
||||
if !rv.is_empty() && stored.remote_version_id != rv {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::NotFound, "NoSuchVersion"));
|
||||
}
|
||||
let bytes = &stored.bytes;
|
||||
|
||||
let start = opts.start_offset.max(0) as usize;
|
||||
|
||||
@@ -2346,10 +2346,6 @@ impl WarmBackend for SharedWarmBackendProxy {
|
||||
self.0.probe_transition_candidate(object).await
|
||||
}
|
||||
|
||||
async fn probe_transition_version(&self, object: &str, remote_version_id: &str) -> io::Result<TransitionCandidateProbe> {
|
||||
self.0.probe_transition_version(object, remote_version_id).await
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> io::Result<bool> {
|
||||
self.0.in_use().await
|
||||
}
|
||||
@@ -2462,15 +2458,6 @@ impl TierOperationLease {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn probe_transition_version(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version_id: &str,
|
||||
) -> io::Result<TransitionCandidateProbe> {
|
||||
self.validate_remote_version_id(remote_version_id)?;
|
||||
self.inner.driver.probe_transition_version(object, remote_version_id).await
|
||||
}
|
||||
|
||||
pub(crate) fn is_current_generation(&self) -> bool {
|
||||
lock_unpoisoned(&self.runtime)
|
||||
.generations
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
|
||||
|
||||
@@ -143,7 +145,7 @@ mod tests {
|
||||
|
||||
assert_eq!(creds.access_key, "access");
|
||||
assert_eq!(creds.secret_key, "secret");
|
||||
assert_eq!(creds.creds_json.as_slice(), service_account);
|
||||
assert_eq!(creds.creds_json.as_slice(), &service_account[..]);
|
||||
|
||||
let wire = serde_json::to_value(&creds).expect("madmin tier credentials should encode");
|
||||
assert_eq!(wire["access"], "access");
|
||||
@@ -160,7 +162,7 @@ mod tests {
|
||||
.expect("the former RustFS field names and byte-array encoding should remain readable");
|
||||
assert_eq!(legacy.access_key, "legacy-access");
|
||||
assert_eq!(legacy.secret_key, "legacy-secret");
|
||||
assert_eq!(legacy.creds_json.as_slice(), service_account);
|
||||
assert_eq!(legacy.creds_json.as_slice(), &service_account[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -460,7 +460,6 @@ where
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -557,7 +556,6 @@ where
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(current_etag.to_string()),
|
||||
..Default::default()
|
||||
|
||||
@@ -494,7 +494,6 @@ where
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
@@ -550,7 +549,6 @@ where
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(current.record_etag.clone()),
|
||||
..Default::default()
|
||||
|
||||
@@ -40,7 +40,6 @@ use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
|
||||
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_error_response::to_error_response,
|
||||
api_put_object::{AdvancedPutOptions, PutObjectOptions},
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
@@ -49,14 +48,11 @@ use rustfs_utils::egress::validate_outbound_url;
|
||||
use rustfs_utils::http::headers::{
|
||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
|
||||
};
|
||||
use s3s::dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus};
|
||||
use s3s::header::{
|
||||
X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_REPLICATION_STATUS,
|
||||
X_AMZ_STORAGE_CLASS,
|
||||
};
|
||||
use s3s::{
|
||||
S3ErrorCode,
|
||||
dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus},
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -145,42 +141,6 @@ pub trait WarmBackend {
|
||||
async fn probe_transition_candidate(&self, _object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
Ok(TransitionCandidateProbe::Unsupported)
|
||||
}
|
||||
async fn probe_transition_version(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version_id: &str,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
if remote_version_id.is_empty() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
"an exact tier probe requires a remote version ID",
|
||||
));
|
||||
}
|
||||
self.validate_remote_version_id(remote_version_id)?;
|
||||
match self
|
||||
.get(
|
||||
object,
|
||||
remote_version_id,
|
||||
WarmBackendGetOpts {
|
||||
start_offset: 0,
|
||||
length: 1,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())),
|
||||
Err(err) if matches!(to_error_response(&err).code, S3ErrorCode::InvalidRange) => {
|
||||
Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string()))
|
||||
}
|
||||
Err(err)
|
||||
if err.kind() == std::io::ErrorKind::NotFound
|
||||
|| matches!(to_error_response(&err).code, S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) =>
|
||||
{
|
||||
Ok(TransitionCandidateProbe::Missing)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error>;
|
||||
}
|
||||
|
||||
@@ -477,17 +437,6 @@ impl WarmBackend for MeteredWarmBackend {
|
||||
Self::record(TierRequestOperation::Probe, result)
|
||||
}
|
||||
|
||||
async fn probe_transition_version(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version_id: &str,
|
||||
) -> Result<TransitionCandidateProbe, std::io::Error> {
|
||||
Self::record(
|
||||
TierRequestOperation::Probe,
|
||||
self.inner.probe_transition_version(object, remote_version_id).await,
|
||||
)
|
||||
}
|
||||
|
||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||
Self::record(TierRequestOperation::InUse, self.inner.in_use().await)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::future::Future;
|
||||
@@ -144,11 +146,11 @@ pub struct WarmBackendGCS {
|
||||
|
||||
impl WarmBackendGCS {
|
||||
pub async fn new(conf: &TierGCS, tier: &str) -> Result<Self, std::io::Error> {
|
||||
if conf.creds.is_empty() {
|
||||
if conf.creds == "" {
|
||||
return Err(std::io::Error::other("both access and secret keys are required"));
|
||||
}
|
||||
|
||||
if conf.bucket.is_empty() {
|
||||
if conf.bucket == "" {
|
||||
return Err(std::io::Error::other("no bucket name was provided"));
|
||||
}
|
||||
|
||||
@@ -193,11 +195,11 @@ impl WarmBackendGCS {
|
||||
}
|
||||
|
||||
pub fn get_dest(&self, object: &str) -> String {
|
||||
if self.prefix.is_empty() {
|
||||
object.to_string()
|
||||
} else {
|
||||
format!("{}/{}", self.prefix, object)
|
||||
let mut dest_obj = object.to_string();
|
||||
if self.prefix != "" {
|
||||
dest_obj = format!("{}/{}", &self.prefix, object);
|
||||
}
|
||||
return dest_obj;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,7 +223,7 @@ impl WarmBackend for WarmBackendGCS {
|
||||
let bucket = gcs_bucket_resource_name(&self.bucket);
|
||||
let Ok(res) = Box::pin(
|
||||
self.client
|
||||
.write_object(&bucket, self.get_dest(object), Bytes::from(d))
|
||||
.write_object(&bucket, &self.get_dest(object), Bytes::from(d))
|
||||
.send_buffered(),
|
||||
)
|
||||
.await
|
||||
@@ -238,7 +240,7 @@ impl WarmBackend for WarmBackendGCS {
|
||||
|
||||
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
|
||||
let bucket = gcs_bucket_resource_name(&self.bucket);
|
||||
let mut req = self.client.read_object(&bucket, self.get_dest(object));
|
||||
let mut req = self.client.read_object(&bucket, &self.get_dest(object));
|
||||
let mut max_response_bytes = None;
|
||||
if let Some(generation) = parse_generation(rv)? {
|
||||
req = req.set_generation(generation);
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
@@ -529,10 +529,6 @@ mod tests {
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
];
|
||||
let mut requests = Vec::new();
|
||||
for response in responses {
|
||||
@@ -626,52 +622,15 @@ mod tests {
|
||||
.await
|
||||
.expect_err("an authorization failure must not be mistaken for a missing key");
|
||||
assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate("delete-marker-hidden")
|
||||
.await
|
||||
.expect("a current delete marker should hide the data version"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_version("delete-marker-hidden", "historical-version")
|
||||
.await
|
||||
.expect("the stored historical version should be probed exactly"),
|
||||
TransitionCandidateProbe::VersionedPresent("historical-version".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_version("delete-marker-hidden", "missing-version")
|
||||
.await
|
||||
.expect("a missing exact version should be classified"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_version("missing-object", "historical-version")
|
||||
.await
|
||||
.expect("a missing key for an exact version probe should be classified"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
|
||||
let requests = fixture.await.expect("candidate fixture should join");
|
||||
for request in &requests[..6] {
|
||||
for request in requests {
|
||||
let request = request.to_ascii_lowercase();
|
||||
assert!(request.starts_with("get /bucket/"), "candidate discovery must use object GET");
|
||||
assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
|
||||
assert!(!request.contains("?versioning"));
|
||||
assert!(!request.contains("?versions"));
|
||||
}
|
||||
for request in &requests[6..] {
|
||||
let request = request.to_ascii_lowercase();
|
||||
assert!(request.starts_with("get /bucket/"), "exact discovery must use object GET");
|
||||
assert!(request.contains("\r\nrange: bytes=0-0\r\n"));
|
||||
}
|
||||
assert!(!requests[5].to_ascii_lowercase().contains("versionid="));
|
||||
assert!(requests[6].to_ascii_lowercase().contains("?versionid=historical-version"));
|
||||
assert!(requests[7].to_ascii_lowercase().contains("?versionid=missing-version"));
|
||||
assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version"));
|
||||
}
|
||||
|
||||
fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,385 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Pure metadata quorum and early-stop decisions for `SetDisks` reads.
|
||||
//!
|
||||
//! Disk scheduling, coalescing, cancellation, and late shard materialization
|
||||
//! remain with their existing owners; this module only classifies observations.
|
||||
|
||||
use crate::diagnostics::get::{
|
||||
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
||||
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
|
||||
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
||||
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
||||
};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS;
|
||||
use crate::set_disk::file_info_is_valid_for_metadata;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
|
||||
pub(in crate::set_disk) reason: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(in crate::set_disk) struct MetadataQuorumAccumulator {
|
||||
pub(in crate::set_disk) total_disks: usize,
|
||||
pub(in crate::set_disk) default_parity_count: usize,
|
||||
pub(in crate::set_disk) allow_early_stop: bool,
|
||||
pub(in crate::set_disk) valid_responses: usize,
|
||||
pub(in crate::set_disk) not_found_responses: usize,
|
||||
pub(in crate::set_disk) version_not_found_responses: usize,
|
||||
pub(in crate::set_disk) ignored_errors: usize,
|
||||
pub(in crate::set_disk) hard_errors: usize,
|
||||
pub(in crate::set_disk) candidate: Option<FileInfo>,
|
||||
pub(in crate::set_disk) candidate_votes: usize,
|
||||
// Bitset of shard indexes whose metadata matches the candidate. Erasure
|
||||
// layouts are capped at 16 shards, so this stays allocation-free on the
|
||||
// GET metadata hot path.
|
||||
candidate_shard_mask: u16,
|
||||
pub(in crate::set_disk) conflicting_metadata: bool,
|
||||
pub(in crate::set_disk) delete_marker_seen: bool,
|
||||
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
|
||||
pub(in crate::set_disk) delete_marker_votes: usize,
|
||||
pub(in crate::set_disk) requested_version_id: String,
|
||||
pub(in crate::set_disk) matching_version_votes: usize,
|
||||
}
|
||||
|
||||
impl MetadataQuorumAccumulator {
|
||||
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
|
||||
Self {
|
||||
total_disks,
|
||||
default_parity_count,
|
||||
allow_early_stop,
|
||||
valid_responses: 0,
|
||||
not_found_responses: 0,
|
||||
version_not_found_responses: 0,
|
||||
ignored_errors: 0,
|
||||
hard_errors: 0,
|
||||
candidate: None,
|
||||
candidate_votes: 0,
|
||||
candidate_shard_mask: 0,
|
||||
conflicting_metadata: false,
|
||||
delete_marker_seen: false,
|
||||
delete_marker_candidates: Vec::new(),
|
||||
delete_marker_votes: 0,
|
||||
requested_version_id: String::new(),
|
||||
matching_version_votes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
|
||||
self.requested_version_id = version_id.to_string();
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(None, file_info);
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(Some(disk_index), file_info);
|
||||
}
|
||||
|
||||
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
|
||||
if !file_info_is_valid_for_metadata(file_info) {
|
||||
self.hard_errors = self.hard_errors.saturating_add(1);
|
||||
return;
|
||||
}
|
||||
|
||||
self.valid_responses = self.valid_responses.saturating_add(1);
|
||||
|
||||
// Track version match for versioned requests
|
||||
if !self.requested_version_id.is_empty()
|
||||
&& let Some(ref vid) = file_info.version_id
|
||||
&& vid.to_string() == self.requested_version_id
|
||||
{
|
||||
self.matching_version_votes = self.matching_version_votes.saturating_add(1);
|
||||
}
|
||||
|
||||
if file_info.is_canonical_delete_marker() {
|
||||
self.delete_marker_seen = true;
|
||||
if let Some((_, votes)) = self
|
||||
.delete_marker_candidates
|
||||
.iter_mut()
|
||||
.find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
|
||||
{
|
||||
*votes = votes.saturating_add(1);
|
||||
} else {
|
||||
self.delete_marker_candidates.push((file_info.clone(), 1));
|
||||
}
|
||||
self.delete_marker_votes = self
|
||||
.delete_marker_candidates
|
||||
.iter()
|
||||
.map(|(_, votes)| *votes)
|
||||
.max()
|
||||
.unwrap_or_default();
|
||||
self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
|
||||
return;
|
||||
}
|
||||
|
||||
match &self.candidate {
|
||||
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
|
||||
self.candidate_votes = self.candidate_votes.saturating_add(1);
|
||||
if let Some(disk_index) = disk_index
|
||||
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
|
||||
{
|
||||
self.candidate_shard_mask |= bit;
|
||||
}
|
||||
}
|
||||
Some(_) => {
|
||||
self.conflicting_metadata = true;
|
||||
}
|
||||
None => {
|
||||
self.candidate = Some(file_info.clone());
|
||||
self.candidate_votes = 1;
|
||||
if let Some(disk_index) = disk_index
|
||||
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
|
||||
{
|
||||
self.candidate_shard_mask |= bit;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
|
||||
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
|
||||
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
|
||||
return None;
|
||||
}
|
||||
Some(1u16 << (erasure_index - 1))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
|
||||
self.candidate_read_reserve_target()
|
||||
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
|
||||
let candidate = self.candidate.as_ref()?;
|
||||
Some(
|
||||
candidate
|
||||
.erasure
|
||||
.data_blocks
|
||||
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
|
||||
)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
|
||||
match err {
|
||||
DiskError::FileNotFound | DiskError::VolumeNotFound => {
|
||||
self.not_found_responses = self.not_found_responses.saturating_add(1);
|
||||
}
|
||||
DiskError::FileVersionNotFound => {
|
||||
self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
|
||||
}
|
||||
_ if is_metadata_fanout_ignored_error(err) => {
|
||||
self.ignored_errors = self.ignored_errors.saturating_add(1);
|
||||
}
|
||||
_ => {
|
||||
self.hard_errors = self.hard_errors.saturating_add(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
|
||||
if !self.allow_early_stop {
|
||||
return None;
|
||||
}
|
||||
if self.delete_marker_votes >= self.default_write_quorum() {
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
||||
});
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if self
|
||||
.candidate
|
||||
.as_ref()
|
||||
.and_then(|candidate| self.candidate_latest_quorum(candidate))
|
||||
.is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
|
||||
{
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Check if a versioned request can early-stop because the requested
|
||||
/// version_id has reached quorum across disks.
|
||||
pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
|
||||
if !self.allow_early_stop {
|
||||
return None;
|
||||
}
|
||||
if self.requested_version_id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if self.matching_version_votes >= self.read_quorum_for_version() {
|
||||
return Some(MetadataEarlyStopDecision {
|
||||
reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
|
||||
if !self.allow_early_stop {
|
||||
return false;
|
||||
}
|
||||
if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
|
||||
return true;
|
||||
}
|
||||
if self.conflicting_metadata
|
||||
|| self.delete_marker_seen
|
||||
|| self.not_found_responses > 0
|
||||
|| self.version_not_found_responses > 0
|
||||
|| self.hard_errors > 0
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if !self.requested_version_id.is_empty()
|
||||
&& self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
match &self.candidate {
|
||||
Some(candidate) => self
|
||||
.candidate_latest_quorum(candidate)
|
||||
.is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
|
||||
None => pending >= self.default_write_quorum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the read quorum threshold for version-aware early-stop.
|
||||
/// Uses `total_disks / 2` (like `missing_response_quorum`) when
|
||||
/// `default_parity_count` is set, otherwise requires all disks.
|
||||
pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
|
||||
self.missing_response_quorum()
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
|
||||
if !self.allow_early_stop {
|
||||
return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
|
||||
}
|
||||
if self.conflicting_metadata {
|
||||
return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
|
||||
}
|
||||
if self.delete_marker_seen {
|
||||
return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
|
||||
}
|
||||
let missing_response_quorum = self.missing_response_quorum();
|
||||
if self.version_not_found_responses >= missing_response_quorum {
|
||||
return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
|
||||
}
|
||||
if self.not_found_responses >= missing_response_quorum {
|
||||
return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
|
||||
}
|
||||
if self.hard_errors > 0 {
|
||||
return GET_METADATA_EARLY_STOP_REASON_ERROR;
|
||||
}
|
||||
if self.ignored_errors > 0 {
|
||||
return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
|
||||
}
|
||||
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
|
||||
if self.default_parity_count == 0 {
|
||||
return Some(self.total_disks);
|
||||
}
|
||||
if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
|
||||
return None;
|
||||
}
|
||||
let data_blocks = candidate.erasure.data_blocks;
|
||||
Some(if data_blocks == candidate.erasure.parity_blocks {
|
||||
data_blocks.saturating_add(1)
|
||||
} else {
|
||||
data_blocks
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn default_write_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
return self.total_disks;
|
||||
}
|
||||
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
|
||||
if data_blocks == self.default_parity_count {
|
||||
data_blocks.saturating_add(1)
|
||||
} else {
|
||||
data_blocks
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
self.total_disks
|
||||
} else {
|
||||
self.total_disks / 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
|
||||
left.volume == right.volume
|
||||
&& left.name == right.name
|
||||
&& left.version_id == right.version_id
|
||||
&& left.is_latest == right.is_latest
|
||||
&& left.deleted == right.deleted
|
||||
&& left.mark_deleted == right.mark_deleted
|
||||
&& left.transition_status == right.transition_status
|
||||
&& left.transitioned_objname == right.transitioned_objname
|
||||
&& left.transition_tier == right.transition_tier
|
||||
&& left.transition_version_id == right.transition_version_id
|
||||
&& left.transition_version == right.transition_version
|
||||
&& left.transition_version_state == right.transition_version_state
|
||||
&& left.expire_restored == right.expire_restored
|
||||
&& left.size == right.size
|
||||
&& left.mod_time == right.mod_time
|
||||
&& left.mode == right.mode
|
||||
&& left.written_by_version == right.written_by_version
|
||||
&& left.metadata == right.metadata
|
||||
&& left.replication_state_internal == right.replication_state_internal
|
||||
&& left.parts == right.parts
|
||||
&& left.checksum == right.checksum
|
||||
&& left.versioned == right.versioned
|
||||
&& left.num_versions == right.num_versions
|
||||
&& left.successor_mod_time == right.successor_mod_time
|
||||
&& left.data_dir == right.data_dir
|
||||
&& left.erasure.algorithm == right.erasure.algorithm
|
||||
&& left.erasure.data_blocks == right.erasure.data_blocks
|
||||
&& left.erasure.parity_blocks == right.erasure.parity_blocks
|
||||
&& left.erasure.block_size == right.erasure.block_size
|
||||
&& left.erasure.distribution == right.erasure.distribution
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
|
||||
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
|
||||
}
|
||||
@@ -18,4 +18,3 @@
|
||||
//! duplicating read/write/erasure logic.
|
||||
|
||||
pub(crate) mod io_primitives;
|
||||
mod metadata_quorum;
|
||||
|
||||
@@ -876,7 +876,7 @@ pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
pub(crate) use ops::object::DeleteObjectCommitBarrier;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::TransitionUploadedCommitBarrier as SetDiskTransitionUploadedCommitBarrier;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
|
||||
@@ -299,11 +299,11 @@ use crate::error::is_err_invalid_upload_id;
|
||||
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
|
||||
use crate::object_api::{
|
||||
NamespaceLockFence, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, WriteCompletion,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY,
|
||||
};
|
||||
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
|
||||
use crate::set_disk::core::io_primitives::{RenameRollbackReceipt, RenameTailCleanup, finish_rename_tail_heal};
|
||||
use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal};
|
||||
#[cfg(test)]
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||
#[cfg(test)]
|
||||
@@ -3548,7 +3548,6 @@ impl SetDisks {
|
||||
(None, None, None)
|
||||
};
|
||||
let mut tmp_cleanup_owned = false;
|
||||
let rollback_receipt = RenameRollbackReceipt::default();
|
||||
let operation = async {
|
||||
let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
|
||||
|
||||
@@ -4257,7 +4256,6 @@ impl SetDisks {
|
||||
let commit_bucket = bucket.to_owned();
|
||||
let commit_object = object.to_owned();
|
||||
let commit_tmp_dir = tmp_dir.clone();
|
||||
let commit_rollback_receipt = rollback_receipt.clone();
|
||||
let commit_object_lock_guard = object_lock_guard.take();
|
||||
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
|
||||
let commit_publication_guard = publication_commit_guard.take();
|
||||
@@ -4268,17 +4266,13 @@ impl SetDisks {
|
||||
// complete rename fan-out drains. Keep this path synchronous so
|
||||
// its terminal state is known before the coordinator releases
|
||||
// remote leases.
|
||||
let commit_owns_namespace_guard = commit_object_lock_guard.is_some()
|
||||
|| commit_decommission_object_lock_guard.is_some()
|
||||
|| commit_publication_guard.is_some();
|
||||
let commit_allows_early_ack = opts.write_completion == WriteCompletion::Quorum
|
||||
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& commit_owns_namespace_guard
|
||||
let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& (commit_object_lock_guard.is_some()
|
||||
|| commit_decommission_object_lock_guard.is_some()
|
||||
|| commit_publication_guard.is_some())
|
||||
&& commit_scanner_publication_scope.is_none();
|
||||
// Full-tail callers also transfer owned guards to the coordinator:
|
||||
// cancelling their ACK waiter must not cancel an in-flight rename.
|
||||
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|
||||
|| commit_owns_namespace_guard
|
||||
|| commit_allows_early_ack
|
||||
|| commit_bucket_lifecycle_guard.is_some()
|
||||
|| quota_mutation_fence;
|
||||
let commit_write_path_label = write_path.metric_label();
|
||||
@@ -4458,8 +4452,7 @@ impl SetDisks {
|
||||
write_quorum,
|
||||
commit_scanner_publication_lease_tokens.as_ref(),
|
||||
)
|
||||
.with_publication_scope(commit_scanner_publication_scope.clone())
|
||||
.with_rollback_receipt(commit_rollback_receipt.clone()),
|
||||
.with_publication_scope(commit_scanner_publication_scope.clone()),
|
||||
)
|
||||
.await;
|
||||
if let Some(scope) = commit_scanner_publication_scope.as_ref() {
|
||||
@@ -4592,11 +4585,6 @@ impl SetDisks {
|
||||
let rename_commit = match rename_result {
|
||||
Ok(commit) => commit,
|
||||
Err(err) => {
|
||||
if commit_rollback_receipt.is_incomplete() {
|
||||
// Incomplete undo retains the staging source and
|
||||
// rollback backup for recovery; cleanup is unsafe.
|
||||
return Err(err.into());
|
||||
}
|
||||
if let Err(cleanup_err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_dir).await {
|
||||
warn!(tmp_dir = %commit_tmp_dir, error = ?cleanup_err, "failed to cleanup put_object temporary data");
|
||||
} else if issue3031_diag_enabled() {
|
||||
@@ -4629,8 +4617,9 @@ impl SetDisks {
|
||||
request.object_version_id = committed_version_id
|
||||
.or_else(|| commit_version_suspended.then(Uuid::nil))
|
||||
.map(|version_id| version_id.to_string());
|
||||
let heal_set = commit_set.clone();
|
||||
tokio::spawn(async move { heal_set.submit_rename_tail_heal(request).await });
|
||||
tokio::spawn(async move {
|
||||
let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await;
|
||||
});
|
||||
}
|
||||
|
||||
let rename_stage_elapsed = rename_stage_start.elapsed();
|
||||
@@ -4896,7 +4885,7 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
});
|
||||
} else if !rollback_receipt.is_incomplete() {
|
||||
} else {
|
||||
// Failure path (quorum loss / rollback): keep the cleanup inline so
|
||||
// a failed PUT never returns while its tmp shards are still on disk
|
||||
// (state-residue hardening tracked by backlog#864 / backlog#898).
|
||||
@@ -17505,69 +17494,27 @@ mod put_object_tmp_cleanup_tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn put_object_failure_cleans_tmp_workspace_inline() {
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
|
||||
let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "tmp-clean-missing-bucket";
|
||||
let object = "orphan-object";
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
|
||||
let writer = Arc::clone(&set_disks);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("missing-bucket PUT must stage before rename");
|
||||
let staged = non_trash_tmp_entries(&temp_dirs).await;
|
||||
assert_eq!(staged.len(), 4, "every disk must have a staged workspace before rejection");
|
||||
for workspace in staged {
|
||||
let mut entries = tokio::fs::read_dir(&workspace)
|
||||
.await
|
||||
.expect("staged workspace should be readable");
|
||||
let mut shards = 0;
|
||||
while let Some(entry) = entries.next_entry().await.expect("staged data directory should be readable") {
|
||||
if entry.file_type().await.expect("staged entry type").is_dir() {
|
||||
let part = tokio::fs::metadata(entry.path().join("part.1"))
|
||||
.await
|
||||
.expect("staging must contain an actual erasure shard");
|
||||
assert!(part.len() > 0, "the shard must be written before the missing-bucket failure");
|
||||
shards += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(shards, 1);
|
||||
}
|
||||
assert!(temp_dirs.iter().all(|dir| !dir.path().join(bucket).exists()));
|
||||
barrier.release();
|
||||
let err = tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("missing-bucket PUT must finish")
|
||||
.expect("PUT task should join")
|
||||
.expect_err("put_object into a missing bucket volume must fail");
|
||||
assert!(matches!(err, StorageError::VolumeNotFound), "original disk error expected: {err}");
|
||||
let (temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
|
||||
// No polling: known pre-publication rejection must clean staging
|
||||
// inline, before PUT returns (backlog#864 / backlog#898).
|
||||
let leftovers = non_trash_tmp_entries(&temp_dirs).await;
|
||||
assert!(
|
||||
leftovers.is_empty(),
|
||||
"failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
|
||||
);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
// The bucket volume is never created, so the shards are written into
|
||||
// the tmp workspace and the commit fails at rename_data with a quorum
|
||||
// error — exercising the failure-path cleanup.
|
||||
let mut reader = PutObjReader::from_vec(vec![9u8; TEST_OBJECT_SIZE]);
|
||||
let err = set_disks
|
||||
.put_object("tmp-clean-missing-bucket", "orphan-object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("put_object into a missing bucket volume must fail");
|
||||
|
||||
// No polling: the failure path must clean the tmp workspace inline,
|
||||
// before put_object returns (backlog#864 / backlog#898 hardening).
|
||||
let leftovers = non_trash_tmp_entries(&temp_dirs).await;
|
||||
assert!(
|
||||
leftovers.is_empty(),
|
||||
"failed PUT must not leave tmp shards behind, leftovers: {leftovers:?}, err: {err}"
|
||||
);
|
||||
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -18210,354 +18157,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn make_completion_test_bucket(disks: &[DiskStore], bucket: &str) {
|
||||
for disk in disks {
|
||||
disk.make_volume(bucket)
|
||||
.await
|
||||
.expect("completion test bucket should be created");
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe the actual metadata quorum while the remaining rename is parked.
|
||||
/// A completed task count alone can race tasks that have not started yet.
|
||||
async fn wait_for_paused_tail_metadata_quorum(disks: &[DiskStore], bucket: &str, object: &str) {
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let mut committed = 0;
|
||||
for disk in disks {
|
||||
match disk.read_version("", bucket, object, "", &ReadOptions::default()).await {
|
||||
Ok(_) => committed += 1,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {}
|
||||
Err(err) => panic!("unexpected metadata error while observing {bucket}/{object}: {err}"),
|
||||
}
|
||||
}
|
||||
if committed == 3 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("three disks must publish metadata while the fourth rename remains paused");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tail_drained_put_waits_for_tail_and_allows_immediate_cas() {
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
for size in [4096, 1024 * 1024] {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-full-tail-cas";
|
||||
let object = "full-tail-cas-object";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; size]);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("full-tail PUT must reach the rename barrier");
|
||||
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
|
||||
assert!(!put.is_finished(), "full-tail PUT must remain pending after metadata quorum");
|
||||
let mut lock_probe = Box::pin(set.acquire_write_lock_diag("full_tail_probe", bucket, object));
|
||||
assert!(
|
||||
futures::poll!(lock_probe.as_mut()).is_pending(),
|
||||
"the owned namespace guard must remain held"
|
||||
);
|
||||
barrier.release();
|
||||
let written = tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("full-tail PUT should finish after release")
|
||||
.expect("full-tail PUT task should join")
|
||||
.expect("full-tail PUT must commit");
|
||||
assert_eq!(tasks.running(), 0, "full-tail response must follow every rename task");
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(5), lock_probe)
|
||||
.await
|
||||
.expect("same-key lock should be available on return")
|
||||
.expect("same-key lock probe should succeed"),
|
||||
);
|
||||
for disk in &disks {
|
||||
disk.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("successful full-tail PUT must publish on every healthy disk");
|
||||
}
|
||||
drop(barrier);
|
||||
let mut replacement = PutObjReader::from_vec(b"cas successor".to_vec());
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut replacement,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: written.etag,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("immediate same-key CAS must acquire the namespace guard");
|
||||
let mut read = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("CAS successor must be immediately readable");
|
||||
let mut body = Vec::new();
|
||||
read.stream.read_to_end(&mut body).await.expect("successor body must drain");
|
||||
assert_eq!(body, b"cas successor");
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tail_drained_put_preserves_quorum_success_and_heals_failed_tail() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-full-tail-heal";
|
||||
let object = "full-tail-heal-object";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let mut heals = set.capture_test_rename_tail_heals();
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let _fault = rename_fault_injection::fail_rename_on(object, &[0]);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("failed tail must first reach the rename barrier");
|
||||
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
|
||||
assert!(!put.is_finished(), "committed quorum must still wait for the failing tail");
|
||||
barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("failed tail should drain")
|
||||
.expect("PUT task should join")
|
||||
.expect("a minority tail error must not negate committed quorum");
|
||||
assert_eq!(tasks.running(), 0);
|
||||
let heal = tokio::time::timeout(Duration::from_secs(30), heals.recv())
|
||||
.await
|
||||
.expect("failed tail must schedule heal")
|
||||
.expect("heal capture must remain connected");
|
||||
assert_eq!(heal.bucket, bucket);
|
||||
assert_eq!(heal.object_prefix.as_deref(), Some(object));
|
||||
let info = set
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed object must remain readable despite the failed tail");
|
||||
assert_eq!(info.size, TEST_OBJECT_SIZE as i64);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tail_drained_put_rejects_quorum_minus_one() {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-full-tail-no-quorum";
|
||||
let object = "full-tail-no-quorum-object";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]);
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
let err = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("draining two successful disks cannot satisfy write quorum three");
|
||||
assert!(
|
||||
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
|
||||
"original quorum error expected: {err}"
|
||||
);
|
||||
assert_eq!(tasks.running(), 0, "failed fan-out and rollback must complete before return");
|
||||
assert!(
|
||||
set.get_object_info(bucket, object, &ObjectOptions::default()).await.is_err(),
|
||||
"failed fresh write must not become visible"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() {
|
||||
use crate::set_disk::core::io_primitives::rollback_fault_injection;
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
|
||||
for fault in [
|
||||
rollback_fault_injection::Fault::Io,
|
||||
rollback_fault_injection::Fault::VolumeNotFoundAfterRename,
|
||||
] {
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-incomplete-undo";
|
||||
let object = "incomplete-undo-object";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
let mut old_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut old_reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("old generation should be completely committed");
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "old PUT must leave no unrelated staging").await;
|
||||
let old = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("old metadata must be readable");
|
||||
let old_data_dir = old.data_dir.expect("non-inline old version needs a data directory");
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let _rename_fault = rename_fault_injection::fail_rename_on(object, &[2, 3]);
|
||||
let _undo_fault = rollback_fault_injection::arm(object, 0, fault);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("overwrite must enter the actual rename fan-out before failure injection");
|
||||
barrier.release();
|
||||
let err = tokio::time::timeout(Duration::from_secs(30), put)
|
||||
.await
|
||||
.expect("incomplete undo must return without hanging")
|
||||
.expect("PUT task should join")
|
||||
.expect_err("two renamed disks cannot satisfy write quorum three");
|
||||
assert!(
|
||||
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
|
||||
"original quorum error expected: {err}"
|
||||
);
|
||||
assert_eq!(tasks.running(), 0, "every rename and undo task must be reaped before return");
|
||||
let leftovers = non_trash_tmp_entries(&dirs).await;
|
||||
assert!(!leftovers.is_empty(), "incomplete undo must retain the new staging source for recovery");
|
||||
let backups = dirs
|
||||
.iter()
|
||||
.filter(|dir| {
|
||||
dir.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(old_data_dir.to_string())
|
||||
.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP)
|
||||
.exists()
|
||||
})
|
||||
.count();
|
||||
assert_eq!(backups, 1, "exactly the failed undo disk must retain its old-version backup");
|
||||
// The remaining three disks still serve the old generation;
|
||||
// the failed minority must never become an acknowledged write.
|
||||
let mut read = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("old generation must remain readable after incomplete rollback");
|
||||
let mut body = Vec::new();
|
||||
read.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("old generation should stream");
|
||||
assert_eq!(body, vec![b'0'; TEST_OBJECT_SIZE]);
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn tail_drained_put_owned_commit_survives_waiter_cancellation() {
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = RUSTFS_META_BUCKET;
|
||||
let object = "full-tail-cancelled-receipt";
|
||||
// Internal config writes do not own a bucket lifecycle guard. The object
|
||||
// guard alone must keep the full-tail coordinator alive after cancellation.
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
writer
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("cancelled receipt must first reach the rename barrier");
|
||||
wait_for_paused_tail_metadata_quorum(&disks, bucket, object).await;
|
||||
put.abort();
|
||||
assert!(put.await.expect_err("ACK waiter should cancel").is_cancelled());
|
||||
let mut lock_probe = Box::pin(set.acquire_write_lock_diag("cancelled_full_tail_probe", bucket, object));
|
||||
assert!(
|
||||
futures::poll!(lock_probe.as_mut()).is_pending(),
|
||||
"owned coordinator must retain the namespace guard after waiter cancellation"
|
||||
);
|
||||
barrier.release();
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(30), lock_probe)
|
||||
.await
|
||||
.expect("cancelled coordinator must eventually release its guard")
|
||||
.expect("post-commit lock probe should succeed"),
|
||||
);
|
||||
assert_eq!(tasks.running(), 0, "cancelled coordinator must reap every rename task");
|
||||
for disk in &disks {
|
||||
disk.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("caller cancellation must not interrupt committed receipt materialization");
|
||||
}
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn no_lock_put_waits_for_rename_tail_under_outer_guard() {
|
||||
@@ -18585,7 +18184,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -18611,18 +18209,7 @@ mod put_object_tmp_cleanup_tests {
|
||||
put.await
|
||||
.expect("no-lock PUT task should join")
|
||||
.expect("no-lock PUT should commit after the rename tail releases");
|
||||
let mut lock_probe = Box::pin(set_disks.acquire_write_lock_diag("borrowed_full_tail_probe", bucket, object));
|
||||
assert!(
|
||||
futures::poll!(lock_probe.as_mut()).is_pending(),
|
||||
"full-tail PUT must not release the caller's outer guard"
|
||||
);
|
||||
drop(outer_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(5), lock_probe)
|
||||
.await
|
||||
.expect("outer owner releasing its guard should unblock the probe")
|
||||
.expect("post-outer-guard probe should succeed"),
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ use super::{
|
||||
};
|
||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
|
||||
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
||||
use crate::object_api::WriteCompletion;
|
||||
use crate::services::tier::test_util::register_mock_tier;
|
||||
use crate::storage_api_contracts::bucket::BucketOperations;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
@@ -73,7 +72,7 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -186,7 +185,7 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
|
||||
@@ -2979,33 +2979,6 @@ mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
const DECOMMISSION_TEST_FAULT_STAGE_TIERED: &str = "decommission_tiered_object";
|
||||
|
||||
fn decommission_retry_fault_hook(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
faults: Arc<AtomicUsize>,
|
||||
) -> crate::core::pools::DecommissionTestFaultDecision {
|
||||
let target_bucket = bucket.to_string();
|
||||
let target_object = object.to_string();
|
||||
Arc::new(move |stage, bucket, object, _attempt, succeeded| {
|
||||
if !succeeded
|
||||
|| stage != DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
|
||||
|| bucket != target_bucket
|
||||
|| object != target_object
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Entry retries reset the local attempt; real copy errors can skip
|
||||
// successful attempts. Only injected faults spend this global budget.
|
||||
faults
|
||||
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
|
||||
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1))
|
||||
.then_some(faults.saturating_add(1))
|
||||
})
|
||||
.is_ok()
|
||||
})
|
||||
}
|
||||
|
||||
async fn seed_decommission_source(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
@@ -5147,33 +5120,6 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_retry_fault_budget_counts_successes_across_attempt_changes() {
|
||||
for attempts in [[1, 2, 3], [1, 1, 2], [1, 3, 3]] {
|
||||
let faults = Arc::new(AtomicUsize::new(0));
|
||||
let hook = decommission_retry_fault_hook("bucket", "object", Arc::clone(&faults));
|
||||
|
||||
for (stage, bucket, object, succeeded) in [
|
||||
("other-stage", "bucket", "object", true),
|
||||
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "other-bucket", "object", true),
|
||||
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "other-object", true),
|
||||
(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", false),
|
||||
] {
|
||||
assert!(!hook(stage, bucket, object, 1, succeeded));
|
||||
}
|
||||
assert_eq!(faults.load(Ordering::SeqCst), 0, "unrelated or failed copies must not consume faults");
|
||||
|
||||
for (index, attempt) in attempts.into_iter().enumerate() {
|
||||
assert_eq!(
|
||||
hook(DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT, "bucket", "object", attempt, true),
|
||||
index < 2,
|
||||
"attempts={attempts:?}, index={index}"
|
||||
);
|
||||
}
|
||||
assert_eq!(faults.load(Ordering::SeqCst), 2, "attempts={attempts:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn decommission_entry_retries_source_changed_without_canceling_other_bucket() {
|
||||
@@ -5268,8 +5214,31 @@ mod tests {
|
||||
));
|
||||
|
||||
let ordinary_faults = Arc::new(AtomicUsize::new(0));
|
||||
let fault_hook = decommission_retry_fault_hook(&other_bucket, other_object, Arc::clone(&ordinary_faults));
|
||||
let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(fault_hook);
|
||||
let ordinary_faults_for_hook = Arc::clone(&ordinary_faults);
|
||||
let fault_bucket = other_bucket.clone();
|
||||
let _fault_guard = crate::core::pools::DecommissionTestFaultGuard::install(Arc::new(
|
||||
move |stage, bucket, object, attempt, succeeded| {
|
||||
let candidate = succeeded
|
||||
&& stage == DECOMMISSION_TEST_FAULT_STAGE_MIGRATE_OBJECT
|
||||
&& bucket == fault_bucket.as_str()
|
||||
&& object == other_object;
|
||||
if !candidate {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Keep the fault budget global across any
|
||||
// entry-level re-list; its inner attempt counter
|
||||
// restarts after SourceChanged.
|
||||
ordinary_faults_for_hook
|
||||
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |faults| {
|
||||
let next_fault = faults.saturating_add(1);
|
||||
(faults < crate::core::pools::DECOMMISSION_VERSION_COPY_ATTEMPTS.saturating_sub(1)
|
||||
&& attempt == next_fault)
|
||||
.then_some(next_fault)
|
||||
})
|
||||
.is_ok()
|
||||
},
|
||||
));
|
||||
|
||||
let rx = CancellationToken::new();
|
||||
let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
|
||||
@@ -8076,15 +8045,10 @@ mod tests {
|
||||
);
|
||||
assert!(com::read_config(store.pools[0].clone(), &second_page_path).await.is_ok());
|
||||
|
||||
let full_tail = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone(), &full_tail)
|
||||
com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, receipt_bytes.clone())
|
||||
.await
|
||||
.expect("second page receipt should restore");
|
||||
com::save_config_with_opts(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec(), &full_tail)
|
||||
com::save_config(store.pools[target_pool_idx].clone(), &second_page_path, b"{corrupt".to_vec())
|
||||
.await
|
||||
.expect("second page receipt should corrupt deterministically");
|
||||
let corrupt = store
|
||||
@@ -11611,7 +11575,6 @@ mod tests {
|
||||
pool_index: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
minio_unversioned: bool,
|
||||
) {
|
||||
for disk_index in 0..4 {
|
||||
let metadata_path =
|
||||
@@ -11645,11 +11608,6 @@ mod tests {
|
||||
] {
|
||||
rustfs_utils::http::metadata_compat::remove_bytes(&mut object_meta.meta_sys, suffix);
|
||||
}
|
||||
if minio_unversioned {
|
||||
object_meta
|
||||
.meta_sys
|
||||
.insert("x-minio-internal-transitioned-versionID".to_string(), Vec::new());
|
||||
}
|
||||
*shallow = rustfs_filemeta::FileMetaShallowVersion::try_from(version)
|
||||
.expect("legacy transitioned version should re-encode");
|
||||
}
|
||||
@@ -11660,152 +11618,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn read_store_body(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
range: Option<HTTPRangeSpec>,
|
||||
opts: &ObjectOptions,
|
||||
) -> Vec<u8> {
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, object, range, HeaderMap::new(), opts)
|
||||
.await
|
||||
.expect("object reader should open");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("object body should drain");
|
||||
body
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn legacy_unknown_unversioned_transition_supports_head_get_and_range_without_backfill() {
|
||||
let temp_dir = tempfile::tempdir().expect("create legacy unknown unversioned store dir");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-unknown-unversioned-read", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let tier_name = "LEGACY-UNKNOWN-UNVERSIONED-READ";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
backend.set_put_remote_version(Some(String::new())).await;
|
||||
let bucket = "legacy-unknown-unversioned-read-bucket";
|
||||
let object = "object.bin";
|
||||
let payload = b"legacy unversioned remote tier object remains readable".repeat(1024);
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("legacy source bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let source = store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("legacy source should be written");
|
||||
store
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: source.etag.clone().expect("legacy source should have an etag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("legacy source should transition");
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, true).await;
|
||||
backend.clear_op_log().await;
|
||||
|
||||
let opts = ObjectOptions {
|
||||
metadata_cache_safe: false,
|
||||
..Default::default()
|
||||
};
|
||||
let head = store
|
||||
.get_object_info(bucket, object, &opts)
|
||||
.await
|
||||
.expect("legacy transitioned HEAD should use local metadata");
|
||||
assert_eq!(head.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
assert!(head.transitioned_object.version_id.is_empty());
|
||||
assert_eq!(
|
||||
head.user_defined
|
||||
.get("x-minio-internal-transitioned-versionID")
|
||||
.map(String::as_str),
|
||||
Some(""),
|
||||
"the MinIO empty version-key provenance must survive xl.meta decoding"
|
||||
);
|
||||
assert!(
|
||||
!rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&head.user_defined,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
),
|
||||
"the compatibility read must not synthesize version-state metadata"
|
||||
);
|
||||
|
||||
let full_body = read_store_body(&store, bucket, object, None, &opts).await;
|
||||
assert_eq!(full_body, payload);
|
||||
|
||||
let range = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 7,
|
||||
end: 38,
|
||||
};
|
||||
let ranged_body = read_store_body(&store, bucket, object, Some(range), &opts).await;
|
||||
assert_eq!(ranged_body, &payload[7..=38]);
|
||||
|
||||
let after_read = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("legacy metadata should remain readable after GET")
|
||||
.expect("legacy object metadata should remain on disk")
|
||||
.versions
|
||||
.into_iter()
|
||||
.find(|version| version.transition_status == rustfs_filemeta::TRANSITION_COMPLETE)
|
||||
.expect("legacy transitioned source should remain visible after GET");
|
||||
assert_eq!(after_read.transition_version_state, rustfs_filemeta::TransitionVersionState::Unknown);
|
||||
assert!(after_read.transition_version.is_none());
|
||||
assert!(after_read.transition_version_id.is_none());
|
||||
assert_eq!(
|
||||
after_read
|
||||
.metadata
|
||||
.get("x-minio-internal-transitioned-versionID")
|
||||
.map(String::as_str),
|
||||
Some(""),
|
||||
"the MinIO empty version-key provenance must remain after GET and Range GET"
|
||||
);
|
||||
assert!(
|
||||
!rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&after_read.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
),
|
||||
"the compatibility read must remain side-effect free"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
backend.op_log().await,
|
||||
vec![
|
||||
MockWarmOp::Probe {
|
||||
object: after_read.transitioned_objname.clone(),
|
||||
},
|
||||
MockWarmOp::Get {
|
||||
object: after_read.transitioned_objname.clone(),
|
||||
},
|
||||
MockWarmOp::Probe {
|
||||
object: after_read.transitioned_objname.clone(),
|
||||
},
|
||||
MockWarmOp::Get {
|
||||
object: after_read.transitioned_objname,
|
||||
},
|
||||
],
|
||||
"legacy reads should probe before each unversioned GET and never mutate local metadata"
|
||||
);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -11846,7 +11658,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("legacy source should transition");
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, false).await;
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object).await;
|
||||
let legacy = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
@@ -12987,7 +12799,7 @@ mod tests {
|
||||
.expect("merge-loser source should transition");
|
||||
copy_test_xlmeta_between_pools(temp_dir.path(), 0, 1, bucket, object).await;
|
||||
}
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin", false).await;
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 1, bucket, "legacy/item.bin").await;
|
||||
backend.set_remove_failure(true);
|
||||
store.pools[1]
|
||||
.delete_object(bucket, "hidden/item.bin", ObjectOptions::default())
|
||||
@@ -17054,10 +16866,6 @@ mod tests {
|
||||
.find(|version| version.version_id == history.version_id)
|
||||
.expect("transitioned history should exist");
|
||||
transitioned.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
rustfs_utils::http::metadata_compat::remove_str(
|
||||
&mut transitioned.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
);
|
||||
metadata
|
||||
.add_version(transitioned)
|
||||
.expect("unknown state should replace the transitioned version");
|
||||
|
||||
@@ -425,7 +425,7 @@ pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
mod object;
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use object::DeleteAfterObjectLockSnapshotBarrier;
|
||||
pub(crate) use object::{
|
||||
DecommissionFixedReadAnchor, ObjectLockDiagGuard, RemoteTuplePublicationCommitGuard, RemoteTuplePublicationFence,
|
||||
|
||||
@@ -297,20 +297,6 @@ fn transitioned_version_from_bytes(value: Option<&[u8]>, state: TransitionVersio
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_version_metadata_value(raw: &[u8], decoded: Option<&str>) -> String {
|
||||
decoded.map(str::to_owned).unwrap_or_else(|| {
|
||||
if raw.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
String::from_utf8_lossy(raw).into_owned()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn is_transition_version_metadata_key(key: &str) -> bool {
|
||||
strip_internal_prefix_preserving_case(key).is_some_and(|suffix| suffix.eq_ignore_ascii_case(SUFFIX_TRANSITIONED_VERSION_ID))
|
||||
}
|
||||
|
||||
fn validate_transition_version_state(state: TransitionVersionState, version: Option<&str>) -> Result<()> {
|
||||
let valid = match state {
|
||||
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
|
||||
@@ -380,26 +366,14 @@ impl<'a> DerivedInternalMetadata<'a> {
|
||||
}
|
||||
*slot = Some(value.as_slice());
|
||||
}
|
||||
fn merge_consistent<'a>(canonical: Option<&'a [u8]>, legacy: Option<&'a [u8]>) -> Result<Option<&'a [u8]>> {
|
||||
if let (Some(canonical), Some(legacy)) = (canonical, legacy)
|
||||
&& canonical != legacy
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
Ok(canonical.or(legacy))
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
checksum: canonical.checksum.or(legacy.checksum),
|
||||
part_checksums: canonical.part_checksums.or(legacy.part_checksums),
|
||||
transition_status: merge_consistent(canonical.transition_status, legacy.transition_status)?,
|
||||
transitioned_object: merge_consistent(canonical.transitioned_object, legacy.transitioned_object)?,
|
||||
transitioned_version: merge_consistent(canonical.transitioned_version, legacy.transitioned_version)?,
|
||||
transitioned_version_state: merge_consistent(
|
||||
canonical.transitioned_version_state,
|
||||
legacy.transitioned_version_state,
|
||||
)?,
|
||||
transition_tier: merge_consistent(canonical.transition_tier, legacy.transition_tier)?,
|
||||
transition_status: canonical.transition_status.or(legacy.transition_status),
|
||||
transitioned_object: canonical.transitioned_object.or(legacy.transitioned_object),
|
||||
transitioned_version: canonical.transitioned_version.or(legacy.transitioned_version),
|
||||
transitioned_version_state: canonical.transitioned_version_state.or(legacy.transitioned_version_state),
|
||||
transition_tier: canonical.transition_tier.or(legacy.transition_tier),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -464,14 +438,8 @@ impl FileInfo {
|
||||
}
|
||||
}
|
||||
|
||||
fn set_transition_version_state(
|
||||
meta_sys: &mut HashMap<String, Vec<u8>>,
|
||||
state: TransitionVersionState,
|
||||
source_metadata: &HashMap<String, String>,
|
||||
) {
|
||||
if state == TransitionVersionState::Unknown
|
||||
&& !rustfs_utils::http::metadata_compat::contains_key_str(source_metadata, SUFFIX_TRANSITIONED_VERSION_STATE)
|
||||
{
|
||||
fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: TransitionVersionState) {
|
||||
if state == TransitionVersionState::Unknown {
|
||||
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
|
||||
} else {
|
||||
insert_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, state.as_str().as_bytes().to_vec());
|
||||
@@ -2675,11 +2643,6 @@ impl MetaObject {
|
||||
if derived_metadata.transitioned_version_state.is_some() {
|
||||
validate_transition_version_state(transition_version_state, transition_version.as_deref())?;
|
||||
}
|
||||
for (key, value) in &self.meta_sys {
|
||||
if is_transition_version_metadata_key(key) {
|
||||
metadata.insert(key.to_owned(), transition_version_metadata_value(value, transition_version.as_deref()));
|
||||
}
|
||||
}
|
||||
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
|
||||
let transition_tier = derived_metadata
|
||||
.transition_tier
|
||||
@@ -2726,7 +2689,7 @@ impl MetaObject {
|
||||
} else {
|
||||
remove_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
|
||||
}
|
||||
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state, &fi.metadata);
|
||||
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state);
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec());
|
||||
if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) {
|
||||
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes());
|
||||
@@ -2867,7 +2830,7 @@ impl From<FileInfo> for MetaObject {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
}
|
||||
if !value.transition_status.is_empty() {
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata);
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state);
|
||||
}
|
||||
|
||||
if !value.transition_tier.is_empty() {
|
||||
@@ -3022,12 +2985,6 @@ impl MetaDeleteMarker {
|
||||
fi.transition_version_state = transition_version_state_from_bytes(derived_metadata.transitioned_version_state)?;
|
||||
fi.transition_version =
|
||||
transitioned_version_from_bytes(derived_metadata.transitioned_version, fi.transition_version_state);
|
||||
for (key, value) in &self.meta_sys {
|
||||
if is_transition_version_metadata_key(key) {
|
||||
fi.metadata
|
||||
.insert(key.to_owned(), transition_version_metadata_value(value, fi.transition_version.as_deref()));
|
||||
}
|
||||
}
|
||||
fi.transition_version_id = fi.transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
|
||||
if derived_metadata.transitioned_version_state.is_some() {
|
||||
validate_transition_version_state(fi.transition_version_state, fi.transition_version.as_deref())?;
|
||||
@@ -3195,7 +3152,7 @@ impl From<FileInfo> for MetaDeleteMarker {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
|
||||
}
|
||||
if !value.transition_status.is_empty() || value.tier_free_version() {
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state, &value.metadata);
|
||||
set_transition_version_state(&mut meta_sys, value.transition_version_state);
|
||||
}
|
||||
if !value.transition_tier.is_empty() {
|
||||
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
|
||||
@@ -4617,7 +4574,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("into_fileinfo");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(String::new()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4629,10 +4585,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("into_fileinfo");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert!(
|
||||
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
|
||||
"nil UUID bytes must remain distinguishable from an empty MinIO version"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4646,7 +4598,6 @@ mod tests {
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4686,36 +4637,6 @@ mod tests {
|
||||
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_explicit_unknown_is_not_legacy_missing() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
TransitionVersionState::Unknown.as_str().to_string(),
|
||||
);
|
||||
let fi = FileInfo {
|
||||
transition_status: "complete".to_string(),
|
||||
transition_version_state: TransitionVersionState::Unknown,
|
||||
metadata,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let object = MetaObject::from(fi);
|
||||
assert_eq!(
|
||||
get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE),
|
||||
Some(b"unknown".as_slice())
|
||||
);
|
||||
let decoded = object
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("explicit unknown state should decode");
|
||||
assert_eq!(decoded.transition_version_state, TransitionVersionState::Unknown);
|
||||
assert_eq!(
|
||||
rustfs_utils::http::metadata_compat::get_consistent_str(&decoded.metadata, SUFFIX_TRANSITIONED_VERSION_STATE,),
|
||||
Some("unknown")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_exact_round_trips_dual_keys() {
|
||||
let id = sample_version_id();
|
||||
@@ -4832,10 +4753,6 @@ mod tests {
|
||||
.expect("invalid transition version bytes must not fail the object read");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert_eq!(fi.transition_version, None);
|
||||
assert!(
|
||||
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
|
||||
"invalid raw bytes must remain distinguishable from an empty MinIO version"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4878,10 +4795,6 @@ mod tests {
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect("nil tier version should remain an absent remote version");
|
||||
assert_eq!(fi.transition_version_id, None);
|
||||
assert!(
|
||||
get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID).is_some_and(|value| !value.is_empty()),
|
||||
"nil UUID bytes must remain distinguishable from an empty MinIO version"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4899,7 +4812,6 @@ mod tests {
|
||||
.expect("legacy binary UUID tier version should decode");
|
||||
assert_eq!(fi.transition_version_id, Some(id));
|
||||
assert_eq!(fi.transition_version, Some(id.to_string()));
|
||||
assert_eq!(get_str(&fi.metadata, SUFFIX_TRANSITIONED_VERSION_ID), Some(id.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4998,23 +4910,6 @@ mod tests {
|
||||
assert_eq!(err, Error::FileCorrupt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_object_transition_version_state_mixed_case_alias_conflict_fails_closed() {
|
||||
let sys = HashMap::from([
|
||||
(
|
||||
format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"),
|
||||
b"unknown".to_vec(),
|
||||
),
|
||||
("X-Minio-Internal-transitioned-version-state".to_string(), b"exact".to_vec()),
|
||||
]);
|
||||
|
||||
let err = make_meta_object_with_sys(sys)
|
||||
.into_fileinfo("b", "k", false)
|
||||
.expect_err("mixed-case transition state aliases must agree");
|
||||
|
||||
assert_eq!(err, Error::FileCorrupt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_header_sorts_before_prefers_object_over_delete_marker_on_equal_mod_time() {
|
||||
let object = FileMetaVersionHeader {
|
||||
|
||||
@@ -45,11 +45,6 @@ use tracing::{debug, error, info, warn};
|
||||
use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
|
||||
|
||||
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
|
||||
// Each cache includes alias tokens in its count and byte budget. Eviction
|
||||
// removes every token sharing a snapshot; neither cache retains repair state.
|
||||
const MAX_COMPLETED_HEAL_TOKENS: usize = 1024;
|
||||
const MAX_COMPLETED_HEAL_BYTES: usize = 64 * 1024 * 1024;
|
||||
const MAX_COMPLETED_HEAL_RESULT_BYTES: usize = 1024 * 1024;
|
||||
const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again";
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
|
||||
@@ -185,8 +180,6 @@ fn record_displaced_terminal(
|
||||
request: &HealRequest,
|
||||
) -> Arc<CompletedHealStatus> {
|
||||
let terminal = Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type.clone(),
|
||||
status: HealTaskStatus::Failed {
|
||||
error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"),
|
||||
@@ -200,7 +193,6 @@ fn record_displaced_terminal(
|
||||
let mut terminals = lock_displaced_terminals(registry);
|
||||
prune_completed_heal_statuses(&mut terminals);
|
||||
terminals.insert(request.id.clone(), Arc::clone(&terminal));
|
||||
prune_completed_heal_statuses(&mut terminals);
|
||||
terminal
|
||||
}
|
||||
|
||||
@@ -217,15 +209,9 @@ async fn remove_displaced_task_aliases(
|
||||
.collect::<Vec<_>>();
|
||||
let mut displaced_terminals = lock_displaced_terminals(terminals);
|
||||
prune_completed_heal_statuses(&mut displaced_terminals);
|
||||
if displaced_terminals
|
||||
.get(task_id)
|
||||
.is_some_and(|current| Arc::ptr_eq(current, terminal))
|
||||
{
|
||||
for alias_id in alias_ids {
|
||||
displaced_terminals.insert(alias_id, Arc::clone(terminal));
|
||||
}
|
||||
for alias_id in alias_ids {
|
||||
displaced_terminals.insert(alias_id, Arc::clone(terminal));
|
||||
}
|
||||
prune_completed_heal_statuses(&mut displaced_terminals);
|
||||
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
|
||||
}
|
||||
|
||||
@@ -236,36 +222,6 @@ async fn remove_task_aliases_for_task(registry: &Arc<Mutex<HashMap<String, HealT
|
||||
.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
|
||||
}
|
||||
|
||||
// Callers hold active ownership until publication. Lock order is active ->
|
||||
// retrying (when needed) -> aliases -> completed; queries release aliases
|
||||
// before looking up active state. Publishing aliases before removing their
|
||||
// mapping keeps both an already-resolved token and a new lookup valid.
|
||||
async fn publish_completed_heal(
|
||||
completed_heals: &Mutex<HashMap<String, Arc<CompletedHealStatus>>>,
|
||||
task_aliases: &Mutex<HashMap<String, HealTaskAlias>>,
|
||||
task_id: &str,
|
||||
completed: CompletedHealStatus,
|
||||
terminal: bool,
|
||||
) {
|
||||
let completed = Arc::new(completed);
|
||||
completed.retained_bytes();
|
||||
let mut aliases = task_aliases.lock().await;
|
||||
let mut retained = completed_heals.lock().await;
|
||||
if let Some(previous) = retained.get(task_id).cloned() {
|
||||
for entry in retained.values_mut().filter(|entry| Arc::ptr_eq(entry, &previous)) {
|
||||
*entry = Arc::clone(&completed);
|
||||
}
|
||||
}
|
||||
retained.insert(task_id.to_owned(), Arc::clone(&completed));
|
||||
if terminal {
|
||||
for (alias_id, _) in aliases.iter().filter(|(_, alias)| alias.task_id == task_id) {
|
||||
retained.insert(alias_id.clone(), Arc::clone(&completed));
|
||||
}
|
||||
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
|
||||
}
|
||||
prune_completed_heal_statuses(&mut retained);
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HealTaskReport {
|
||||
pub status: HealTaskStatus,
|
||||
@@ -312,7 +268,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
|
||||
let result_items = match since {
|
||||
None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(),
|
||||
Some(cursor) => {
|
||||
if cursor.saturating_add(1) < completed.min_seq {
|
||||
if cursor + 1 < completed.min_seq {
|
||||
lagged = true;
|
||||
}
|
||||
completed
|
||||
@@ -327,7 +283,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
|
||||
status: completed.status.clone(),
|
||||
result_items,
|
||||
result_items_truncated: completed.result_items_truncated || lagged,
|
||||
progress: completed.progress.clone(),
|
||||
progress: None,
|
||||
next_seq: completed.next_seq,
|
||||
min_seq: completed.min_seq,
|
||||
}
|
||||
@@ -1891,14 +1847,14 @@ impl HealManager {
|
||||
|
||||
pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
let progress = match self.lookup_task_state(&canonical_task_id, None).await {
|
||||
TaskStateLookup::Active(task) => Some(task.get_progress().await),
|
||||
TaskStateLookup::Completed(completed) => completed.progress.clone(),
|
||||
_ => None,
|
||||
};
|
||||
progress.ok_or_else(|| Error::TaskNotFound {
|
||||
task_id: task_id.to_string(),
|
||||
})
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
if let Some(task) = active_heals.get(&canonical_task_id) {
|
||||
Ok(task.get_progress().await)
|
||||
} else {
|
||||
Err(Error::TaskNotFound {
|
||||
task_id: task_id.to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Cancel task
|
||||
@@ -1908,8 +1864,6 @@ impl HealManager {
|
||||
let mut active_heals = self.active_heals.lock().await;
|
||||
if let Some(task) = active_heals.get(&canonical_task_id) {
|
||||
task.cancel().await?;
|
||||
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
|
||||
publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await;
|
||||
active_heals.remove(&canonical_task_id);
|
||||
publish_active_heal_count(&active_heals);
|
||||
info!(
|
||||
@@ -1986,8 +1940,6 @@ impl HealManager {
|
||||
for task_id in &task_ids {
|
||||
if let Some(task) = active_heals.get(task_id) {
|
||||
task.cancel().await?;
|
||||
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
|
||||
publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await;
|
||||
}
|
||||
active_heals.remove(task_id);
|
||||
cancelled += 1;
|
||||
|
||||
@@ -82,8 +82,6 @@ pub(super) enum QueuePushOutcome {
|
||||
pub(super) struct CompletedHealStatus {
|
||||
pub(super) heal_type: HealType,
|
||||
pub(super) status: HealTaskStatus,
|
||||
pub(super) progress: Option<HealProgress>,
|
||||
pub(super) retained_bytes: std::sync::OnceLock<usize>,
|
||||
pub(super) result_items_truncated: bool,
|
||||
pub(super) completed_at: SystemTime,
|
||||
/// Sequence-stamped retained window, archived with the completion so
|
||||
@@ -94,133 +92,6 @@ pub(super) struct CompletedHealStatus {
|
||||
pub(super) min_seq: u64,
|
||||
}
|
||||
|
||||
impl CompletedHealStatus {
|
||||
// Account for owned capacities, including nested drive arrays. Aliases
|
||||
// conservatively charge the shared allocation again, keeping both token
|
||||
// count and retained payload bounded without a second ownership index.
|
||||
pub(super) fn retained_bytes(&self) -> usize {
|
||||
*self.retained_bytes.get_or_init(|| self.measure_retained_bytes())
|
||||
}
|
||||
|
||||
fn measure_retained_bytes(&self) -> usize {
|
||||
let mut bytes = size_of::<Self>();
|
||||
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
|
||||
match &self.heal_type {
|
||||
HealType::Cluster => {}
|
||||
HealType::Bucket { bucket } => add(bucket.capacity()),
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
| HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
add(bucket.capacity());
|
||||
add(object.capacity());
|
||||
add(version_id.as_ref().map_or(0, String::capacity));
|
||||
}
|
||||
HealType::Prefix { bucket, prefix } => {
|
||||
add(bucket.capacity());
|
||||
add(prefix.capacity());
|
||||
}
|
||||
HealType::Metadata { bucket, object } => {
|
||||
add(bucket.capacity());
|
||||
add(object.capacity());
|
||||
}
|
||||
HealType::ErasureSet { buckets, set_disk_id } => {
|
||||
add(buckets.capacity().saturating_mul(size_of::<String>()));
|
||||
for bucket in buckets {
|
||||
add(bucket.capacity());
|
||||
}
|
||||
add(set_disk_id.capacity());
|
||||
}
|
||||
}
|
||||
if let HealTaskStatus::Failed { error } | HealTaskStatus::Retrying { error, .. } = &self.status {
|
||||
add(error.capacity());
|
||||
}
|
||||
add(self
|
||||
.progress
|
||||
.as_ref()
|
||||
.and_then(|progress| progress.current_object.as_ref())
|
||||
.map_or(0, String::capacity));
|
||||
add(self.seqed_items.capacity().saturating_mul(size_of::<(u64, HealResultItem)>()));
|
||||
for (_, item) in &self.seqed_items {
|
||||
add(Self::result_item_heap_bytes(item));
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
fn result_item_heap_bytes(item: &HealResultItem) -> usize {
|
||||
let mut bytes = 0usize;
|
||||
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
|
||||
for value in [
|
||||
&item.heal_item_type,
|
||||
&item.bucket,
|
||||
&item.object,
|
||||
&item.version_id,
|
||||
&item.detail,
|
||||
] {
|
||||
add(value.capacity());
|
||||
}
|
||||
for infos in [&item.before, &item.after] {
|
||||
add(infos
|
||||
.drives
|
||||
.capacity()
|
||||
.saturating_mul(size_of::<rustfs_madmin::heal_commands::HealDriveInfo>()));
|
||||
for drive in &infos.drives {
|
||||
add(drive.uuid.capacity());
|
||||
add(drive.endpoint.capacity());
|
||||
add(drive.state.capacity());
|
||||
}
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
pub(super) fn bound_result_window(&mut self) {
|
||||
let mut bytes = 0usize;
|
||||
let retained = self
|
||||
.seqed_items
|
||||
.iter()
|
||||
.rev()
|
||||
.take_while(|(_, item)| {
|
||||
bytes = bytes
|
||||
.saturating_add(size_of::<(u64, HealResultItem)>())
|
||||
.saturating_add(Self::result_item_heap_bytes(item));
|
||||
bytes <= MAX_COMPLETED_HEAL_RESULT_BYTES
|
||||
})
|
||||
.count();
|
||||
let truncated = retained < self.seqed_items.len();
|
||||
if truncated {
|
||||
self.seqed_items.drain(..self.seqed_items.len() - retained);
|
||||
self.seqed_items.shrink_to_fit();
|
||||
self.min_seq = self.seqed_items.first().map_or(self.next_seq, |(seq, _)| *seq);
|
||||
self.result_items_truncated = true;
|
||||
self.retained_bytes.take();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn snapshot(task: &HealTask, status: HealTaskStatus) -> Self {
|
||||
let seqed_items = task.get_seqed_result_items().await;
|
||||
let (next_seq, min_seq) = task.result_seq_cursors();
|
||||
let mut snapshot = Self {
|
||||
heal_type: task.heal_type.clone(),
|
||||
status,
|
||||
progress: Some(task.get_progress().await),
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
result_items_truncated: task.result_items_truncated(),
|
||||
completed_at: SystemTime::now(),
|
||||
seqed_items,
|
||||
next_seq,
|
||||
min_seq,
|
||||
};
|
||||
snapshot.bound_result_window();
|
||||
snapshot
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct HealTaskAlias {
|
||||
pub(super) task_id: String,
|
||||
|
||||
@@ -264,7 +264,7 @@ impl HealManager {
|
||||
error: error.clone(),
|
||||
retry_attempt: request.retry_attempts,
|
||||
});
|
||||
let mut retry_request_for_queue = retry_request;
|
||||
let retry_request_for_queue = retry_request;
|
||||
let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new());
|
||||
if retry_request_for_queue.is_none() {
|
||||
replacement_recovery_anchors_clone
|
||||
@@ -272,35 +272,7 @@ impl HealManager {
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.remove(&task_id);
|
||||
}
|
||||
let mut completed_status = match retry_request_for_status {
|
||||
Some(status) => status,
|
||||
None => task.get_status().await,
|
||||
};
|
||||
let mut completed_status_entry = CompletedHealStatus::snapshot(&task, completed_status.clone()).await;
|
||||
let completed_progress = task.get_progress().await;
|
||||
#[cfg(test)]
|
||||
tests::pause_completed_retention_before_publish(&task_id, &completed_status).await;
|
||||
let mut active_heals_guard = active_heals_clone.lock().await;
|
||||
let owns_completion = active_heals_guard.contains_key(&task_id);
|
||||
let cancelled_completion = if owns_completion {
|
||||
false
|
||||
} else {
|
||||
// Cancellation can win while a finished worker waits
|
||||
// for active ownership. It must not resurrect a retry
|
||||
// or replace an acknowledged cancellation with success.
|
||||
retry_request_for_queue = None;
|
||||
completed_heals_clone
|
||||
.lock()
|
||||
.await
|
||||
.get(&task_id)
|
||||
.is_some_and(|completed| completed.status == HealTaskStatus::Cancelled)
|
||||
};
|
||||
if cancelled_completion {
|
||||
completed_status = HealTaskStatus::Cancelled;
|
||||
completed_status_entry.status = HealTaskStatus::Cancelled;
|
||||
}
|
||||
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
|
||||
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
|
||||
// Keep retry ownership continuous: status snapshots acquire
|
||||
// these locks in the same active -> retrying order.
|
||||
let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
|
||||
@@ -323,16 +295,6 @@ impl HealManager {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if owns_completion || cancelled_completion {
|
||||
publish_completed_heal(
|
||||
&completed_heals_clone,
|
||||
&task_aliases_clone,
|
||||
&task_id,
|
||||
completed_status_entry,
|
||||
terminal_completion,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let completed_task = active_heals_guard.remove(&task_id);
|
||||
if let Some(completed_task) = completed_task.as_ref() {
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
@@ -342,10 +304,33 @@ impl HealManager {
|
||||
drop(retrying_heals_guard.take());
|
||||
drop(active_heals_guard);
|
||||
|
||||
#[cfg(test)]
|
||||
tests::pause_completed_retention_handoff(&task_id).await;
|
||||
|
||||
if completed_task.is_some() {
|
||||
if let Some(completed_task) = completed_task {
|
||||
let completed_status = if let Some(status) = retry_request_for_status {
|
||||
status
|
||||
} else {
|
||||
completed_task.get_status().await
|
||||
};
|
||||
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
|
||||
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
|
||||
let completed_progress = completed_task.get_progress().await;
|
||||
// Single snapshot of the retained window: the task is
|
||||
// finished and already off the active map, so there is
|
||||
// no concurrent writer to race with.
|
||||
let seqed_items = completed_task.get_seqed_result_items().await;
|
||||
let (next_seq, min_seq) = completed_task.result_seq_cursors();
|
||||
let completed_status_entry = CompletedHealStatus {
|
||||
heal_type: completed_task.heal_type.clone(),
|
||||
status: completed_status.clone(),
|
||||
result_items_truncated: completed_task.result_items_truncated(),
|
||||
completed_at: SystemTime::now(),
|
||||
seqed_items,
|
||||
next_seq,
|
||||
min_seq,
|
||||
};
|
||||
let mut completed_heals_guard = completed_heals_clone.lock().await;
|
||||
prune_completed_heal_statuses(&mut completed_heals_guard);
|
||||
completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry));
|
||||
drop(completed_heals_guard);
|
||||
// update statistics
|
||||
let mut stats = statistics_clone.write().await;
|
||||
match completed_status {
|
||||
@@ -367,6 +352,10 @@ impl HealManager {
|
||||
} else {
|
||||
release_mrf_repair_notice_targets(notice_targets);
|
||||
}
|
||||
task_aliases_clone
|
||||
.lock()
|
||||
.await
|
||||
.retain(|alias_id, alias| alias_id != &task_id && alias.task_id != task_id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -729,42 +718,17 @@ pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
|
||||
}
|
||||
|
||||
pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>) {
|
||||
prune_completed_heal_statuses_at(completed_heals, SystemTime::now());
|
||||
}
|
||||
let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else {
|
||||
return;
|
||||
};
|
||||
|
||||
pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>, now: SystemTime) {
|
||||
completed_heals.retain(|_, completed| {
|
||||
now.duration_since(completed.completed_at)
|
||||
.map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
|
||||
completed
|
||||
.completed_at
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION)
|
||||
.unwrap_or(false)
|
||||
});
|
||||
let entry_bytes = |key: &String, value: &Arc<CompletedHealStatus>| {
|
||||
key.capacity()
|
||||
.saturating_add(size_of::<(String, Arc<CompletedHealStatus>)>())
|
||||
.saturating_add(value.retained_bytes())
|
||||
};
|
||||
let mut bytes = completed_heals
|
||||
.iter()
|
||||
.fold(0usize, |total, (key, value)| total.saturating_add(entry_bytes(key, value)));
|
||||
while completed_heals.len() > MAX_COMPLETED_HEAL_TOKENS || bytes > MAX_COMPLETED_HEAL_BYTES {
|
||||
let Some(oldest) = completed_heals
|
||||
.iter()
|
||||
.min_by(|(left_id, left), (right_id, right)| {
|
||||
left.completed_at.cmp(&right.completed_at).then_with(|| left_id.cmp(right_id))
|
||||
})
|
||||
.map(|(_, value)| Arc::clone(value))
|
||||
else {
|
||||
break;
|
||||
};
|
||||
completed_heals.retain(|key, value| {
|
||||
if Arc::ptr_eq(value, &oldest) {
|
||||
bytes = bytes.saturating_sub(entry_bytes(key, value));
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn can_schedule_request(
|
||||
|
||||
@@ -101,326 +101,6 @@ async fn process_manager_queue_once(manager: &HealManager) {
|
||||
|
||||
struct MockStorage;
|
||||
|
||||
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
heal_type: HealType::Cluster,
|
||||
status: HealTaskStatus::Completed,
|
||||
progress: Some(HealProgress {
|
||||
objects_scanned: 9,
|
||||
objects_healed: 8,
|
||||
objects_failed: 1,
|
||||
..Default::default()
|
||||
}),
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
result_items_truncated: false,
|
||||
completed_at,
|
||||
seqed_items: vec![(3, HealResultItem::default()), (4, HealResultItem::default())],
|
||||
next_seq: 5,
|
||||
min_seq: 3,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_cursor_boundaries_preserve_progress() {
|
||||
let completed = completed_retention_fixture(SystemTime::now());
|
||||
for (cursor, count, lagged) in [
|
||||
(0, 2, true),
|
||||
(1, 2, true),
|
||||
(2, 2, false),
|
||||
(3, 1, false),
|
||||
(4, 0, false),
|
||||
(5, 0, false),
|
||||
(u64::MAX, 0, false),
|
||||
] {
|
||||
let report = completed_task_report(&completed, Some(cursor));
|
||||
assert_eq!(report.result_items.len(), count, "cursor={cursor}");
|
||||
assert_eq!(report.result_items_truncated, lagged, "cursor={cursor}");
|
||||
assert_eq!(report.progress, completed.progress);
|
||||
assert_eq!((report.next_seq, report.min_seq), (5, 3));
|
||||
}
|
||||
assert_eq!(completed_task_report(&completed, None).result_items.len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() {
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::bucket("bucket".to_string());
|
||||
manager.insert_task_alias("alias", &request.id).await;
|
||||
let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
|
||||
lock_displaced_terminals(&manager.displaced_terminals).remove(&request.id);
|
||||
remove_displaced_task_aliases(&manager.task_aliases, &manager.displaced_terminals, &request.id, &terminal).await;
|
||||
for token in [&request.id, &"alias".to_string()] {
|
||||
assert!(matches!(manager.get_task_report(token).await, Err(Error::TaskNotFound { .. })));
|
||||
}
|
||||
assert!(manager.task_aliases.lock().await.is_empty());
|
||||
assert!(lock_displaced_terminals(&manager.displaced_terminals).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_count_ttl_and_alias_eviction_are_bounded() {
|
||||
let now = SystemTime::now();
|
||||
let mut entries = HashMap::new();
|
||||
let oldest = Arc::new(completed_retention_fixture(now - KEEP_HEAL_TASK_STATUS_DURATION));
|
||||
entries.insert("oldest".to_string(), Arc::clone(&oldest));
|
||||
entries.insert("oldest-alias".to_string(), Arc::clone(&oldest));
|
||||
for index in 2..MAX_COMPLETED_HEAL_TOKENS {
|
||||
entries.insert(format!("task-{index}"), Arc::new(completed_retention_fixture(now)));
|
||||
}
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS);
|
||||
entries.insert("cap-plus-one".to_string(), Arc::new(completed_retention_fixture(now)));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1);
|
||||
assert!(!entries.contains_key("oldest"));
|
||||
assert!(!entries.contains_key("oldest-alias"));
|
||||
entries.clear();
|
||||
entries.insert("ttl-boundary".to_string(), oldest);
|
||||
entries.insert(
|
||||
"expired".to_string(),
|
||||
Arc::new(completed_retention_fixture(
|
||||
now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1),
|
||||
)),
|
||||
);
|
||||
entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1))));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), 1);
|
||||
assert!(entries.contains_key("ttl-boundary"));
|
||||
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1));
|
||||
assert!(entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_total_byte_cap_and_cap_plus_one() {
|
||||
let now = SystemTime::now();
|
||||
let key = "large".to_string();
|
||||
let mut entry = completed_retention_fixture(now);
|
||||
let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
|
||||
entry.retained_bytes.take();
|
||||
entry.status = HealTaskStatus::Failed {
|
||||
error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
|
||||
};
|
||||
assert_eq!(
|
||||
entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
|
||||
MAX_COMPLETED_HEAL_BYTES
|
||||
);
|
||||
let mut entries = HashMap::from([(key, Arc::new(entry))]);
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert_eq!(entries.len(), 1, "exact byte cap remains retained");
|
||||
let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
|
||||
over.retained_bytes.take();
|
||||
if let HealTaskStatus::Failed { error } = &mut over.status {
|
||||
*error = "x".repeat(error.len() + 1);
|
||||
}
|
||||
entries.insert("large".to_string(), Arc::new(over));
|
||||
prune_completed_heal_statuses_at(&mut entries, now);
|
||||
assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_large_window_keeps_cursors_and_progress() {
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), Arc::new(MockStorage));
|
||||
let mut snapshot = completed_retention_fixture(SystemTime::now());
|
||||
snapshot.seqed_items[0].1.detail = "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES);
|
||||
snapshot.bound_result_window();
|
||||
assert_eq!(snapshot.seqed_items.len(), 1);
|
||||
assert_eq!((snapshot.min_seq, snapshot.next_seq), (4, 5));
|
||||
assert!(snapshot.result_items_truncated);
|
||||
assert!(snapshot.retained_bytes() < MAX_COMPLETED_HEAL_RESULT_BYTES);
|
||||
let report = completed_task_report(&snapshot, Some(0));
|
||||
assert_eq!(report.progress.expect("progress retained").objects_scanned, 9);
|
||||
assert!(report.result_items_truncated);
|
||||
let active_max = task.get_result_items_since(Some(u64::MAX)).await;
|
||||
assert!(active_max.items.is_empty());
|
||||
assert!(!active_max.lagged);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_retention_result_byte_cap_and_cap_plus_one() {
|
||||
for extra in [0, 1] {
|
||||
let mut snapshot = completed_retention_fixture(SystemTime::now());
|
||||
snapshot.seqed_items = vec![(
|
||||
4,
|
||||
HealResultItem {
|
||||
detail: "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES - size_of::<(u64, HealResultItem)>() + extra),
|
||||
..Default::default()
|
||||
},
|
||||
)];
|
||||
snapshot.min_seq = 4;
|
||||
snapshot.bound_result_window();
|
||||
assert_eq!(snapshot.seqed_items.len(), 1 - extra);
|
||||
assert_eq!(snapshot.result_items_truncated, extra == 1);
|
||||
assert_eq!(snapshot.min_seq, if extra == 0 { 4 } else { 5 });
|
||||
assert_eq!(snapshot.next_seq, 5);
|
||||
assert_eq!(snapshot.progress.as_ref().expect("progress retained").objects_scanned, 9);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CompletedRetentionHook {
|
||||
started: Notify,
|
||||
execute: Notify,
|
||||
handoff: Notify,
|
||||
finish: Notify,
|
||||
pause_before_publish: bool,
|
||||
before_publish: Notify,
|
||||
publish: Notify,
|
||||
prepared_status: Mutex<Option<HealTaskStatus>>,
|
||||
}
|
||||
|
||||
static COMPLETED_RETENTION_HOOKS: LazyLock<Mutex<HashMap<String, Arc<CompletedRetentionHook>>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
pub(super) async fn pause_completed_retention_handoff(task_id: &str) {
|
||||
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
|
||||
if let Some(hook) = hook {
|
||||
hook.handoff.notify_one();
|
||||
hook.finish.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn pause_completed_retention_before_publish(task_id: &str, status: &HealTaskStatus) {
|
||||
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
|
||||
if let Some(hook) = hook.filter(|hook| hook.pause_before_publish) {
|
||||
*hook.prepared_status.lock().await = Some(status.clone());
|
||||
hook.before_publish.notify_one();
|
||||
hook.publish.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
|
||||
let bucket = "completed-retention-retry-cancel";
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
let task_id = request.id.clone();
|
||||
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
let alias = duplicate.id.clone();
|
||||
let hook = Arc::new(CompletedRetentionHook {
|
||||
pause_before_publish: true,
|
||||
..Default::default()
|
||||
});
|
||||
{
|
||||
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
|
||||
hooks.insert(bucket.to_string(), Arc::clone(&hook));
|
||||
hooks.insert(task_id.clone(), Arc::clone(&hook));
|
||||
}
|
||||
manager.submit_heal_request(request).await.expect("admit original");
|
||||
manager.submit_heal_request(duplicate).await.expect("admit alias");
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
|
||||
.await
|
||||
.expect("scheduler starts");
|
||||
let task = manager.active_heals.lock().await.get(&task_id).cloned().expect("active task");
|
||||
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
|
||||
hook.execute.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.before_publish.notified())
|
||||
.await
|
||||
.expect("retry snapshot prepared");
|
||||
manager.cancel_task(&alias).await.expect("cancel wins active ownership");
|
||||
assert!(matches!(*hook.prepared_status.lock().await, Some(HealTaskStatus::Retrying { .. })));
|
||||
hook.publish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
|
||||
.await
|
||||
.expect("scheduler finishes handoff");
|
||||
for token in [&task_id, &alias] {
|
||||
let report = manager.get_task_report(token).await.expect("cancelled token retained");
|
||||
assert_eq!(report.status, HealTaskStatus::Cancelled);
|
||||
assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
|
||||
}
|
||||
assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
|
||||
assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id));
|
||||
hook.finish.notify_one();
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.retain(|key, _| key != bucket && key != &task_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_handoff() {
|
||||
for outcome in ["success", "failed", "cancelled"] {
|
||||
let bucket = format!("completed-retention-{outcome}");
|
||||
let hook = Arc::new(CompletedRetentionHook::default());
|
||||
let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None));
|
||||
let request = HealRequest::object(bucket.clone(), "object".to_string(), None);
|
||||
let task_id = request.id.clone();
|
||||
let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None);
|
||||
let alias = duplicate.id.clone();
|
||||
{
|
||||
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
|
||||
hooks.insert(bucket.clone(), Arc::clone(&hook));
|
||||
hooks.insert(task_id.clone(), Arc::clone(&hook));
|
||||
}
|
||||
manager.submit_heal_request(request).await.expect("admit original");
|
||||
manager.submit_heal_request(duplicate).await.expect("admit alias");
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
|
||||
.await
|
||||
.expect("scheduler reaches storage");
|
||||
let task = manager
|
||||
.active_heals
|
||||
.lock()
|
||||
.await
|
||||
.get(&task_id)
|
||||
.cloned()
|
||||
.expect("task is active");
|
||||
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
|
||||
let before = manager.get_task_report(&alias).await.expect("alias resolves active progress");
|
||||
assert_eq!(before.progress.as_ref().expect("active progress").objects_scanned, 1);
|
||||
let poll_manager = Arc::clone(&manager);
|
||||
let poll_alias = alias.clone();
|
||||
let stop = CancellationToken::new();
|
||||
let poll_stop = stop.clone();
|
||||
let polling = tokio::spawn(async move {
|
||||
while !poll_stop.is_cancelled() {
|
||||
let report = poll_manager
|
||||
.get_task_report(&poll_alias)
|
||||
.await
|
||||
.expect("handoff must never return NotFound");
|
||||
assert!(report.progress.expect("progress never disappears").objects_scanned >= 1);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
});
|
||||
if outcome == "cancelled" {
|
||||
manager.cancel_task(&alias).await.expect("cancel active task by alias");
|
||||
} else {
|
||||
hook.execute.notify_one();
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
|
||||
.await
|
||||
.expect("scheduler archives terminal");
|
||||
assert!(!manager.active_heals.lock().await.contains_key(&task_id));
|
||||
let expected = task.get_progress().await;
|
||||
for token in [&task_id, &alias] {
|
||||
assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
|
||||
let report = manager
|
||||
.get_task_report_for_path_since(&format!("{bucket}/object"), token, Some(u64::MAX))
|
||||
.await
|
||||
.expect("terminal token remains queryable at handoff");
|
||||
assert_eq!(report.progress.as_ref(), Some(&expected));
|
||||
assert!(report.result_items.is_empty());
|
||||
match outcome {
|
||||
"success" => assert_eq!(report.status, HealTaskStatus::Completed),
|
||||
"failed" => assert!(matches!(report.status, HealTaskStatus::Failed { .. })),
|
||||
_ => assert_eq!(report.status, HealTaskStatus::Cancelled),
|
||||
}
|
||||
}
|
||||
let retained = manager.completed_heals.lock().await;
|
||||
assert!(Arc::ptr_eq(&retained[&task_id], &retained[&alias]));
|
||||
drop(retained);
|
||||
stop.cancel();
|
||||
polling.await.expect("concurrent polling succeeds");
|
||||
// Archived progress must not alias a mutable live progress object.
|
||||
task.progress.write().await.objects_scanned = 999;
|
||||
assert_eq!(manager.get_task_report(&alias).await.expect("frozen report").progress, Some(expected));
|
||||
hook.finish.notify_one();
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.retain(|key, _| key != &bucket && key != &task_id);
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl HealStorageAPI for MockStorage {
|
||||
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> {
|
||||
@@ -443,12 +123,6 @@ impl HealStorageAPI for MockStorage {
|
||||
}
|
||||
|
||||
async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> {
|
||||
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(bucket).cloned();
|
||||
if let Some(hook) = hook {
|
||||
hook.started.notify_one();
|
||||
hook.execute.notified().await;
|
||||
return Ok(true);
|
||||
}
|
||||
Ok(bucket == "retry-transition")
|
||||
}
|
||||
|
||||
@@ -459,18 +133,13 @@ impl HealStorageAPI for MockStorage {
|
||||
_version_id: Option<&str>,
|
||||
_opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
if bucket == "completed-retention-failed" {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "retention fixture failure".to_string(),
|
||||
});
|
||||
}
|
||||
if let Some(hook) = manager_recovery_test_hook() {
|
||||
*hook
|
||||
.heal_object_calls
|
||||
.lock()
|
||||
.expect("manager recovery object call lock should not poison") += 1;
|
||||
}
|
||||
if matches!(bucket, "retry-transition" | "completed-retention-retry-cancel") {
|
||||
if bucket == "retry-transition" {
|
||||
return Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||
@@ -1476,13 +1145,7 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
|
||||
.expect("duplicate token should cancel merged active task");
|
||||
|
||||
assert!(manager.active_heals.lock().await.get(&active_task_id).is_none());
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_task_status(&active_task_id)
|
||||
.await
|
||||
.expect("cancelled task remains queryable"),
|
||||
HealTaskStatus::Cancelled
|
||||
);
|
||||
assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1975,8 +1638,6 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
|
||||
manager.completed_heals.lock().await.insert(
|
||||
task_id,
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type,
|
||||
status: HealTaskStatus::Retrying {
|
||||
error: "Lock acquisition timeout".to_string(),
|
||||
@@ -2392,7 +2053,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() {
|
||||
"the overlapping admin task must be cancelled (removed from the active table) before the new one starts"
|
||||
);
|
||||
assert!(
|
||||
matches!(manager.get_task_status(&old_id).await, Ok(HealTaskStatus::Cancelled)),
|
||||
matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })),
|
||||
"a cancelled task must no longer resolve as an active heal"
|
||||
);
|
||||
}
|
||||
@@ -2699,8 +2360,6 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
|
||||
manager.completed_heals.lock().await.insert(
|
||||
task_id.clone(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type.clone(),
|
||||
status: HealTaskStatus::Retrying {
|
||||
error: "transient disk failure".to_string(),
|
||||
@@ -2736,8 +2395,6 @@ async fn test_get_task_status_reads_recent_completed_status() {
|
||||
manager.completed_heals.lock().await.insert(
|
||||
"completed-token".to_string(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: HealType::Bucket {
|
||||
bucket: "bucket".to_string(),
|
||||
},
|
||||
@@ -2767,8 +2424,6 @@ async fn test_get_task_report_for_path_reads_completed_items() {
|
||||
manager.completed_heals.lock().await.insert(
|
||||
"completed-token".to_string(),
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
|
||||
@@ -999,7 +999,7 @@ impl HealTask {
|
||||
let items = match since {
|
||||
None => result_items.iter().map(|(_, item)| item.clone()).collect::<Vec<_>>(),
|
||||
Some(cursor) => {
|
||||
if cursor.saturating_add(1) < min_seq {
|
||||
if cursor + 1 < min_seq {
|
||||
lagged = true;
|
||||
}
|
||||
result_items
|
||||
|
||||
@@ -182,9 +182,6 @@ pub struct BackgroundHealStatus {
|
||||
pub heal_active_tasks: u64,
|
||||
#[serde(default)]
|
||||
pub cluster_status_complete: bool,
|
||||
/// Missing on older servers; absent coverage or counts mean unknown.
|
||||
#[serde(default)]
|
||||
pub coverage: Option<BackgroundHealCoverage>,
|
||||
#[serde(default)]
|
||||
pub progress: Option<serde_json::Value>,
|
||||
/// Remaining wire fields (flattened `BackgroundHealInfo` plus the
|
||||
@@ -193,22 +190,6 @@ pub struct BackgroundHealStatus {
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Node coverage of a background heal status snapshot. Counters describe only
|
||||
/// nodes with usable snapshots; unknown peers may still be running heal work.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackgroundHealCoverage {
|
||||
#[serde(default)]
|
||||
pub expected: Option<usize>,
|
||||
#[serde(default)]
|
||||
pub responded: Option<usize>,
|
||||
#[serde(default)]
|
||||
pub unknown: Option<usize>,
|
||||
/// Stable reason codes; unknown future codes are preserved verbatim.
|
||||
#[serde(default)]
|
||||
pub reasons: Vec<String>,
|
||||
}
|
||||
|
||||
/// `GET /v3/scanner/status` response, typed at the fields operators branch
|
||||
/// on; everything else passes through verbatim.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
@@ -649,39 +630,9 @@ mod tests {
|
||||
assert_eq!(status.state, "active");
|
||||
assert_eq!(status.heal_queue_length, 3);
|
||||
assert!(status.cluster_status_complete);
|
||||
assert!(status.coverage.is_none(), "legacy payloads have unknown coverage");
|
||||
assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_heal_status_missing_coverage_fields_remain_unknown() {
|
||||
for raw in [json!({"state": "degraded"}), json!({"state": "degraded", "coverage": {}})] {
|
||||
let status: BackgroundHealStatus = serde_json::from_value(raw).expect("partial legacy payload decodes");
|
||||
assert!(!status.cluster_status_complete);
|
||||
if let Some(coverage) = status.coverage {
|
||||
assert_eq!(coverage.expected, None);
|
||||
assert_eq!(coverage.responded, None);
|
||||
assert_eq!(coverage.unknown, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn background_heal_status_preserves_future_fields_and_reasons() {
|
||||
let raw = json!({
|
||||
"state": "degraded", "clusterStatusComplete": false,
|
||||
"coverage": {"expected": 3, "responded": 1, "unknown": 2, "reasons": ["future_reason"], "futureCoverage": true},
|
||||
"futureStatus": {"value": 7}
|
||||
});
|
||||
let status: BackgroundHealStatus = serde_json::from_value(raw).expect("future additive fields decode");
|
||||
assert_eq!(status.extra["futureStatus"]["value"], 7);
|
||||
let coverage = status.coverage.expect("coverage supplied");
|
||||
assert_eq!(coverage.expected, Some(3));
|
||||
assert_eq!(coverage.responded, Some(1));
|
||||
assert_eq!(coverage.unknown, Some(2));
|
||||
assert_eq!(coverage.reasons, ["future_reason"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_status_defaults_freshness_to_unknown() {
|
||||
let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}});
|
||||
@@ -770,7 +721,6 @@ mod tests {
|
||||
|
||||
let status = client.background_heal_status().await.expect("status decodes");
|
||||
assert_eq!(status.state, "idle");
|
||||
assert!(status.coverage.is_none(), "older HTTP responses retain unknown coverage");
|
||||
let request = server.recorded();
|
||||
// The server registers this route POST-only; a GET here answers 405.
|
||||
assert_eq!(request.method, "POST");
|
||||
@@ -778,28 +728,6 @@ mod tests {
|
||||
assert_eq!(request.query, "");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_heal_status_decodes_partial_coverage_over_http() {
|
||||
let body = r#"{"state":"degraded","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":false,"coverage":{"expected":3,"responded":1,"unknown":2,"reasons":["notification_system_unavailable"]},"futureStatus":true}"#;
|
||||
let server = TestServer::spawn(body, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").expect("client builds");
|
||||
let status = client
|
||||
.background_heal_status()
|
||||
.await
|
||||
.expect("partial status is a successful response");
|
||||
assert_eq!(status.state, "degraded");
|
||||
assert!(!status.cluster_status_complete);
|
||||
assert_eq!(status.extra["futureStatus"], true);
|
||||
let coverage = status.coverage.expect("partial coverage supplied");
|
||||
assert_eq!(coverage.expected, Some(3));
|
||||
assert_eq!(coverage.responded, Some(1));
|
||||
assert_eq!(coverage.unknown, Some(2));
|
||||
assert_eq!(coverage.reasons, ["notification_system_unavailable"]);
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.query, "", "reading status must not send heal control parameters");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_error_status_maps_to_a_typed_error_with_body() {
|
||||
let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await;
|
||||
|
||||
@@ -1283,54 +1283,6 @@ pub struct ScannerDirtyUsageSnapshotResponse {
|
||||
#[prost(bytes = "bytes", tag = "7")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// Receiver-only protocol. Producers must retain whole-cycle ACK until they
|
||||
/// have a durable per-bucket publication proof.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerScopedDirtyUsageEntry {
|
||||
#[prost(string, tag = "1")]
|
||||
pub bucket: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "2")]
|
||||
pub bucket_incarnation: ::prost::bytes::Bytes,
|
||||
#[prost(uint64, tag = "3")]
|
||||
pub generation: u64,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct ScannerScopedDirtyUsageAckRequest {
|
||||
#[prost(bytes = "bytes", tag = "1")]
|
||||
pub challenge: ::prost::bytes::Bytes,
|
||||
#[prost(uint32, tag = "2")]
|
||||
pub protocol_version: u32,
|
||||
#[prost(string, tag = "3")]
|
||||
pub owner_id: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "4")]
|
||||
pub instance_id: ::prost::alloc::string::String,
|
||||
/// Only scope 1 (a complete bucket) is supported; zero is invalid.
|
||||
#[prost(uint32, tag = "5")]
|
||||
pub scope: u32,
|
||||
#[prost(bool, tag = "6")]
|
||||
pub probe_only: bool,
|
||||
#[prost(message, repeated, tag = "7")]
|
||||
pub entries: ::prost::alloc::vec::Vec<ScannerScopedDirtyUsageEntry>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerScopedDirtyUsageAckResponse {
|
||||
#[prost(uint32, tag = "1")]
|
||||
pub protocol_version: u32,
|
||||
#[prost(string, tag = "2")]
|
||||
pub owner_id: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub instance_id: ::prost::alloc::string::String,
|
||||
#[prost(bool, tag = "4")]
|
||||
pub supported: bool,
|
||||
#[prost(uint32, tag = "5")]
|
||||
pub max_entries: u32,
|
||||
#[prost(uint32, tag = "6")]
|
||||
pub max_request_bytes: u32,
|
||||
#[prost(uint64, tag = "7")]
|
||||
pub cleared: u64,
|
||||
#[prost(bytes = "bytes", tag = "8")]
|
||||
pub response_proof: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// A short-lived storage-owned read admission used only around a final
|
||||
/// scanner metadata publication. It is intentionally separate from the
|
||||
/// ScannerActivity observation wire so v6/v7 rolling compatibility remains
|
||||
@@ -6330,244 +6282,6 @@ pub mod node_service_server {
|
||||
}
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod scanner_control_service_client {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
use tonic::codegen::http::Uri;
|
||||
use tonic::codegen::*;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScannerControlServiceClient<T> {
|
||||
inner: tonic::client::Grpc<T>,
|
||||
}
|
||||
impl ScannerControlServiceClient<tonic::transport::Channel> {
|
||||
/// Attempt to create a new client by connecting to a given endpoint.
|
||||
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
|
||||
where
|
||||
D: TryInto<tonic::transport::Endpoint>,
|
||||
D::Error: Into<StdError>,
|
||||
{
|
||||
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
|
||||
Ok(Self::new(conn))
|
||||
}
|
||||
}
|
||||
impl<T> ScannerControlServiceClient<T>
|
||||
where
|
||||
T: tonic::client::GrpcService<tonic::body::Body>,
|
||||
T::Error: Into<StdError>,
|
||||
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
|
||||
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
|
||||
{
|
||||
pub fn new(inner: T) -> Self {
|
||||
let inner = tonic::client::Grpc::new(inner);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_origin(inner: T, origin: Uri) -> Self {
|
||||
let inner = tonic::client::Grpc::with_origin(inner, origin);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_interceptor<F>(inner: T, interceptor: F) -> ScannerControlServiceClient<InterceptedService<T, F>>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
T::ResponseBody: Default,
|
||||
T: tonic::codegen::Service<
|
||||
http::Request<tonic::body::Body>,
|
||||
Response = http::Response<<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody>,
|
||||
>,
|
||||
<T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
|
||||
Into<StdError> + std::marker::Send + std::marker::Sync,
|
||||
{
|
||||
ScannerControlServiceClient::new(InterceptedService::new(inner, interceptor))
|
||||
}
|
||||
/// Compress requests with the given encoding.
|
||||
///
|
||||
/// This requires the server to support it otherwise it might respond with an
|
||||
/// error.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.send_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Enable decompressing responses.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.accept_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_decoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_encoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
pub async fn scanner_scoped_dirty_usage_ack(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::ScannerScopedDirtyUsageAckRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerScopedDirtyUsageAckResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.ScannerControlService/ScannerScopedDirtyUsageAck");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.ScannerControlService", "ScannerScopedDirtyUsageAck"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated server implementations.
|
||||
pub mod scanner_control_service_server {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
use tonic::codegen::*;
|
||||
/// Generated trait containing gRPC methods that should be implemented for use with ScannerControlServiceServer.
|
||||
#[async_trait]
|
||||
pub trait ScannerControlService: std::marker::Send + std::marker::Sync + 'static {
|
||||
async fn scanner_scoped_dirty_usage_ack(
|
||||
&self,
|
||||
request: tonic::Request<super::ScannerScopedDirtyUsageAckRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::ScannerScopedDirtyUsageAckResponse>, tonic::Status>;
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct ScannerControlServiceServer<T> {
|
||||
inner: Arc<T>,
|
||||
accept_compression_encodings: EnabledCompressionEncodings,
|
||||
send_compression_encodings: EnabledCompressionEncodings,
|
||||
max_decoding_message_size: Option<usize>,
|
||||
max_encoding_message_size: Option<usize>,
|
||||
}
|
||||
impl<T> ScannerControlServiceServer<T> {
|
||||
pub fn new(inner: T) -> Self {
|
||||
Self::from_arc(Arc::new(inner))
|
||||
}
|
||||
pub fn from_arc(inner: Arc<T>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
accept_compression_encodings: Default::default(),
|
||||
send_compression_encodings: Default::default(),
|
||||
max_decoding_message_size: None,
|
||||
max_encoding_message_size: None,
|
||||
}
|
||||
}
|
||||
pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
{
|
||||
InterceptedService::new(Self::new(inner), interceptor)
|
||||
}
|
||||
/// Enable decompressing requests with the given encoding.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.accept_compression_encodings.enable(encoding);
|
||||
self
|
||||
}
|
||||
/// Compress responses with the given encoding, if the client supports it.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.send_compression_encodings.enable(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.max_decoding_message_size = Some(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.max_encoding_message_size = Some(limit);
|
||||
self
|
||||
}
|
||||
}
|
||||
impl<T, B> tonic::codegen::Service<http::Request<B>> for ScannerControlServiceServer<T>
|
||||
where
|
||||
T: ScannerControlService,
|
||||
B: Body + std::marker::Send + 'static,
|
||||
B::Error: Into<StdError> + std::marker::Send + 'static,
|
||||
{
|
||||
type Response = http::Response<tonic::body::Body>;
|
||||
type Error = std::convert::Infallible;
|
||||
type Future = BoxFuture<Self::Response, Self::Error>;
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
fn call(&mut self, req: http::Request<B>) -> Self::Future {
|
||||
match req.uri().path() {
|
||||
"/node_service.ScannerControlService/ScannerScopedDirtyUsageAck" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ScannerScopedDirtyUsageAckSvc<T: ScannerControlService>(pub Arc<T>);
|
||||
impl<T: ScannerControlService> tonic::server::UnaryService<super::ScannerScopedDirtyUsageAckRequest>
|
||||
for ScannerScopedDirtyUsageAckSvc<T>
|
||||
{
|
||||
type Response = super::ScannerScopedDirtyUsageAckResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::ScannerScopedDirtyUsageAckRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move {
|
||||
<T as ScannerControlService>::scanner_scoped_dirty_usage_ack(&inner, request).await
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = ScannerScopedDirtyUsageAckSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
_ => Box::pin(async move {
|
||||
let mut response = http::Response::new(tonic::body::Body::default());
|
||||
let headers = response.headers_mut();
|
||||
headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
|
||||
headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
|
||||
Ok(response)
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<T> Clone for ScannerControlServiceServer<T> {
|
||||
fn clone(&self) -> Self {
|
||||
let inner = self.inner.clone();
|
||||
Self {
|
||||
inner,
|
||||
accept_compression_encodings: self.accept_compression_encodings,
|
||||
send_compression_encodings: self.send_compression_encodings,
|
||||
max_decoding_message_size: self.max_decoding_message_size,
|
||||
max_encoding_message_size: self.max_encoding_message_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated gRPC service name
|
||||
pub const SERVICE_NAME: &str = "node_service.ScannerControlService";
|
||||
impl<T> tonic::server::NamedService for ScannerControlServiceServer<T> {
|
||||
const NAME: &'static str = SERVICE_NAME;
|
||||
}
|
||||
}
|
||||
/// Generated client implementations.
|
||||
pub mod heal_control_service_client {
|
||||
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
|
||||
use tonic::codegen::http::Uri;
|
||||
|
||||
@@ -541,8 +541,6 @@ pub fn canonical_scanner_activity_v7_response_body(
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub mod scoped_dirty_usage;
|
||||
|
||||
pub fn canonical_scanner_dirty_usage_snapshot_request_body(
|
||||
request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
|
||||
@@ -903,36 +903,6 @@ message ScannerDirtyUsageSnapshotResponse {
|
||||
bytes response_proof = 7;
|
||||
}
|
||||
|
||||
// Receiver-only protocol. Producers must retain whole-cycle ACK until they
|
||||
// have a durable per-bucket publication proof.
|
||||
message ScannerScopedDirtyUsageEntry {
|
||||
string bucket = 1;
|
||||
bytes bucket_incarnation = 2;
|
||||
uint64 generation = 3;
|
||||
}
|
||||
|
||||
message ScannerScopedDirtyUsageAckRequest {
|
||||
bytes challenge = 1;
|
||||
uint32 protocol_version = 2;
|
||||
string owner_id = 3;
|
||||
string instance_id = 4;
|
||||
// Only scope 1 (a complete bucket) is supported; zero is invalid.
|
||||
uint32 scope = 5;
|
||||
bool probe_only = 6;
|
||||
repeated ScannerScopedDirtyUsageEntry entries = 7;
|
||||
}
|
||||
|
||||
message ScannerScopedDirtyUsageAckResponse {
|
||||
uint32 protocol_version = 1;
|
||||
string owner_id = 2;
|
||||
string instance_id = 3;
|
||||
bool supported = 4;
|
||||
uint32 max_entries = 5;
|
||||
uint32 max_request_bytes = 6;
|
||||
uint64 cleared = 7;
|
||||
bytes response_proof = 8;
|
||||
}
|
||||
|
||||
// A short-lived storage-owned read admission used only around a final
|
||||
// scanner metadata publication. It is intentionally separate from the
|
||||
// ScannerActivity observation wire so v6/v7 rolling compatibility remains
|
||||
@@ -1275,10 +1245,6 @@ service NodeService {
|
||||
rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only
|
||||
}
|
||||
|
||||
service ScannerControlService {
|
||||
rpc ScannerScopedDirtyUsageAck(ScannerScopedDirtyUsageAckRequest) returns (ScannerScopedDirtyUsageAckResponse) {}; // auth-policy: body-bound
|
||||
}
|
||||
|
||||
service HealControlService {
|
||||
rpc HealControl(HealControlRequest) returns (HealControlResponse) {};
|
||||
}
|
||||
|
||||
@@ -1,213 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Bounded, authenticated receiver contract for per-bucket dirty acknowledgements.
|
||||
|
||||
use crate::CanonicalBodyBuilder;
|
||||
use crate::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
|
||||
use prost::Message;
|
||||
|
||||
pub const SCOPED_DIRTY_USAGE_PROTOCOL_VERSION: u32 = 1;
|
||||
pub const SCOPED_DIRTY_USAGE_BUCKET_SCOPE: u32 = 1;
|
||||
pub const SCOPED_DIRTY_USAGE_MAX_ENTRIES: u32 = 32;
|
||||
pub const SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES: u32 = 8192;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ScopedDirtyUsageRequestError {
|
||||
UnsupportedProtocol,
|
||||
UnsupportedScope,
|
||||
InvalidIdentity,
|
||||
InvalidGeneration,
|
||||
InvalidEntries,
|
||||
TooLarge,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ScopedDirtyUsageRequestError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::UnsupportedProtocol => "unsupported scoped dirty usage protocol",
|
||||
Self::UnsupportedScope => "unsupported scoped dirty usage scope",
|
||||
Self::InvalidIdentity => "invalid scoped dirty usage identity",
|
||||
Self::InvalidGeneration => "invalid scoped dirty usage generation",
|
||||
Self::InvalidEntries => "scoped dirty usage entries must be nonempty and strictly ordered",
|
||||
Self::TooLarge => "scoped dirty usage request exceeds its budget",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ScopedDirtyUsageRequestError {}
|
||||
|
||||
pub fn validate_scoped_dirty_usage_request(
|
||||
request: &ScannerScopedDirtyUsageAckRequest,
|
||||
) -> Result<(), ScopedDirtyUsageRequestError> {
|
||||
use ScopedDirtyUsageRequestError as E;
|
||||
if request.entries.len() > SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize
|
||||
|| request.encoded_len() > SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize
|
||||
{
|
||||
return Err(E::TooLarge);
|
||||
}
|
||||
if request.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION {
|
||||
return Err(E::UnsupportedProtocol);
|
||||
}
|
||||
if request.scope != SCOPED_DIRTY_USAGE_BUCKET_SCOPE {
|
||||
return Err(E::UnsupportedScope);
|
||||
}
|
||||
if request.challenge.len() != 16 || request.owner_id.len() != 36 || request.instance_id.len() != 32 {
|
||||
return Err(E::InvalidIdentity);
|
||||
}
|
||||
if request.entries.is_empty() || request.entries.windows(2).any(|pair| pair[0].bucket >= pair[1].bucket) {
|
||||
return Err(E::InvalidEntries);
|
||||
}
|
||||
for entry in &request.entries {
|
||||
if entry.bucket.is_empty()
|
||||
|| entry.bucket.len() > 63
|
||||
|| entry.bucket_incarnation.len() != 16
|
||||
|| entry.bucket_incarnation.iter().all(|byte| *byte == 0)
|
||||
{
|
||||
return Err(E::InvalidIdentity);
|
||||
}
|
||||
if entry.generation == 0 || entry.generation == u64::MAX {
|
||||
return Err(E::InvalidGeneration);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn canonical_scoped_dirty_usage_request(
|
||||
request: &ScannerScopedDirtyUsageAckRequest,
|
||||
) -> Result<Vec<u8>, ScopedDirtyUsageRequestError> {
|
||||
validate_scoped_dirty_usage_request(request)?;
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-request-v1\0");
|
||||
let encode = |_: std::num::TryFromIntError| ScopedDirtyUsageRequestError::TooLarge;
|
||||
body.push_bytes(request.challenge.as_ref()).map_err(encode)?;
|
||||
body.push_u32(request.protocol_version);
|
||||
body.push_str(&request.owner_id).map_err(encode)?;
|
||||
body.push_str(&request.instance_id).map_err(encode)?;
|
||||
body.push_u32(request.scope);
|
||||
body.push_bool(request.probe_only);
|
||||
body.push_count(request.entries.len()).map_err(encode)?;
|
||||
for entry in &request.entries {
|
||||
body.push_str(&entry.bucket).map_err(encode)?;
|
||||
body.push_bytes(entry.bucket_incarnation.as_ref()).map_err(encode)?;
|
||||
body.push_u64(entry.generation);
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_scoped_dirty_usage_response(
|
||||
request_body: &[u8],
|
||||
response: &ScannerScopedDirtyUsageAckResponse,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-response-v1\0");
|
||||
body.push_bytes(request_body)?;
|
||||
body.push_u32(response.protocol_version);
|
||||
body.push_str(&response.owner_id)?;
|
||||
body.push_str(&response.instance_id)?;
|
||||
body.push_bool(response.supported);
|
||||
body.push_u32(response.max_entries);
|
||||
body.push_u32(response.max_request_bytes);
|
||||
body.push_u64(response.cleared);
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
|
||||
|
||||
fn request() -> ScannerScopedDirtyUsageAckRequest {
|
||||
ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: vec![1; 16].into(),
|
||||
protocol_version: 1,
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".into(),
|
||||
instance_id: "a".repeat(32),
|
||||
scope: 1,
|
||||
probe_only: false,
|
||||
entries: vec![ScannerScopedDirtyUsageEntry {
|
||||
bucket: "photos".into(),
|
||||
bucket_incarnation: vec![2; 16].into(),
|
||||
generation: 8,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_binds_every_request_field() {
|
||||
let base = request();
|
||||
let baseline = canonical_scoped_dirty_usage_request(&base).expect("valid request");
|
||||
for field in 0..9 {
|
||||
let mut changed = base.clone();
|
||||
match field {
|
||||
0 => changed.challenge = vec![3; 16].into(),
|
||||
1 => changed.protocol_version += 1,
|
||||
2 => changed.owner_id = "22222222-2222-2222-2222-222222222222".into(),
|
||||
3 => changed.instance_id = "b".repeat(32),
|
||||
4 => changed.scope += 1,
|
||||
5 => changed.probe_only = true,
|
||||
6 => changed.entries[0].bucket = "videos".into(),
|
||||
7 => changed.entries[0].bucket_incarnation = vec![3; 16].into(),
|
||||
_ => changed.entries[0].generation += 1,
|
||||
}
|
||||
assert!(canonical_scoped_dirty_usage_request(&changed).map_or(true, |body| body != baseline));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_binds_capability_and_ack_to_exact_request() {
|
||||
let request = canonical_scoped_dirty_usage_request(&request()).expect("valid request");
|
||||
let response = ScannerScopedDirtyUsageAckResponse {
|
||||
protocol_version: 1,
|
||||
owner_id: "owner".into(),
|
||||
instance_id: "process".into(),
|
||||
supported: true,
|
||||
max_entries: 32,
|
||||
max_request_bytes: 8192,
|
||||
cleared: 1,
|
||||
response_proof: vec![1; 32].into(),
|
||||
};
|
||||
let baseline = canonical_scoped_dirty_usage_response(&request, &response).expect("valid response");
|
||||
for field in 0..7 {
|
||||
let mut changed = response.clone();
|
||||
match field {
|
||||
0 => changed.protocol_version += 1,
|
||||
1 => changed.owner_id.push('x'),
|
||||
2 => changed.instance_id.push('x'),
|
||||
3 => changed.supported = false,
|
||||
4 => changed.max_entries += 1,
|
||||
5 => changed.max_request_bytes += 1,
|
||||
_ => changed.cleared += 1,
|
||||
}
|
||||
assert_ne!(
|
||||
canonical_scoped_dirty_usage_response(&request, &changed).expect("response variant"),
|
||||
baseline
|
||||
);
|
||||
}
|
||||
assert_ne!(
|
||||
canonical_scoped_dirty_usage_response(b"another request", &response).expect("request variant"),
|
||||
baseline
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_rejects_overflow_unknown_and_duplicate_entries() {
|
||||
let base = request();
|
||||
let mut invalid = base.clone();
|
||||
invalid.entries = vec![base.entries[0].clone(); SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize + 1];
|
||||
assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge));
|
||||
invalid = base.clone();
|
||||
invalid.entries[0].bucket = "x".repeat(SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize);
|
||||
assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge));
|
||||
invalid = base.clone();
|
||||
invalid.entries.push(base.entries[0].clone());
|
||||
assert_eq!(
|
||||
validate_scoped_dirty_usage_request(&invalid),
|
||||
Err(ScopedDirtyUsageRequestError::InvalidEntries)
|
||||
);
|
||||
invalid = base;
|
||||
invalid.entries[0].bucket_incarnation = vec![0; 16].into();
|
||||
assert_eq!(
|
||||
validate_scoped_dirty_usage_request(&invalid),
|
||||
Err(ScopedDirtyUsageRequestError::InvalidIdentity)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -90,9 +90,8 @@ pub use scanner::{
|
||||
};
|
||||
pub use scanner_io::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
|
||||
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
};
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
@@ -52,9 +52,6 @@ static REMOTE_SCANNER_CYCLE_REFRESH: LazyLock<AsyncMutex<()>> = LazyLock::new(||
|
||||
|
||||
mod stream;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use stream::checkpoint_fixture_partial_return;
|
||||
|
||||
pub use stream::{RemoteScannerAdmission, RemoteScannerRequest, serve_remote_scanner_request};
|
||||
pub(crate) use stream::{RemoteScannerOutcome, RemoteScannerScanSpec, scan_remote_bucket};
|
||||
use stream::{RemoteScannerReplayCache, RemoteScannerRequestWire, RemoteScannerValidatedCycle};
|
||||
|
||||
@@ -1017,48 +1017,6 @@ fn finish_remote_scanner_stream(
|
||||
#[cfg(test)]
|
||||
const TEST_NEXT_CYCLE: u64 = 11;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn checkpoint_fixture_partial_return(progress: (u64, u64), entries_visited: u64) {
|
||||
let request_id = Uuid::new_v4();
|
||||
let writer_auth = FrameAuthenticator::for_test(request_id);
|
||||
let reader_auth = FrameAuthenticator::for_test(request_id);
|
||||
let mut bytes = Vec::new();
|
||||
write_frame(
|
||||
&mut bytes,
|
||||
&writer_auth,
|
||||
&mut 0,
|
||||
&RemoteScannerFrame::terminal(
|
||||
RemoteScannerProgress {
|
||||
objects_scanned: progress.0,
|
||||
directories_started: progress.1,
|
||||
entries_visited,
|
||||
},
|
||||
RemoteScannerFrameResult::Partial,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("checkpoint partial frame must encode");
|
||||
let frame = read_frame(&mut std::io::Cursor::new(bytes.as_slice()), &reader_auth, &mut 0)
|
||||
.await
|
||||
.expect("checkpoint progress frame must authenticate");
|
||||
assert_eq!(frame.progress.entries_visited, entries_visited);
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
|
||||
let result = consume_remote_scanner_stream(
|
||||
std::io::Cursor::new(bytes),
|
||||
parent,
|
||||
budget.clone(),
|
||||
"bucket",
|
||||
DataUsageCacheSource::new(0, 0),
|
||||
DataUsageScanPlanDigest([17; 32]),
|
||||
reader_auth,
|
||||
)
|
||||
.await
|
||||
.expect("checkpoint partial frame must decode");
|
||||
assert!(matches!(result, RemoteScannerOutcome::Partial));
|
||||
assert_eq!(budget.progress(), progress);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn consume_remote_scanner_stream<R>(
|
||||
reader: R,
|
||||
|
||||
@@ -1703,18 +1703,14 @@ where
|
||||
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
||||
|
||||
let done_cycle = Metrics::time(Metric::ScanCycle);
|
||||
let scan_result = crate::scanner_io::nsscanner_with_storage_status_scoped(
|
||||
let scan_result = crate::scanner_io::nsscanner_with_storage_status(
|
||||
storeapi.as_ref(),
|
||||
crate::scanner_io::ScannerCycleRequest {
|
||||
ctx: cycle_budget.token(),
|
||||
budget: cycle_budget.clone(),
|
||||
updates: sender,
|
||||
want_cycle: cycle_info.current,
|
||||
leader_epoch,
|
||||
scan_mode,
|
||||
scan_scope: crate::scanner_io::ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: usage_persist_baseline.data.clone(),
|
||||
},
|
||||
cycle_budget.token(),
|
||||
cycle_budget.clone(),
|
||||
sender,
|
||||
cycle_info.current,
|
||||
leader_epoch,
|
||||
scan_mode,
|
||||
)
|
||||
.await;
|
||||
let publication_defer_reason = match &scan_result {
|
||||
@@ -3428,13 +3424,10 @@ use cycle_state::*;
|
||||
use leadership::*;
|
||||
use usage_store::*;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use activity::scanner_activity_snapshot_digest;
|
||||
pub use activity::scanner_topology_digest;
|
||||
pub(crate) use activity::{
|
||||
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
|
||||
scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_structural_digest,
|
||||
scanner_dirty_usage_acknowledgements,
|
||||
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements,
|
||||
};
|
||||
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
||||
pub use backlog::{
|
||||
|
||||
@@ -902,7 +902,6 @@ where
|
||||
observation
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
@@ -926,30 +925,6 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
/// Hash the activity inputs that make an existing scanner cache unsafe to
|
||||
/// reuse. Regular namespace writes and dirty-usage generations are omitted:
|
||||
/// their affected buckets are tracked separately and may be refreshed from a
|
||||
/// complete authoritative cache baseline.
|
||||
pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
for (host, activity) in snapshot {
|
||||
let host = host.as_bytes();
|
||||
let instance_id = activity.instance_id.as_bytes();
|
||||
hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
hasher.update(host);
|
||||
hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
hasher.update(instance_id);
|
||||
hasher.update(activity.maintenance_generation.to_be_bytes());
|
||||
hasher.update(activity.protocol_version.to_be_bytes());
|
||||
hasher.update(activity.topology_digest);
|
||||
hasher.update([u8::from(activity.data_movement_active)]);
|
||||
hasher.update(activity.movement_generation.to_be_bytes());
|
||||
hasher.update([u8::from(activity.publication_blocked)]);
|
||||
}
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
|
||||
!snapshot.is_empty()
|
||||
&& snapshot.values().all(|activity| {
|
||||
@@ -980,22 +955,6 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_activity_dirty_usage_state_for_host<'a>(
|
||||
snapshot: &'a ScannerActivitySnapshot,
|
||||
host: &str,
|
||||
) -> Option<(&'a str, u64, bool)> {
|
||||
snapshot
|
||||
.get(host)
|
||||
.filter(|_| host != LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
.map(|activity| {
|
||||
(
|
||||
activity.instance_id.as_str(),
|
||||
activity.dirty_usage_generation,
|
||||
activity.dirty_usage_pending,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] {
|
||||
let endpoint_pools = storeapi.endpoints();
|
||||
let mut hasher = Sha256::new();
|
||||
|
||||
@@ -379,12 +379,6 @@ pub(super) fn decode_recovery_marker_for_reset(
|
||||
if !matches!(marker_revision, DataUsageCacheRevision::Etag(_)) {
|
||||
return Err(ScannerError::Other("cycle recovery marker has no object revision".to_string()));
|
||||
}
|
||||
if let Ok(value) = serde_json::from_slice::<serde_json::Value>(data)
|
||||
&& let Some(state) = value.get("state")
|
||||
&& !matches!(state.as_str(), Some("blocked" | "cleanup-pending"))
|
||||
{
|
||||
return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string()));
|
||||
}
|
||||
let compat = serde_json::from_slice::<ScannerCycleRecoveryMarkerCompat>(data).ok();
|
||||
let _schema_version = compat.as_ref().and_then(|marker| marker.schema_version);
|
||||
let primary_revision = compat
|
||||
@@ -412,10 +406,7 @@ pub(super) fn decode_recovery_marker_for_reset(
|
||||
};
|
||||
let state = match compat.as_ref().and_then(|marker| marker.state.as_deref()) {
|
||||
Some("cleanup-pending") => "cleanup-pending",
|
||||
Some("blocked") | None => "blocked",
|
||||
Some(_) => {
|
||||
return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string()));
|
||||
}
|
||||
_ => "blocked",
|
||||
};
|
||||
let now = unix_now_secs();
|
||||
Ok(ScannerCycleRecoveryMarker {
|
||||
@@ -730,19 +721,17 @@ async fn mark_cycle_recovery_cleanup_pending(
|
||||
mut marker: ScannerCycleRecoveryMarker,
|
||||
marker_revision: &DataUsageCacheRevision,
|
||||
expected_epoch: u64,
|
||||
owns_reset: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> {
|
||||
marker.state = "cleanup-pending".to_string();
|
||||
marker.last_attempt_at_unix_secs = unix_now_secs();
|
||||
let bytes = serde_json::to_vec(&marker)
|
||||
.map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?;
|
||||
let info = save_reset_config(
|
||||
let info = save_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
bytes,
|
||||
marker_revision.preconditions(),
|
||||
expected_epoch,
|
||||
owns_reset,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?;
|
||||
@@ -944,7 +933,6 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
.get_write_lock_quiet(Duration::from_secs(5))
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("scanner leader lock is busy: {err}")))?;
|
||||
let owns_reset = || !guard.is_lock_lost() && !ctx.is_cancelled();
|
||||
|
||||
if guard.is_lock_lost() {
|
||||
return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string()));
|
||||
@@ -964,27 +952,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
}
|
||||
Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))),
|
||||
};
|
||||
let Some(marker_data) = marker_data else {
|
||||
// A delete may commit before its reply is lost. Confirm both durable
|
||||
// fences before treating a retry without its marker as completed.
|
||||
let (cycle, epoch, revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
|
||||
let floor = persisted_usage_floor(storeapi.clone()).await?;
|
||||
if !matches!(revision, DataUsageCacheRevision::Etag(_))
|
||||
|| epoch < floor.leader_epoch
|
||||
|| cycle.next < floor.next_cycle
|
||||
|| !owns_reset()
|
||||
|| scanner_publication_admission_for_epoch(storeapi.clone(), reset_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return Err(ScannerError::Other(
|
||||
"scanner cycle recovery marker is absent without a completed reset fence".to_string(),
|
||||
));
|
||||
}
|
||||
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
|
||||
super::notify_scanner_cycle_recovery_wake();
|
||||
return Ok(());
|
||||
};
|
||||
let marker_data = marker_data.ok_or_else(|| ScannerError::Other("scanner cycle recovery marker is absent".to_string()))?;
|
||||
let (marker, force_full_rescan) = match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&marker_data) {
|
||||
Ok(marker) if validate_recovery_marker(&marker).is_ok() => (marker, false),
|
||||
_ => (decode_recovery_marker_for_reset(&marker_data, &marker_revision)?, true),
|
||||
@@ -1058,10 +1026,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
}
|
||||
};
|
||||
if let Some((primary_cycle, primary_epoch)) = primary_state {
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
let (cleanup_marker, cleanup_marker_revision) =
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch, &owns_reset)
|
||||
.await?;
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?;
|
||||
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
|
||||
let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
|
||||
let fence_epoch = primary_epoch
|
||||
@@ -1081,14 +1047,12 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"preserved scanner cycle state exceeds the bounded object size".to_string(),
|
||||
));
|
||||
}
|
||||
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
|
||||
let preserved_info = save_reset_config(
|
||||
let preserved_info = save_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
preserved_data,
|
||||
primary_revision.preconditions(),
|
||||
reset_epoch,
|
||||
&owns_reset,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
@@ -1108,17 +1072,9 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner leader lock was lost after fencing newer cycle state".to_string(),
|
||||
));
|
||||
}
|
||||
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
|
||||
fence_scanner_usage_epoch_with_expected_epoch(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
fence_epoch,
|
||||
Some(reset_epoch),
|
||||
false,
|
||||
&owns_reset,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch), false)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner leader lock was lost after fencing newer cycle state".to_string(),
|
||||
@@ -1132,8 +1088,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner cycle state changed before recovery marker cleanup".to_string(),
|
||||
));
|
||||
}
|
||||
verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
|
||||
delete_reset_config(
|
||||
delete_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
@@ -1145,7 +1100,6 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
..Default::default()
|
||||
},
|
||||
reset_epoch,
|
||||
&owns_reset,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
@@ -1195,20 +1149,17 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
// Persist the cleanup-pending phase before rewriting the primary. If the
|
||||
// process dies after the rewrite, startup still sees a durable fence and
|
||||
// cannot mistake the partially completed reset for a healthy state.
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
let (marker, marker_revision) = if marker.state == "cleanup-pending" {
|
||||
(marker, marker_revision)
|
||||
} else {
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch, &owns_reset).await?
|
||||
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await?
|
||||
};
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
let rebuilt_info = save_reset_config(
|
||||
let rebuilt_info = save_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
data,
|
||||
primary_revision.preconditions(),
|
||||
reset_epoch,
|
||||
&owns_reset,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
@@ -1227,10 +1178,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
"scanner leader lock was lost after rebuilding cycle state".to_string(),
|
||||
));
|
||||
}
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
if let Err(err) =
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false, &owns_reset)
|
||||
.await
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false).await
|
||||
{
|
||||
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
|
||||
path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
|
||||
@@ -1300,8 +1249,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
));
|
||||
}
|
||||
|
||||
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
|
||||
if let Err(err) = delete_reset_config(
|
||||
if let Err(err) = delete_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
RUSTFS_META_BUCKET,
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
@@ -1313,7 +1261,6 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
..Default::default()
|
||||
},
|
||||
reset_epoch,
|
||||
&owns_reset,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1363,57 +1310,6 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn verify_cycle_reset_intent(
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
expected_revision: &DataUsageCacheRevision,
|
||||
owns_reset: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<(), ScannerError> {
|
||||
let revision = read_config_revision(storeapi, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to verify scanner cycle reset intent: {err}")))?;
|
||||
if &revision != expected_revision {
|
||||
return Err(ScannerError::Other("scanner cycle reset intent changed".to_string()));
|
||||
}
|
||||
if !owns_reset() {
|
||||
return Err(ScannerError::Other("scanner cycle reset ownership was lost".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn save_reset_config(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
path: &str,
|
||||
data: Vec<u8>,
|
||||
preconditions: crate::HTTPPreconditions,
|
||||
expected_epoch: u64,
|
||||
owns_reset: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<crate::ScannerObjectInfo, EcstoreError> {
|
||||
let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
};
|
||||
if !owns_reset() {
|
||||
return Err(EcstoreError::other("scanner reset ownership was lost before write"));
|
||||
}
|
||||
save_config_with_preconditions(storeapi, path, data, preconditions).await
|
||||
}
|
||||
|
||||
async fn delete_reset_config(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
bucket: &str,
|
||||
path: &str,
|
||||
options: ScannerObjectOptions,
|
||||
expected_epoch: u64,
|
||||
owns_reset: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<crate::ScannerObjectInfo, EcstoreError> {
|
||||
let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
|
||||
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
};
|
||||
if !owns_reset() {
|
||||
return Err(EcstoreError::other("scanner reset ownership was lost before delete"));
|
||||
}
|
||||
storeapi.delete_config_object(bucket, path, options).await
|
||||
}
|
||||
|
||||
fn scanner_usage_state_reset_paths() -> Vec<String> {
|
||||
vec![
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
|
||||
@@ -1437,14 +1333,8 @@ pub(super) async fn read_usage_state_reset_slots(
|
||||
Ok(slots)
|
||||
}
|
||||
|
||||
enum ScannerUsageResetFloor {
|
||||
Missing,
|
||||
Trusted(PersistedUsageFloor),
|
||||
Corrupt,
|
||||
}
|
||||
|
||||
fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<ScannerUsageResetFloor, ScannerError> {
|
||||
let mut floor = None;
|
||||
fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<PersistedUsageFloor, ScannerError> {
|
||||
let mut floor = PersistedUsageFloor::default();
|
||||
for slot in slots {
|
||||
let Some(data) = slot.data.as_deref() else {
|
||||
continue;
|
||||
@@ -1452,21 +1342,9 @@ fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<Scann
|
||||
let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(data) else {
|
||||
continue;
|
||||
};
|
||||
if !data_usage_info_has_persisted_baseline_identity(&usage)
|
||||
&& !(slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str() && data_usage_info_is_bootstrap_pending(&usage))
|
||||
&& legacy_incomplete_usage_fence(data, &usage)
|
||||
.and_then(|fence| fence.claimable_epoch())
|
||||
.is_none()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
update_persisted_usage_floor(floor.get_or_insert_with(PersistedUsageFloor::default), &usage, &slot.path)?;
|
||||
update_persisted_usage_floor(&mut floor, &usage, &slot.path)?;
|
||||
}
|
||||
Ok(match floor {
|
||||
Some(floor) => ScannerUsageResetFloor::Trusted(floor),
|
||||
None if slots.iter().any(|slot| slot.data.is_some()) => ScannerUsageResetFloor::Corrupt,
|
||||
None => ScannerUsageResetFloor::Missing,
|
||||
})
|
||||
Ok(floor)
|
||||
}
|
||||
|
||||
async fn read_cycle_state_for_usage_reset(
|
||||
@@ -1523,12 +1401,11 @@ async fn delete_usage_state_reset_slot(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
slot: &ScannerUsageStateResetSlot,
|
||||
expected_epoch: u64,
|
||||
owns_reset: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<bool, ScannerError> {
|
||||
if matches!(slot.revision, DataUsageCacheRevision::Missing) {
|
||||
return Ok(false);
|
||||
}
|
||||
let delete_result = delete_reset_config(
|
||||
let delete_result = delete_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&slot.path,
|
||||
@@ -1538,7 +1415,6 @@ async fn delete_usage_state_reset_slot(
|
||||
..Default::default()
|
||||
},
|
||||
expected_epoch,
|
||||
owns_reset,
|
||||
)
|
||||
.await;
|
||||
match delete_result {
|
||||
@@ -1610,24 +1486,21 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary(
|
||||
expected_publication_epoch: u64,
|
||||
leader_epoch: Option<u64>,
|
||||
context: ScannerUsageBootstrapPublishContext,
|
||||
owns_publication: impl Fn() -> bool + Sync,
|
||||
) -> Result<(), ScannerError> {
|
||||
async fn inner(
|
||||
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
|
||||
expected_revision: &DataUsageCacheRevision,
|
||||
expected_publication_epoch: u64,
|
||||
leader_epoch: Option<u64>,
|
||||
owns_publication: &(impl Fn() -> bool + Sync),
|
||||
) -> Result<(), ScannerUsageBootstrapPublishError> {
|
||||
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::now(), leader_epoch);
|
||||
let data = serde_json::to_vec(&marker).map_err(ScannerUsageBootstrapPublishError::Encode)?;
|
||||
let save_result = save_reset_config(
|
||||
let save_result = save_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
data.clone(),
|
||||
expected_revision.preconditions(),
|
||||
expected_publication_epoch,
|
||||
owns_publication,
|
||||
)
|
||||
.await;
|
||||
if save_result
|
||||
@@ -1651,7 +1524,7 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary(
|
||||
})
|
||||
}
|
||||
|
||||
inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch, &owns_publication)
|
||||
inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch)
|
||||
.await
|
||||
.map_err(|err| err.into_scanner_error(context))
|
||||
}
|
||||
@@ -1661,108 +1534,32 @@ pub(super) async fn reset_scanner_usage_state_slots_for_full_rebuild(
|
||||
slots: &[ScannerUsageStateResetSlot],
|
||||
expected_epoch: u64,
|
||||
leader_epoch: u64,
|
||||
owns_reset: impl Fn() -> bool + Sync,
|
||||
) -> Result<Vec<String>, ScannerError> {
|
||||
let mut reset_paths = Vec::new();
|
||||
let primary = slots
|
||||
.iter()
|
||||
.find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.ok_or_else(|| ScannerError::Other("scanner usage reset primary slot was not inspected".to_string()))?;
|
||||
if !owns_reset() {
|
||||
return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string()));
|
||||
}
|
||||
let resume_epoch = usage_state_reset_resume_epoch(slots)?;
|
||||
match resume_epoch {
|
||||
Some(epoch) if epoch == leader_epoch => {}
|
||||
Some(_) => return Err(ScannerError::Other("scanner usage reset bootstrap epoch changed".to_string())),
|
||||
None => {
|
||||
publish_scanner_usage_bootstrap_primary(
|
||||
storeapi.clone(),
|
||||
&primary.revision,
|
||||
expected_epoch,
|
||||
Some(leader_epoch),
|
||||
ScannerUsageBootstrapPublishContext::Reset,
|
||||
&owns_reset,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
let (data, intent_revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to inspect scanner usage reset intent: {err}")))?;
|
||||
data.as_deref()
|
||||
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok())
|
||||
.filter(|usage| data_usage_info_is_bootstrap_pending(usage) && usage.scanner_epoch == Some(leader_epoch))
|
||||
.ok_or_else(|| ScannerError::Other("scanner usage reset intent changed before cleanup".to_string()))?;
|
||||
if !matches!(intent_revision, DataUsageCacheRevision::Etag(_))
|
||||
|| (resume_epoch.is_some() && intent_revision != primary.revision)
|
||||
{
|
||||
return Err(ScannerError::Other("scanner usage reset intent revision changed".to_string()));
|
||||
}
|
||||
publish_scanner_usage_bootstrap_primary(
|
||||
storeapi.clone(),
|
||||
&primary.revision,
|
||||
expected_epoch,
|
||||
Some(leader_epoch),
|
||||
ScannerUsageBootstrapPublishContext::Reset,
|
||||
)
|
||||
.await?;
|
||||
reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string());
|
||||
|
||||
for slot in slots.iter().filter(|slot| slot.path != DATA_USAGE_OBJ_NAME_PATH.as_str()) {
|
||||
if let Some(usage) = slot
|
||||
.data
|
||||
.as_deref()
|
||||
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok())
|
||||
&& usage_epoch(&usage) >= leader_epoch
|
||||
{
|
||||
return Err(ScannerError::Other(format!(
|
||||
"scanner usage reset slot is not older than its intent: {}",
|
||||
slot.path
|
||||
)));
|
||||
}
|
||||
let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to verify scanner usage reset intent: {err}")))?;
|
||||
if revision != intent_revision {
|
||||
return Err(ScannerError::Other("scanner usage reset intent changed during cleanup".to_string()));
|
||||
}
|
||||
if !owns_reset() {
|
||||
return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string()));
|
||||
}
|
||||
if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch, &owns_reset).await? {
|
||||
if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch).await? {
|
||||
reset_paths.push(slot.path.clone());
|
||||
}
|
||||
}
|
||||
let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to confirm scanner usage reset intent: {err}")))?;
|
||||
if revision != intent_revision || !owns_reset() {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner usage reset intent or ownership changed before completion".to_string(),
|
||||
));
|
||||
}
|
||||
invalidate_admin_data_usage_snapshot_cache().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
Ok(reset_paths)
|
||||
}
|
||||
|
||||
fn usage_state_reset_resume_epoch(slots: &[ScannerUsageStateResetSlot]) -> Result<Option<u64>, ScannerError> {
|
||||
let primary = slots.iter().find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let usage = primary
|
||||
.and_then(|slot| slot.data.as_deref())
|
||||
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok());
|
||||
match usage {
|
||||
Some(usage) if usage.usage_snapshot_bootstrap_pending => {
|
||||
if !data_usage_info_is_bootstrap_pending(&usage) {
|
||||
return Err(ScannerError::Other("scanner usage reset bootstrap is invalid".to_string()));
|
||||
}
|
||||
if usage.scanner_epoch.is_none() {
|
||||
// Initial bootstrap has no reset owner yet.
|
||||
return Ok(None);
|
||||
}
|
||||
usage
|
||||
.scanner_epoch
|
||||
.filter(|epoch| *epoch > 0 && *epoch < u64::MAX)
|
||||
.map(Some)
|
||||
.ok_or_else(|| ScannerError::Other("scanner usage reset bootstrap has no valid epoch".to_string()))
|
||||
}
|
||||
_ => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reset_scanner_usage_state_for_full_rebuild(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<ECStore>,
|
||||
@@ -1787,31 +1584,12 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
|
||||
};
|
||||
let (cycle, cycle_epoch, cycle_revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
|
||||
let slots = read_usage_state_reset_slots(storeapi.clone()).await?;
|
||||
let usage_floor = match usage_state_reset_floor(&slots)? {
|
||||
ScannerUsageResetFloor::Trusted(floor) => floor,
|
||||
ScannerUsageResetFloor::Corrupt if matches!(cycle_revision, DataUsageCacheRevision::Missing) => {
|
||||
return Err(ScannerError::Other("scanner usage reset has no trusted cycle or usage floor".to_string()));
|
||||
}
|
||||
ScannerUsageResetFloor::Missing | ScannerUsageResetFloor::Corrupt => PersistedUsageFloor {
|
||||
next_cycle: cycle.next,
|
||||
leader_epoch: cycle_epoch,
|
||||
},
|
||||
};
|
||||
let resume_epoch = usage_state_reset_resume_epoch(&slots)?;
|
||||
let leader_epoch = if let Some(epoch) = resume_epoch {
|
||||
if epoch != cycle_epoch || usage_floor.leader_epoch > epoch || usage_floor.next_cycle > cycle.next {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner usage reset bootstrap conflicts with the persisted cycle fence".to_string(),
|
||||
));
|
||||
}
|
||||
epoch
|
||||
} else {
|
||||
cycle_epoch
|
||||
.max(usage_floor.leader_epoch)
|
||||
.checked_add(1)
|
||||
.filter(|epoch| *epoch < u64::MAX)
|
||||
.ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?
|
||||
};
|
||||
let usage_floor = usage_state_reset_floor(&slots)?;
|
||||
let leader_epoch = cycle_epoch
|
||||
.max(usage_floor.leader_epoch)
|
||||
.checked_add(1)
|
||||
.filter(|epoch| *epoch < u64::MAX)
|
||||
.ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?;
|
||||
let rebuilt_cycle = CurrentCycle {
|
||||
next: cycle.next.max(usage_floor.next_cycle),
|
||||
..Default::default()
|
||||
@@ -1824,24 +1602,21 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
|
||||
"scanner leader lock was lost before fencing usage reset cycle state".to_string(),
|
||||
));
|
||||
}
|
||||
if resume_epoch.is_none() {
|
||||
save_reset_config(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
cycle_data,
|
||||
cycle_revision.preconditions(),
|
||||
reset_epoch,
|
||||
&|| !guard.is_lock_lost() && !ctx.is_cancelled(),
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string())
|
||||
} else {
|
||||
ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}"))
|
||||
}
|
||||
})?;
|
||||
}
|
||||
save_config_with_publication_admission_for_epoch(
|
||||
storeapi.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
cycle_data,
|
||||
cycle_revision.preconditions(),
|
||||
reset_epoch,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if scanner_publication_epoch_changed(&err) {
|
||||
ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string())
|
||||
} else {
|
||||
ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}"))
|
||||
}
|
||||
})?;
|
||||
|
||||
if guard.is_lock_lost() {
|
||||
return Err(ScannerError::Other(
|
||||
@@ -1849,10 +1624,7 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
|
||||
));
|
||||
}
|
||||
let reset_paths =
|
||||
reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch, || {
|
||||
!guard.is_lock_lost() && !ctx.is_cancelled()
|
||||
})
|
||||
.await?;
|
||||
reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch).await?;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(ScannerError::Other(
|
||||
"scanner leader lock was lost after publishing usage reset marker".to_string(),
|
||||
@@ -2363,7 +2135,6 @@ async fn recover_legacy_incomplete_usage_floor(
|
||||
expected_publication_epoch,
|
||||
Some(primary.epoch),
|
||||
ScannerUsageBootstrapPublishContext::Recovery,
|
||||
|| true,
|
||||
)
|
||||
.await?;
|
||||
warn!(
|
||||
|
||||
@@ -191,7 +191,6 @@ pub(super) async fn initialize_usage_baseline_bootstrap(
|
||||
expected_epoch,
|
||||
None,
|
||||
ScannerUsageBootstrapPublishContext::Initial,
|
||||
|| true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -202,10 +201,9 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
|
||||
claimed_epoch: u64,
|
||||
expected_publication_epoch: Option<u64>,
|
||||
allow_bootstrap_pending: bool,
|
||||
owns_fence: impl Fn() -> bool,
|
||||
) -> Result<(), ScannerError> {
|
||||
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
|
||||
if ctx.is_cancelled() || !owns_fence() {
|
||||
if ctx.is_cancelled() {
|
||||
return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string()));
|
||||
}
|
||||
|
||||
@@ -266,9 +264,6 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
|
||||
"scanner usage epoch fence changed while preparing its conditional write".to_string(),
|
||||
));
|
||||
};
|
||||
if ctx.is_cancelled() || !owns_fence() {
|
||||
return Err(ScannerError::Other("scanner leadership was lost before usage fencing".to_string()));
|
||||
}
|
||||
save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions())
|
||||
.await
|
||||
};
|
||||
@@ -324,7 +319,6 @@ pub(super) async fn complete_scanner_leadership_claim(
|
||||
claimed_epoch,
|
||||
expected_publication_epoch,
|
||||
allow_bootstrap_pending,
|
||||
|| true,
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -634,8 +634,6 @@ struct MemoryConfigStore {
|
||||
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
|
||||
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
||||
error_after_commit_deletes: Mutex<HashSet<String>>,
|
||||
cancel_after_deletes: Mutex<HashMap<String, CancellationToken>>,
|
||||
pause_next_publication_admission: Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>,
|
||||
put_counts: Mutex<HashMap<String, usize>>,
|
||||
publication_admission_blocked: AtomicBool,
|
||||
block_publication_after_admissions: AtomicUsize,
|
||||
@@ -4083,9 +4081,6 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
|
||||
revisions.remove(&key);
|
||||
drop(revisions);
|
||||
drop(objects);
|
||||
if let Some(token) = self.cancel_after_deletes.lock().await.remove(&key) {
|
||||
token.cancel();
|
||||
}
|
||||
if self.error_after_commit_deletes.lock().await.remove(&key) {
|
||||
return Err(EcstoreError::other("injected delete error after commit"));
|
||||
}
|
||||
@@ -4093,11 +4088,6 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
|
||||
let pause = self.pause_next_publication_admission.lock().await.take();
|
||||
if let Some((entered, resume)) = pause {
|
||||
entered.notify_one();
|
||||
resume.notified().await;
|
||||
}
|
||||
if self.publication_admission_blocked.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
@@ -4599,7 +4589,7 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met
|
||||
.expect("publication must also read the intact backup");
|
||||
assert_eq!(baseline.data.as_deref(), Some(data.as_slice()));
|
||||
assert_eq!(baseline.revision, DataUsageCacheRevision::Missing);
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false, || true)
|
||||
fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false)
|
||||
.await
|
||||
.expect("legacy backup must be fenced into v2");
|
||||
let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
@@ -4828,28 +4818,6 @@ async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() {
|
||||
);
|
||||
}
|
||||
|
||||
let cycle_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("cycle should remain before retry");
|
||||
let marker_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("bootstrap should remain before retry");
|
||||
let retry = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("completed cleanup should be reentrant");
|
||||
assert_eq!(retry.leader_epoch, result.leader_epoch);
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("cycle should remain"),
|
||||
cycle_before_retry
|
||||
);
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("bootstrap should remain"),
|
||||
marker_before_retry
|
||||
);
|
||||
let (floor, state) = persisted_usage_floor_for_startup(store, false)
|
||||
.await
|
||||
.expect("reset marker should be resumable");
|
||||
@@ -5005,7 +4973,7 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() {
|
||||
|
||||
store.objects.lock().await.insert(key.clone(), b"newer-json".to_vec());
|
||||
store.revisions.lock().await.insert(key, 2);
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true)
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3)
|
||||
.await
|
||||
.expect_err("stale primary revision must not be overwritten");
|
||||
assert!(
|
||||
@@ -5015,348 +4983,6 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_resumes_every_cleanup_boundary_without_rewriting_intent() {
|
||||
for completed in 0..=4 {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str();
|
||||
let cleanup_paths = [
|
||||
format!("{primary_path}.bkp"),
|
||||
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
|
||||
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str().to_string(),
|
||||
];
|
||||
for path in std::iter::once(primary_path).chain(cleanup_paths.iter().map(String::as_str)) {
|
||||
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
usage.scanner_epoch = Some(1);
|
||||
save_config(store.clone(), path, serde_json::to_vec(&usage).expect("fixture should encode"))
|
||||
.await
|
||||
.expect("fixture should persist");
|
||||
}
|
||||
// These objects belong to other owners, even when reset cleanup resumes.
|
||||
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
|
||||
save_config(store.clone(), path, b"retain".to_vec())
|
||||
.await
|
||||
.expect("unrelated state should persist");
|
||||
}
|
||||
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
|
||||
let cancelled = CancellationToken::new();
|
||||
if completed == 0 {
|
||||
store
|
||||
.cancel_after_successful_puts
|
||||
.lock()
|
||||
.await
|
||||
.insert(memory_config_key(RUSTFS_META_BUCKET, primary_path), (2, cancelled.clone()));
|
||||
} else {
|
||||
store
|
||||
.cancel_after_deletes
|
||||
.lock()
|
||||
.await
|
||||
.insert(memory_config_key(RUSTFS_META_BUCKET, &cleanup_paths[completed - 1]), cancelled.clone());
|
||||
}
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled())
|
||||
.await
|
||||
.expect_err("interruption should stop cleanup");
|
||||
assert!(err.to_string().contains("ownership"), "boundary {completed}: {err}");
|
||||
for (index, path) in cleanup_paths.iter().enumerate() {
|
||||
assert_eq!(
|
||||
store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, path)),
|
||||
index >= completed,
|
||||
"boundary {completed}, slot {index}"
|
||||
);
|
||||
}
|
||||
let intent = read_config_with_revision(store.clone(), primary_path)
|
||||
.await
|
||||
.expect("intent should persist");
|
||||
let slots = read_usage_state_reset_slots(store.clone())
|
||||
.await
|
||||
.expect("restart should reload slots");
|
||||
reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
|
||||
.await
|
||||
.expect("restart should complete the same intent");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), primary_path)
|
||||
.await
|
||||
.expect("intent should remain"),
|
||||
intent
|
||||
);
|
||||
assert_eq!(store.put_counts.lock().await[&memory_config_key(RUSTFS_META_BUCKET, primary_path)], 2);
|
||||
for path in cleanup_paths {
|
||||
assert!(
|
||||
!store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, &path))
|
||||
);
|
||||
}
|
||||
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
|
||||
assert_eq!(read_config(store.clone(), path).await.expect("unrelated state should remain"), b"retain");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_stops_usage_fence_after_owner_loss() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
usage.scanner_epoch = Some(1);
|
||||
let bytes = serde_json::to_vec(&usage).expect("baseline should encode");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
|
||||
.await
|
||||
.expect("baseline should persist");
|
||||
let checks = AtomicUsize::new(0);
|
||||
let err = fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 3, Some(0), false, || {
|
||||
checks.fetch_add(1, Ordering::SeqCst) == 0
|
||||
})
|
||||
.await
|
||||
.expect_err("ownership lost during reads must prevent the write");
|
||||
assert!(err.to_string().contains("leadership was lost"), "{err}");
|
||||
assert_eq!(
|
||||
read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("baseline should remain"),
|
||||
bytes
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_cancels_during_publication_admission() {
|
||||
for resuming in [false, true] {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let usage = if resuming {
|
||||
scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3))
|
||||
} else {
|
||||
complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
|
||||
};
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&usage).expect("primary should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("primary should persist");
|
||||
save_config(store.clone(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), b"corrupt".to_vec())
|
||||
.await
|
||||
.expect("cleanup target should persist");
|
||||
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
|
||||
let before = store.objects.lock().await.clone();
|
||||
let revisions_before = store.revisions.lock().await.clone();
|
||||
let entered = Arc::new(tokio::sync::Notify::new());
|
||||
let resume = Arc::new(tokio::sync::Notify::new());
|
||||
*store.pause_next_publication_admission.lock().await = Some((entered.clone(), resume.clone()));
|
||||
let cancelled = CancellationToken::new();
|
||||
let (result, ()) = tokio::join!(
|
||||
reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled()),
|
||||
async {
|
||||
entered.notified().await;
|
||||
cancelled.cancel();
|
||||
resume.notify_one();
|
||||
}
|
||||
);
|
||||
let err = result.expect_err("losing ownership during admission must prevent mutation");
|
||||
assert!(err.to_string().contains("ownership was lost"), "resuming={resuming}: {err}");
|
||||
assert_eq!(*store.objects.lock().await, before);
|
||||
assert_eq!(*store.revisions.lock().await, revisions_before);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_usage_state_reset_rejects_corruption_without_a_trusted_floor() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), b"{corrupt".to_vec())
|
||||
.await
|
||||
.expect("corrupt primary should persist");
|
||||
let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("evidence should load");
|
||||
let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("corruption must not become a zero floor");
|
||||
assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("evidence should remain"),
|
||||
before
|
||||
);
|
||||
assert!(matches!(
|
||||
read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await,
|
||||
Err(EcstoreError::ConfigNotFound)
|
||||
));
|
||||
let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
backup.scanner_epoch = Some(7);
|
||||
backup.scanner_cycle = Some(40);
|
||||
save_config(
|
||||
store.clone(),
|
||||
&format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
|
||||
serde_json::to_vec(&backup).expect("backup should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("valid backup should persist");
|
||||
let result = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store)
|
||||
.await
|
||||
.expect("valid backup should supply the recovery floor");
|
||||
assert_eq!(result.leader_epoch, 8);
|
||||
assert_eq!(result.next_cycle, 41);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_rejects_replaced_intent_and_newer_cleanup_slot() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3));
|
||||
let bytes = serde_json::to_vec(&marker).expect("marker should encode");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
|
||||
.await
|
||||
.expect("intent should persist");
|
||||
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes)
|
||||
.await
|
||||
.expect("another intent should persist");
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
|
||||
.await
|
||||
.expect_err("same epoch cannot replace an intent revision");
|
||||
assert!(err.to_string().contains("intent revision changed"), "{err}");
|
||||
|
||||
let mut newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
|
||||
newer.scanner_epoch = Some(3);
|
||||
let path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
|
||||
let bytes = serde_json::to_vec(&newer).expect("newer snapshot should encode");
|
||||
save_config(store.clone(), &path, bytes.clone())
|
||||
.await
|
||||
.expect("newer snapshot should persist");
|
||||
let slots = read_usage_state_reset_slots(store.clone())
|
||||
.await
|
||||
.expect("slots should reload");
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
|
||||
.await
|
||||
.expect_err("cleanup cannot delete same-epoch progress");
|
||||
assert!(err.to_string().contains("not older than its intent"), "{err}");
|
||||
assert_eq!(read_config(store, &path).await.expect("newer snapshot should remain"), bytes);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_usage_state_reset_rejects_decodable_untrusted_floor() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
|
||||
let invalid_identity = DataUsageInfo {
|
||||
usage_snapshot_complete: true,
|
||||
buckets_count: 1,
|
||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
for usage in [DataUsageInfo::default(), invalid_identity] {
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&usage).expect("fixture should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("untrusted primary should persist");
|
||||
let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("primary should load");
|
||||
let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("valid JSON alone cannot prove a usage floor");
|
||||
assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("evidence should remain"),
|
||||
before
|
||||
);
|
||||
assert!(matches!(
|
||||
read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await,
|
||||
Err(EcstoreError::ConfigNotFound)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_rescan_reset_rejects_unknown_marker_phase_even_with_invalid_compat_fields() {
|
||||
for state in [serde_json::json!("rewrite-v2"), serde_json::json!(7), serde_json::Value::Null] {
|
||||
let marker = serde_json::json!({"state": state, "retry_count": "future-type", "schema_version": 99});
|
||||
let err = super::cycle_state::decode_recovery_marker_for_reset(
|
||||
&serde_json::to_vec(&marker).expect("future marker should encode"),
|
||||
&DataUsageCacheRevision::Etag("intent-1".to_string()),
|
||||
)
|
||||
.expect_err("unknown persistent phases must remain fenced");
|
||||
assert!(err.to_string().contains("state is unsupported"), "{err}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn full_rescan_reset_preserves_unknown_phase_and_retries_completed_cleanup() {
|
||||
let (_temp_dir, store) = setup_scanner_cycle_store().await;
|
||||
save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt".to_vec())
|
||||
.await
|
||||
.expect("corrupt primary should persist");
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
|
||||
br#"{"state":"future-rewrite"}"#.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("future marker should persist");
|
||||
let primary_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("primary should load");
|
||||
let marker_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
|
||||
.await
|
||||
.expect("marker should load");
|
||||
let err = reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect_err("unknown phase must block explicit reset");
|
||||
assert!(err.to_string().contains("state is unsupported"), "{err}");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("primary should remain"),
|
||||
primary_before
|
||||
);
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
|
||||
.await
|
||||
.expect("marker should remain"),
|
||||
marker_before
|
||||
);
|
||||
|
||||
save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{malformed".to_vec())
|
||||
.await
|
||||
.expect("recoverable marker should persist");
|
||||
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("reset should complete");
|
||||
let primary = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("rebuilt primary should load");
|
||||
let usage = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("fenced usage should load");
|
||||
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
|
||||
.await
|
||||
.expect("retry after marker deletion should complete");
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("rebuilt primary should remain"),
|
||||
primary
|
||||
);
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("fenced usage should remain"),
|
||||
usage
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
@@ -5367,7 +4993,7 @@ async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
|
||||
.expect("usage reset slots should be inspected");
|
||||
store.publication_admission_blocked.store(true, Ordering::Release);
|
||||
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true)
|
||||
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3)
|
||||
.await
|
||||
.expect_err("movement admission loss must defer reset");
|
||||
assert!(
|
||||
@@ -8543,44 +8169,6 @@ fn scanner_activity_snapshot_digest_fences_dirty_usage_state() {
|
||||
assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_structural_digest_ignores_regular_bucket_writes() {
|
||||
let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
||||
let mut written = baseline.clone();
|
||||
let activity = written.get_mut("node-2").expect("node should exist");
|
||||
activity.namespace_generation = 8;
|
||||
activity.dirty_usage_generation = 6;
|
||||
activity.dirty_usage_pending = true;
|
||||
|
||||
assert_ne!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&written));
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&written),
|
||||
"bucket writes are refreshed through the dirty-bucket scope rather than invalidating every cache"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_structural_digest_fences_restart_and_maintenance() {
|
||||
let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
||||
let mut restarted = baseline.clone();
|
||||
restarted.get_mut("node-2").expect("node should exist").instance_id = "epoch-b".to_string();
|
||||
let mut maintained = baseline.clone();
|
||||
maintained
|
||||
.get_mut("node-2")
|
||||
.expect("node should exist")
|
||||
.maintenance_generation = 4;
|
||||
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&restarted)
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&maintained)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() {
|
||||
let snapshot = BTreeMap::from([
|
||||
|
||||
@@ -230,7 +230,7 @@ fn scanner_abandoned_child_list_options() -> ListPathRawOptions {
|
||||
}
|
||||
|
||||
pub fn data_usage_update_dir_cycles() -> u32 {
|
||||
rustfs_utils::get_env_u32(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, DATA_USAGE_UPDATE_DIR_CYCLES)
|
||||
rustfs_utils::get_env_u32(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, DATA_USAGE_UPDATE_DIR_CYCLES).max(1)
|
||||
}
|
||||
|
||||
pub fn heal_object_select_prob() -> u32 {
|
||||
@@ -806,6 +806,7 @@ impl FolderScanner {
|
||||
fn prune_failed_objects_cache(&mut self) {
|
||||
let ttl = self.failed_object_ttl_secs;
|
||||
if ttl == 0 {
|
||||
self.new_cache.info.failed_objects.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -963,6 +964,27 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
async fn preserve_failed_child(
|
||||
&mut self,
|
||||
parent: &Option<DataUsageHash>,
|
||||
child_hash: &DataUsageHash,
|
||||
parent_entry: &mut DataUsageEntry,
|
||||
child_entry: &DataUsageEntry,
|
||||
) {
|
||||
// A failed walk proves neither deletion nor a complete replacement.
|
||||
// Keep the previous subtree and mark this snapshot incomplete even
|
||||
// when the failed-object retry cache is disabled or at capacity.
|
||||
parent_entry.failed_objects = parent_entry.failed_objects.saturating_add(1);
|
||||
if self.old_cache.cache.contains_key(&child_hash.key()) {
|
||||
self.new_cache.delete_recursive(child_hash);
|
||||
self.new_cache.copy_with_children(&self.old_cache, child_hash, parent);
|
||||
parent_entry.add_child(child_hash);
|
||||
} else {
|
||||
self.preserve_partial_child_progress(parent, child_hash, parent_entry, child_entry)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn alert_excessive_folders(&self, folder: &str, total_folders: usize) {
|
||||
let threshold = scanner_excess_folders_threshold();
|
||||
if u64::try_from(total_folders).unwrap_or(u64::MAX) <= threshold {
|
||||
@@ -1177,8 +1199,6 @@ impl FolderScanner {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
|
||||
self.prune_failed_objects_cache();
|
||||
|
||||
let mut abandoned_children: DataUsageHashMap = HashSet::new();
|
||||
if !into.compacted {
|
||||
abandoned_children = self.old_cache.find_children_copy(this_hash.clone());
|
||||
@@ -1221,7 +1241,9 @@ impl FolderScanner {
|
||||
};
|
||||
let active_object_lock = self.old_cache.info.object_lock.clone();
|
||||
|
||||
self.sleeper.sleep_folder().await;
|
||||
ctx.run_until_cancelled(self.sleeper.sleep_folder())
|
||||
.await
|
||||
.ok_or_else(|| ScannerError::Other("Operation cancelled".to_string()))?;
|
||||
|
||||
let mut existing_folders: Vec<CachedFolder> = Vec::new();
|
||||
let mut new_folders: Vec<CachedFolder> = Vec::new();
|
||||
@@ -1448,7 +1470,7 @@ impl FolderScanner {
|
||||
|
||||
let heal_enabled = this_hash.mod_alt(
|
||||
self.old_cache.info.next_cycle as u32 / folder.object_heal_prob_div,
|
||||
self.heal_object_select / folder.object_heal_prob_div,
|
||||
(self.heal_object_select / folder.object_heal_prob_div).max(1),
|
||||
) && self.should_heal().await;
|
||||
|
||||
let mut item = ScannerItem {
|
||||
@@ -1465,12 +1487,10 @@ impl FolderScanner {
|
||||
file_type: entry_type,
|
||||
};
|
||||
|
||||
// If this path is already known as failed, just skip it.
|
||||
// We intentionally do NOT call `record_failed` or bump `failed_objects` here,
|
||||
// because the failure was recorded when the original error occurred
|
||||
// (e.g. in the get_size error branch below). This branch only accounts
|
||||
// for subsequent skips of already-failed paths.
|
||||
// Count unresolved objects in each snapshot without extending
|
||||
// the retry TTL or emitting another failure event.
|
||||
if self.should_skip_failed(&item.path) {
|
||||
into.failed_objects = into.failed_objects.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1485,7 +1505,7 @@ impl FolderScanner {
|
||||
|
||||
if failure_action != GetSizeFailureAction::Skip {
|
||||
// Track failed objects to prevent infinite retry loops
|
||||
into.failed_objects += 1;
|
||||
into.failed_objects = into.failed_objects.saturating_add(1);
|
||||
self.record_failed(&item.path);
|
||||
|
||||
if should_log_failed_object(into.failed_objects) {
|
||||
@@ -1564,12 +1584,15 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
timer.sleep().await;
|
||||
ctx.run_until_cancelled(timer.sleep())
|
||||
.await
|
||||
.ok_or_else(|| ScannerError::Other("Operation cancelled".to_string()))?;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
found_object_metadata = true;
|
||||
self.new_cache.info.failed_objects.remove(&item.path);
|
||||
|
||||
item.transform_meta_dir();
|
||||
|
||||
@@ -1581,7 +1604,9 @@ impl FolderScanner {
|
||||
object_count += 1;
|
||||
self.budget.record_object_scanned();
|
||||
|
||||
timer.sleep().await;
|
||||
ctx.run_until_cancelled(timer.sleep())
|
||||
.await
|
||||
.ok_or_else(|| ScannerError::Other("Operation cancelled".to_string()))?;
|
||||
|
||||
if ctx.is_cancelled() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
@@ -1622,9 +1647,9 @@ impl FolderScanner {
|
||||
if self.is_erasure_mode && found_erasure_data_directory && !found_object_metadata {
|
||||
found_object_metadata = true;
|
||||
let metadata_path = path_join_buf(&[&dir_path, STORAGE_FORMAT_FILE]);
|
||||
into.failed_objects = into.failed_objects.saturating_add(1);
|
||||
|
||||
if !self.should_skip_failed(&metadata_path) {
|
||||
into.failed_objects = into.failed_objects.saturating_add(1);
|
||||
self.record_failed(&metadata_path);
|
||||
|
||||
let failed_cache_entries = self.new_cache.info.failed_objects.len();
|
||||
@@ -1835,6 +1860,7 @@ impl FolderScanner {
|
||||
error = %e,
|
||||
"Scanner child folder scan failed"
|
||||
);
|
||||
self.preserve_failed_child(&folder_item.parent, &h, into, &dst).await;
|
||||
continue;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
@@ -2230,6 +2256,7 @@ impl FolderScanner {
|
||||
error = %e,
|
||||
"Scanner heal child folder scan failed"
|
||||
);
|
||||
self.preserve_failed_child(&folder_item.parent, &h, into, &dst).await;
|
||||
continue;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
@@ -2396,6 +2423,9 @@ pub async fn scan_data_folder(
|
||||
};
|
||||
|
||||
let now = FolderScanner::now_secs();
|
||||
// Prune once per bucket walk, not once per directory. Per-path TTL checks
|
||||
// still allow retries during long scans, and insertions enforce the cap.
|
||||
scanner.prune_failed_objects_cache();
|
||||
prune_size_reconciliation(&mut scanner.new_cache.info, now);
|
||||
prune_size_reconciliation(&mut scanner.update_cache.info, now);
|
||||
|
||||
@@ -2422,7 +2452,9 @@ pub async fn scan_data_folder(
|
||||
new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
|
||||
new_cache.info.last_update = Some(SystemTime::now());
|
||||
new_cache.info.next_cycle = cache.info.next_cycle;
|
||||
let unresolved_objects = root.failed_objects > 0
|
||||
let unresolved_objects = new_cache
|
||||
.size_recursive(&cache.info.name)
|
||||
.is_none_or(|root| root.failed_objects > 0)
|
||||
|| !new_cache.info.failed_objects.is_empty()
|
||||
|| !new_cache.info.size_reconciliation.is_empty();
|
||||
new_cache.info.snapshot_complete = !unresolved_objects;
|
||||
|
||||
@@ -1366,6 +1366,104 @@ mod tests {
|
||||
assert_eq!(item.object_path(), "object");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn scanner_blocked_expiry_preserves_usage_replication_and_integrity_work() {
|
||||
use s3s::dto::{LifecycleExpiration, LifecycleRule};
|
||||
|
||||
let lifecycle = Arc::new(BucketLifecycleConfiguration {
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: None,
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
let attempts = |report: &rustfs_scanner_metrics::metrics::ScannerMetricsReport, source: ScannerWorkSource| {
|
||||
report
|
||||
.source_work
|
||||
.iter()
|
||||
.filter(|work| work.source == source.as_str())
|
||||
.map(|work| work.queued + work.skipped + work.missed)
|
||||
.sum::<u64>()
|
||||
};
|
||||
for with_lifecycle in [false, true] {
|
||||
for scan_mode in [HealScanMode::Normal, HealScanMode::Deep] {
|
||||
for guard in ["pending", "failed", "legal_hold"] {
|
||||
let mut metadata = HashMap::new();
|
||||
let replication_status = match guard {
|
||||
"pending" => ReplicationStatusType::Pending,
|
||||
"failed" => ReplicationStatusType::Failed,
|
||||
_ => {
|
||||
metadata.insert("x-amz-object-lock-legal-hold".to_string(), "ON".to_string());
|
||||
ReplicationStatusType::Completed
|
||||
}
|
||||
};
|
||||
let object = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
num_versions: 1,
|
||||
is_latest: true,
|
||||
mod_time: Some(OffsetDateTime::now_utc() - time::Duration::days(90)),
|
||||
size: 4096,
|
||||
actual_size: 4096,
|
||||
replication_status,
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
let events = Evaluator::new(lifecycle.clone())
|
||||
.eval(&[crate::ecstore_object_opts_from_object_info(&object)])
|
||||
.await
|
||||
.expect("evaluate expiry guard");
|
||||
assert_eq!(events[0].action, IlmAction::NoneAction, "expiry must be blocked by {guard}");
|
||||
|
||||
let mut item = scanner_item_with_prefix("");
|
||||
item.object_name = "object".to_string();
|
||||
item.lifecycle = with_lifecycle.then(|| lifecycle.clone());
|
||||
item.replication = Some(Arc::new(ReplicationConfig::new(None, None)));
|
||||
item.heal_enabled = true;
|
||||
item.heal_bitrot = scan_mode == HealScanMode::Deep;
|
||||
let before = global_metrics().report().await;
|
||||
let mut summary = SizeSummary::default();
|
||||
item.apply_actions(vec![object], None, VersioningConfiguration::default(), &[], &mut summary)
|
||||
.await;
|
||||
let after = global_metrics().report().await;
|
||||
assert_eq!(summary.total_size, 4096, "blocked expiry must retain bytes for {guard}");
|
||||
assert_eq!(summary.versions, 1);
|
||||
assert_eq!(summary.delete_markers, 0);
|
||||
assert!(summary.size_reconciliation.is_empty());
|
||||
assert_eq!(
|
||||
attempts(&after, scanner_heal_source(scan_mode)) - attempts(&before, scanner_heal_source(scan_mode)),
|
||||
1,
|
||||
"integrity work must continue with lifecycle={with_lifecycle}, guard={guard}"
|
||||
);
|
||||
assert_eq!(
|
||||
attempts(&after, ScannerWorkSource::BucketReplication)
|
||||
- attempts(&before, ScannerWorkSource::BucketReplication),
|
||||
1,
|
||||
"replication inspection must continue with lifecycle={with_lifecycle}, guard={guard}"
|
||||
);
|
||||
assert_eq!(
|
||||
attempts(&after, ScannerWorkSource::Lifecycle) - attempts(&before, ScannerWorkSource::Lifecycle),
|
||||
0,
|
||||
"blocked expiry must not enqueue destructive lifecycle work"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_tier_never_triggers_transition() {
|
||||
let object = ObjectInfo {
|
||||
|
||||
@@ -24,8 +24,6 @@ use std::io::Write;
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
use std::sync::Mutex;
|
||||
|
||||
mod checkpoint_fixture;
|
||||
|
||||
/// Reset the process-global alert cooldown map; test-only.
|
||||
fn reset_alert_cooldowns() {
|
||||
*SCANNER_ALERT_EMISSION_COOLDOWN
|
||||
@@ -1885,6 +1883,323 @@ async fn test_scan_folder_skips_unreadable_child_directory() {
|
||||
assert!(result.is_ok(), "expected unreadable child directory to be skipped");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_failed_child_retains_usage_and_scans_healthy_sibling() {
|
||||
for with_prior in [false, true] {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(0, 0, &mut scanner, temp_dir.clone());
|
||||
let bad_dir = temp_dir.join("bucket/bad");
|
||||
tokio::fs::create_dir_all(&bad_dir).await.expect("create failing directory");
|
||||
write_test_object_metadata_bytes(
|
||||
&temp_dir,
|
||||
"bucket",
|
||||
"good",
|
||||
&metadata_for_object_version("bucket", "good", Some(Uuid::new_v4())),
|
||||
)
|
||||
.await;
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
let root_hash = hash_path("bucket");
|
||||
let bad_hash = hash_path("bucket/bad");
|
||||
let mut prior = DataUsageEntry {
|
||||
size: 4096,
|
||||
objects: 2,
|
||||
versions: 3,
|
||||
delete_markers: 1,
|
||||
..Default::default()
|
||||
};
|
||||
prior.replication_stats = Some(rustfs_data_usage::ReplicationAllStats {
|
||||
replica_size: 4096,
|
||||
replica_count: 2,
|
||||
..Default::default()
|
||||
});
|
||||
prior.add_tier_sizes(&HashMap::from([(
|
||||
"WARM".to_string(),
|
||||
TierStats {
|
||||
total_size: 4096,
|
||||
num_versions: 3,
|
||||
num_objects: 2,
|
||||
},
|
||||
)]));
|
||||
scanner
|
||||
.old_cache
|
||||
.replace_hashed(&root_hash, &None, &DataUsageEntry::default());
|
||||
if with_prior {
|
||||
scanner.old_cache.replace_hashed(&bad_hash, &Some(root_hash.clone()), &prior);
|
||||
} else {
|
||||
prior = DataUsageEntry::default();
|
||||
}
|
||||
scanner.update_current_path = Arc::new(move |path| {
|
||||
if path == "bucket/bad" {
|
||||
// Replace the directory after enumeration but before descent. This
|
||||
// injects a real read_dir error even when tests run as root.
|
||||
std::fs::remove_dir(&bad_dir).expect("remove enumerated directory");
|
||||
std::fs::write(&bad_dir, b"not a directory").expect("replace enumerated directory");
|
||||
}
|
||||
Box::pin(async {})
|
||||
});
|
||||
let mut root = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut root,
|
||||
)
|
||||
.await
|
||||
.expect("one failed directory must not stop healthy siblings");
|
||||
let total = scanner.new_cache.size_recursive(&root_hash.key()).expect("root usage");
|
||||
assert_eq!(total.size, prior.size + 1, "unreadable child must retain its previous bytes");
|
||||
assert_eq!(total.objects, prior.objects + 1, "healthy sibling must still be counted");
|
||||
assert_eq!(total.versions, prior.versions + 1);
|
||||
assert_eq!(total.delete_markers, prior.delete_markers);
|
||||
assert_eq!(total.failed_objects, 1, "walk error must keep the snapshot incomplete");
|
||||
assert_eq!(
|
||||
serde_json::to_value(&total.replication_stats).expect("serialize replication usage"),
|
||||
serde_json::to_value(&prior.replication_stats).expect("serialize prior replication usage")
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&total.all_tier_stats).expect("serialize tier usage"),
|
||||
serde_json::to_value(&prior.all_tier_stats).expect("serialize prior tier usage")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_nested_metadata_failure_without_retry_cache_is_partial_then_recovers() {
|
||||
let (scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(temp_dir.clone()),
|
||||
};
|
||||
write_test_object_metadata_bytes(&temp_dir, "bucket", "prefix/bad", b"").await;
|
||||
write_test_object_metadata_bytes(
|
||||
&temp_dir,
|
||||
"bucket",
|
||||
"prefix/good",
|
||||
&metadata_for_object_version("bucket", "prefix/good", Some(Uuid::new_v4())),
|
||||
)
|
||||
.await;
|
||||
temp_env::async_with_vars([(ENV_FAILED_OBJECT_TTL_SECS, Some("0"))], async {
|
||||
for inherited_failure in [false, true] {
|
||||
write_test_object_metadata_bytes(&temp_dir, "bucket", "prefix/bad", b"").await;
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: u64::from(
|
||||
(0..DATA_USAGE_UPDATE_DIR_CYCLES)
|
||||
.find(|cycle| !hash_path("bucket/prefix").mod_(*cycle, DATA_USAGE_UPDATE_DIR_CYCLES))
|
||||
.expect("cycle outside the prefix compaction sample"),
|
||||
),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
if inherited_failure {
|
||||
cache
|
||||
.info
|
||||
.failed_objects
|
||||
.insert("removed-object/xl.meta".to_string(), FolderScanner::now_secs());
|
||||
}
|
||||
let budget = ScannerCycleBudget::new(&CancellationToken::new(), Default::default());
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![scanner.local_disk.clone()],
|
||||
scanner.local_disk.clone(),
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest),
|
||||
)
|
||||
.await;
|
||||
let mut partial = match result {
|
||||
Err(ScannerError::PartialCache(cache)) => *cache,
|
||||
other => panic!("nested failure must never publish a complete snapshot: {other:?}"),
|
||||
};
|
||||
assert!(!partial.info.snapshot_complete);
|
||||
assert!(partial.info.failed_objects.is_empty(), "TTL zero disables only the retry cache");
|
||||
let total = partial.size_recursive("bucket").expect("partial root");
|
||||
assert_eq!(total.objects, 1);
|
||||
assert_eq!(total.failed_objects, 1);
|
||||
|
||||
// Reusing the partial compacted subtree must remain partial even
|
||||
// without a retry ledger. Recovery happens on its next selected cycle.
|
||||
let budget = ScannerCycleBudget::new(&CancellationToken::new(), Default::default());
|
||||
let reused = scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![scanner.local_disk.clone()],
|
||||
scanner.local_disk.clone(),
|
||||
partial.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(reused, Err(ScannerError::PartialCache(_))));
|
||||
partial.info.next_cycle = u64::from(
|
||||
(0..DATA_USAGE_UPDATE_DIR_CYCLES)
|
||||
.find(|cycle| hash_path("bucket/prefix").mod_(*cycle, DATA_USAGE_UPDATE_DIR_CYCLES))
|
||||
.expect("next selected directory cycle"),
|
||||
);
|
||||
write_test_object_metadata_bytes(
|
||||
&temp_dir,
|
||||
"bucket",
|
||||
"prefix/bad",
|
||||
&metadata_for_object_version("bucket", "prefix/bad", Some(Uuid::new_v4())),
|
||||
)
|
||||
.await;
|
||||
let budget = ScannerCycleBudget::new(&CancellationToken::new(), Default::default());
|
||||
let recovered = scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![scanner.local_disk.clone()],
|
||||
scanner.local_disk.clone(),
|
||||
partial,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest),
|
||||
)
|
||||
.await
|
||||
.expect("repaired subtree must converge on its next selected cycle");
|
||||
assert!(recovered.info.snapshot_complete);
|
||||
let total = recovered.size_recursive("bucket").expect("recovered root");
|
||||
assert_eq!(total.objects, 2);
|
||||
assert_eq!(total.size, 2);
|
||||
assert_eq!(total.versions, 2);
|
||||
assert_eq!(total.failed_objects, 0);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_compacted_directory_keeps_aggressive_heal_and_bitrot_sampling() {
|
||||
for scan_mode in [HealScanMode::Normal, HealScanMode::Deep] {
|
||||
for select_prob in [0, 1, 8, 16] {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(0, 0, &mut scanner, temp_dir.clone());
|
||||
write_test_object_metadata_bytes(
|
||||
&temp_dir,
|
||||
"bucket",
|
||||
"object",
|
||||
&metadata_for_object_version("bucket", "object", Some(Uuid::new_v4())),
|
||||
)
|
||||
.await;
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.is_erasure_mode = true;
|
||||
scanner.heal_object_select = select_prob;
|
||||
scanner.scan_mode = scan_mode;
|
||||
let root_hash = hash_path("bucket");
|
||||
let object_hash = hash_path("bucket/object");
|
||||
scanner.old_cache.info.next_cycle = u64::from(
|
||||
(0..DATA_USAGE_UPDATE_DIR_CYCLES)
|
||||
.find(|cycle| object_hash.mod_(*cycle, DATA_USAGE_UPDATE_DIR_CYCLES))
|
||||
.expect("selected directory cycle"),
|
||||
);
|
||||
scanner
|
||||
.old_cache
|
||||
.replace_hashed(&root_hash, &None, &DataUsageEntry::default());
|
||||
scanner.old_cache.replace_hashed(
|
||||
&object_hash,
|
||||
&Some(root_hash),
|
||||
&DataUsageEntry {
|
||||
compacted: true,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let attempts = |report: rustfs_scanner_metrics::metrics::ScannerMetricsReport| {
|
||||
report
|
||||
.source_work
|
||||
.iter()
|
||||
.filter(|work| work.source == scanner_heal_source(scan_mode).as_str())
|
||||
.map(|work| work.queued + work.skipped + work.missed)
|
||||
.sum::<u64>()
|
||||
};
|
||||
temp_env::async_with_vars([(ENV_SCANNER_DEEP_VERIFY_COOLDOWN_SECS, Some("0"))], async {
|
||||
let before = attempts(global_metrics().report().await);
|
||||
let mut root = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut root,
|
||||
)
|
||||
.await
|
||||
.expect("scan selected compacted object");
|
||||
assert_eq!(
|
||||
attempts(global_metrics().report().await) - before,
|
||||
u64::from(select_prob != 0),
|
||||
"selected compacted object must reach {scan_mode:?} admission with divisor {select_prob}"
|
||||
);
|
||||
let total = scanner.new_cache.size_recursive("bucket").expect("usage root");
|
||||
assert_eq!(total.objects, 1);
|
||||
assert_eq!(total.versions, 1);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn scanner_cancellation_interrupts_folder_throttle() {
|
||||
use futures::{FutureExt, poll};
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(0, 0, &mut scanner, temp_dir);
|
||||
scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Slowest);
|
||||
let previous_idle = crate::sleeper::SCANNER_IDLE_MODE.swap(true, std::sync::atomic::Ordering::Relaxed);
|
||||
let ctx = CancellationToken::new();
|
||||
let mut root = DataUsageEntry::default();
|
||||
let mut scan = scanner
|
||||
.scan_folder(
|
||||
ctx.clone(),
|
||||
CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut root,
|
||||
)
|
||||
.boxed();
|
||||
assert!(poll!(scan.as_mut()).is_pending(), "scan should be waiting in its folder throttle");
|
||||
ctx.cancel();
|
||||
let outcome = scan.now_or_never();
|
||||
crate::sleeper::SCANNER_IDLE_MODE.store(previous_idle, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(
|
||||
matches!(outcome, Some(Err(_))),
|
||||
"cancellation must finish without advancing the sleep clock"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_zero_directory_cycle_keeps_rescanning_enabled() {
|
||||
temp_env::with_var(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, Some("0"), || {
|
||||
for cycle in 0..32 {
|
||||
assert!(
|
||||
hash_path("bucket/object").mod_(cycle, data_usage_update_dir_cycles()),
|
||||
"zero must not leave compacted usage stale forever"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
|
||||
@@ -2155,7 +2470,7 @@ async fn test_scan_folder_corrupt_xl_meta_stops_erasure_data_dir_descent() {
|
||||
.await
|
||||
.expect("cached metadata failure must still stop erasure data directory descent");
|
||||
|
||||
assert_eq!(retry_into.failed_objects, 0, "cached failure should not be counted twice");
|
||||
assert_eq!(retry_into.failed_objects, 1, "cached failure must remain visible in each snapshot");
|
||||
assert!(!retry_budget.budget_elapsed());
|
||||
assert_eq!(retry_budget.reason(), None);
|
||||
|
||||
@@ -2280,7 +2595,7 @@ async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() {
|
||||
.await
|
||||
.expect("cached missing metadata must still stop erasure data directory descent");
|
||||
|
||||
assert_eq!(retry_into.failed_objects, 0, "cached failure should not be counted twice");
|
||||
assert_eq!(retry_into.failed_objects, 1, "cached failure must remain visible in each snapshot");
|
||||
assert!(!retry_budget.budget_elapsed());
|
||||
assert_eq!(retry_budget.reason(), None);
|
||||
}
|
||||
|
||||
@@ -1,410 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||
use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk};
|
||||
use crate::storage_api::scanner_io::ObjectIO;
|
||||
use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin";
|
||||
const STATIC_OBJECTS: u64 = 24;
|
||||
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
|
||||
const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0);
|
||||
const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]);
|
||||
|
||||
/// Real cache persistence codec and CAS calls, backed by two bounded local files.
|
||||
#[derive(Debug)]
|
||||
struct FixtureStore {
|
||||
root: tempfile::TempDir,
|
||||
reject_save: AtomicBool,
|
||||
}
|
||||
|
||||
impl FixtureStore {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
root: tempfile::tempdir().expect("checkpoint fixture storage directory"),
|
||||
reject_save: AtomicBool::new(false),
|
||||
})
|
||||
}
|
||||
|
||||
fn path(&self, object: &str) -> std::path::PathBuf {
|
||||
assert!(object.ends_with(CACHE_NAME) || object.ends_with(&format!("{CACHE_NAME}.bkp")));
|
||||
self.root
|
||||
.path()
|
||||
.join(if object.ends_with(".bkp") { "backup" } else { "main" })
|
||||
}
|
||||
|
||||
async fn strict_load(&self) -> DataUsageCache {
|
||||
let bytes = tokio::fs::read(self.root.path().join("main"))
|
||||
.await
|
||||
.expect("saved checkpoint fixture must exist");
|
||||
decode_fixture(&bytes).expect("saved checkpoint fixture must contain a valid bucket root")
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectIO for FixtureStore {
|
||||
type Error = crate::EcstoreError;
|
||||
type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec;
|
||||
type HeaderMap = http::HeaderMap;
|
||||
type ObjectOptions = crate::ScannerObjectOptions;
|
||||
type ObjectInfo = crate::ScannerObjectInfo;
|
||||
type GetObjectReader = crate::ScannerGetObjectReader;
|
||||
type PutObjectReader = crate::ScannerPutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
object: &str,
|
||||
_range: Option<Self::RangeSpec>,
|
||||
_headers: Self::HeaderMap,
|
||||
_options: &Self::ObjectOptions,
|
||||
) -> crate::EcstoreResult<Self::GetObjectReader> {
|
||||
let bytes = tokio::fs::read(self.path(object)).await.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
crate::EcstoreError::FileNotFound
|
||||
} else {
|
||||
crate::EcstoreError::from(error)
|
||||
}
|
||||
})?;
|
||||
assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
|
||||
Ok(crate::ScannerGetObjectReader {
|
||||
stream: Box::new(Cursor::new(bytes)),
|
||||
object_info: crate::ScannerObjectInfo {
|
||||
etag: Some("fixture".into()),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
object: &str,
|
||||
data: &mut Self::PutObjectReader,
|
||||
options: &Self::ObjectOptions,
|
||||
) -> crate::EcstoreResult<Self::ObjectInfo> {
|
||||
if self.reject_save.load(Ordering::SeqCst) {
|
||||
return Err(crate::EcstoreError::PreconditionFailed);
|
||||
}
|
||||
let path = self.path(object);
|
||||
let exists = tokio::fs::try_exists(&path).await?;
|
||||
let preconditions = options.http_preconditions.as_ref().expect("checkpoint writes must use CAS");
|
||||
if (exists && preconditions.if_none_match_value() == Some("*"))
|
||||
|| (!exists && preconditions.if_match_value().is_some())
|
||||
|| (exists && preconditions.if_match_value() != Some("fixture"))
|
||||
{
|
||||
return Err(crate::EcstoreError::PreconditionFailed);
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
(&mut data.stream).take(MAX_CACHE_BYTES + 1).read_to_end(&mut bytes).await?;
|
||||
assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
|
||||
tokio::fs::write(path, bytes).await?;
|
||||
Ok(crate::ScannerObjectInfo {
|
||||
etag: Some("fixture".into()),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::ScannerConfigObjectDelete for FixtureStore {
|
||||
async fn delete_config_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_options: crate::ScannerObjectOptions,
|
||||
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
|
||||
Err(crate::EcstoreError::NotImplemented)
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
|
||||
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_fixture(bytes: &[u8]) -> Result<DataUsageCache, &'static str> {
|
||||
if bytes.is_empty() || bytes.len() > usize::try_from(MAX_CACHE_BYTES).expect("fixture bound") {
|
||||
return Err("missing or oversized checkpoint fixture");
|
||||
}
|
||||
let cache = DataUsageCache::unmarshal(bytes).map_err(|_| "corrupt checkpoint fixture")?;
|
||||
if cache.info.name != "bucket" || cache.checked_flatten("bucket").is_none() {
|
||||
return Err("checkpoint fixture has no valid bucket root");
|
||||
}
|
||||
Ok(cache)
|
||||
}
|
||||
|
||||
fn retained(cache: &DataUsageCache) -> u64 {
|
||||
assert!(
|
||||
!cache.root().is_some_and(|root| root.compacted),
|
||||
"a compacted bucket root cannot prove static-prefix coverage"
|
||||
);
|
||||
cache
|
||||
.checked_flatten("bucket/static")
|
||||
.map_or(0, |entry| u64::try_from(entry.objects).expect("fixture object count fits u64"))
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum CoverageDiagnosis {
|
||||
Progress,
|
||||
NoNewWork,
|
||||
LostAtPrepare,
|
||||
LostAtReload,
|
||||
WalkWithoutRetention,
|
||||
}
|
||||
|
||||
fn diagnose(previous: u64, prepared: u64, walked: u64, scanned: u64, reloaded: u64) -> CoverageDiagnosis {
|
||||
if reloaded < scanned {
|
||||
CoverageDiagnosis::LostAtReload
|
||||
} else if prepared < previous {
|
||||
CoverageDiagnosis::LostAtPrepare
|
||||
} else if walked > 0 && reloaded <= previous {
|
||||
CoverageDiagnosis::WalkWithoutRetention
|
||||
} else if reloaded > previous {
|
||||
CoverageDiagnosis::Progress
|
||||
} else {
|
||||
CoverageDiagnosis::NoNewWork
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_diagnosis_rejects_walk_without_retention() {
|
||||
assert_eq!(diagnose(4, 4, 9, 8, 8), CoverageDiagnosis::Progress);
|
||||
assert_eq!(diagnose(4, 4, 9, 4, 4), CoverageDiagnosis::WalkWithoutRetention);
|
||||
assert_eq!(diagnose(4, 0, 9, 4, 4), CoverageDiagnosis::LostAtPrepare);
|
||||
assert_eq!(diagnose(4, 4, 9, 8, 4), CoverageDiagnosis::LostAtReload);
|
||||
assert_eq!(diagnose(4, 4, 0, 4, 4), CoverageDiagnosis::NoNewWork);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_missing_and_corrupt_inputs_fail() {
|
||||
for bytes in [
|
||||
vec![],
|
||||
vec![0xc1],
|
||||
DataUsageCache::default().marshal_msg().expect("empty cache encoding"),
|
||||
vec![0; usize::try_from(MAX_CACHE_BYTES + 1).expect("oversized fixture")],
|
||||
] {
|
||||
assert!(decode_fixture(&bytes).is_err(), "invalid fixture must not become an empty complete root");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_compaction_preserves_aggregate_not_child_enumeration() {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "bucket".to_string();
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
cache.replace("bucket/static", "bucket", DataUsageEntry::default());
|
||||
for index in 0..4 {
|
||||
cache.replace(
|
||||
&format!("bucket/static/{index}"),
|
||||
"bucket/static",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
cache.reduce_children_of(&hash_path("bucket/static"), 1, true);
|
||||
let decoded = decode_fixture(&cache.marshal_msg().expect("encode compacted cache")).expect("decode compacted fixture");
|
||||
let entry = decoded
|
||||
.find("bucket/static")
|
||||
.expect("compaction must retain the static subtree root");
|
||||
assert!(entry.compacted);
|
||||
assert!(entry.children.is_empty());
|
||||
assert_eq!(
|
||||
retained(&decoded),
|
||||
4,
|
||||
"compaction retains aggregate coverage even when leaf keys are absent"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_fixture_save_reload_resume() {
|
||||
run_checkpoint_fixture(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_fixture_hot_digest_diagnostic() {
|
||||
run_checkpoint_fixture(true).await;
|
||||
}
|
||||
|
||||
async fn run_checkpoint_fixture(change_digest: bool) {
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
for index in 0..STATIC_OBJECTS {
|
||||
write_test_object_metadata(&root, "bucket", &format!("static/{index:04}")).await;
|
||||
}
|
||||
let store = FixtureStore::new();
|
||||
let mut previous = 0;
|
||||
let mut visited = 0;
|
||||
for round in 0..3_u8 {
|
||||
write_test_object_metadata(&root, "bucket", "hot/current").await;
|
||||
let mut cache = DataUsageCache::default();
|
||||
let revisions = cache
|
||||
.load_with_revisions(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("load checkpoint revisions");
|
||||
if round > 0 {
|
||||
assert_eq!(retained(&store.strict_load().await), previous);
|
||||
}
|
||||
let plan = crate::scanner_io::checkpoint_fixture_bucket_digest(PLAN, change_digest.then_some(u64::from(round)));
|
||||
crate::scanner_io::current_cache_root_or_prepare_with_generation(
|
||||
&mut cache,
|
||||
"bucket",
|
||||
SOURCE,
|
||||
11,
|
||||
7,
|
||||
plan,
|
||||
crate::scanner_io::DataUsageCacheReuseOptions {
|
||||
require_source: true,
|
||||
tier_registry_generation: None,
|
||||
},
|
||||
);
|
||||
let prepared = retained(&cache);
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(4),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let outcome = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![scanner.local_disk.clone()],
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
)
|
||||
.await
|
||||
.expect("budgeted local disk scan returns partial cache");
|
||||
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
|
||||
panic!("budgeted fixture must remain partial")
|
||||
};
|
||||
assert!(!cache.info.snapshot_complete, "partial must never publish a complete root");
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
|
||||
let scanned = retained(&cache);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist partial checkpoint");
|
||||
let mut loaded = DataUsageCache::default();
|
||||
loaded
|
||||
.load(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("reload persisted partial checkpoint");
|
||||
let reloaded = retained(&loaded);
|
||||
assert_eq!(reloaded, retained(&store.strict_load().await));
|
||||
assert_eq!(scanned, reloaded, "save/load must retain static subtree coverage");
|
||||
assert!(!loaded.info.snapshot_complete);
|
||||
visited += budget.entries_visited();
|
||||
let diagnosis = diagnose(previous, prepared, budget.entries_visited(), scanned, reloaded);
|
||||
eprintln!(
|
||||
"checkpoint_fixture round={round} hot_digest={change_digest} visited_total={visited} before={previous} prepared={prepared} scanned={scanned} reloaded={reloaded} diagnosis={diagnosis:?}"
|
||||
);
|
||||
if !change_digest || std::env::var_os("RUSTFS_CHECKPOINT_REQUIRE_PROGRESS").is_some() {
|
||||
assert_eq!(
|
||||
diagnosis,
|
||||
CoverageDiagnosis::Progress,
|
||||
"visited growth must produce durable static coverage"
|
||||
);
|
||||
}
|
||||
crate::remote_scanner::checkpoint_fixture_partial_return(budget.progress(), budget.entries_visited()).await;
|
||||
previous = reloaded;
|
||||
}
|
||||
assert!(visited > 0, "fixture must exercise the directory walk");
|
||||
assert!(previous > 0, "fixture must retain and enumerate static subtree entries");
|
||||
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let revisions = loaded
|
||||
.load_with_revisions(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("load final checkpoint");
|
||||
let before = tokio::fs::read(store.root.path().join("main"))
|
||||
.await
|
||||
.expect("read durable checkpoint bytes");
|
||||
let epoch_error = loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1)
|
||||
.await
|
||||
.expect_err("stale publication epoch must reject persistence");
|
||||
assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
store.reject_save.store(true, Ordering::SeqCst);
|
||||
loaded.info.next_cycle += 1;
|
||||
loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect_err("injected save failure must not report durable progress");
|
||||
assert_eq!(
|
||||
tokio::fs::read(store.root.path().join("main"))
|
||||
.await
|
||||
.expect("read unchanged checkpoint bytes"),
|
||||
before
|
||||
);
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
parent.cancel();
|
||||
let budget = ScannerCycleBudget::new(&parent, Default::default());
|
||||
let result = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![scanner.local_disk.clone()],
|
||||
loaded.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "pre-scan cancellation must not produce a complete root");
|
||||
assert_eq!(budget.reason(), None, "parent cancellation is not object budget exhaustion");
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&parent, Default::default());
|
||||
let result = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![scanner.local_disk.clone()],
|
||||
loaded,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
)
|
||||
.await
|
||||
.expect("unbounded scan must complete after durable partial progress");
|
||||
let ScannerDiskScanOutcome::Complete(cache) = result else {
|
||||
panic!("unbounded fixture must produce a complete disk cache");
|
||||
};
|
||||
assert!(cache.info.snapshot_complete);
|
||||
assert!(cache.info.scan_checkpoint.is_none());
|
||||
assert_eq!(
|
||||
cache.checked_flatten("bucket").expect("complete bucket root").objects,
|
||||
usize::try_from(STATIC_OBJECTS + 1).expect("fixture object count fits usize")
|
||||
);
|
||||
}
|
||||
@@ -21,7 +21,6 @@ use crate::{
|
||||
DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState,
|
||||
ScannerError, SizeSummary, TierStats,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
use rand::seq::SliceRandom as _;
|
||||
@@ -55,7 +54,6 @@ use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::ScannerObjectInfo as ObjectInfo;
|
||||
use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
|
||||
use crate::storage_api::ScannerStorage;
|
||||
use crate::storage_api::scan::NamespaceLocking as _;
|
||||
use crate::storage_api::scanner_io::{BucketInfo, BucketOptions};
|
||||
@@ -113,121 +111,6 @@ pub(crate) struct ScannerBucketScanScope {
|
||||
baseline_scan_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
}
|
||||
|
||||
impl ScannerBucketScanScope {
|
||||
fn is_default(&self) -> bool {
|
||||
self.selected_buckets.is_none() && self.baseline_scan_plan_digest.is_none()
|
||||
}
|
||||
|
||||
fn from_dirty_buckets(selected_buckets: HashSet<String>, baseline_scan_plan_digest: DataUsageScanPlanDigest) -> Self {
|
||||
Self {
|
||||
selected_buckets: Some(Arc::new(selected_buckets)),
|
||||
baseline_scan_plan_digest: Some(baseline_scan_plan_digest),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct ScannerCacheBaselineProof<'a> {
|
||||
pub(super) data: Option<&'a Bytes>,
|
||||
pub(super) expected_sources: &'a HashSet<DataUsageCacheSource>,
|
||||
pub(super) leader_epoch: u64,
|
||||
pub(super) want_cycle: u64,
|
||||
pub(super) scan_plan_digest: DataUsageScanPlanDigest,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: String,
|
||||
generation: u64,
|
||||
pending: bool,
|
||||
}
|
||||
|
||||
fn verified_remote_dirty_usage_buckets(
|
||||
expected_peers: &HashMap<String, ScannerPeerDirtyUsageExpectation>,
|
||||
peer_snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>,
|
||||
) -> Option<HashSet<String>> {
|
||||
if expected_peers.is_empty() || peer_snapshots.len() != expected_peers.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut received_peers = HashSet::with_capacity(peer_snapshots.len());
|
||||
let mut dirty_buckets = HashSet::new();
|
||||
for (host, snapshot) in peer_snapshots {
|
||||
let expected = expected_peers.get(&host)?;
|
||||
if !received_peers.insert(host)
|
||||
|| snapshot.instance_id != expected.instance_id
|
||||
|| snapshot.generation != expected.generation
|
||||
|| snapshot.generation == u64::MAX
|
||||
|| snapshot.protocol_version != crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION
|
||||
|| !snapshot.complete
|
||||
|| snapshot.pending_bucket_count != u64::try_from(snapshot.buckets.len()).unwrap_or(u64::MAX)
|
||||
|| (expected.pending && snapshot.pending_bucket_count == 0)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
dirty_buckets.extend(snapshot.buckets.into_keys());
|
||||
}
|
||||
|
||||
(received_peers.len() == expected_peers.len()).then_some(dirty_buckets)
|
||||
}
|
||||
|
||||
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
|
||||
let data = proof.data?;
|
||||
let baseline = serde_json::from_slice::<DataUsageInfo>(data).ok()?;
|
||||
if !baseline.is_complete_bucket_usage_snapshot()
|
||||
|| baseline.usage_snapshot_partial
|
||||
|| baseline.usage_snapshot_converged != Some(true)
|
||||
|| baseline.scanner_epoch != Some(proof.leader_epoch)
|
||||
|| baseline.usage_snapshot_set_states.len() != proof.expected_sources.len()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut states = HashSet::with_capacity(baseline.usage_snapshot_set_states.len());
|
||||
for state in &baseline.usage_snapshot_set_states {
|
||||
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
|
||||
if !proof.expected_sources.contains(&source)
|
||||
|| !states.insert(source)
|
||||
|| !state.complete
|
||||
|| state.tombstone
|
||||
|| state.scanner_epoch != Some(proof.leader_epoch)
|
||||
|| state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle)
|
||||
|| state.scan_plan_digest != Some(proof.scan_plan_digest.0)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
(states == *proof.expected_sources).then_some(proof.scan_plan_digest)
|
||||
}
|
||||
|
||||
fn scoped_scan_scope_from_dirty_buckets(
|
||||
requested_scope: ScannerBucketScanScope,
|
||||
dirty_buckets: HashSet<String>,
|
||||
dirty_snapshot_complete: bool,
|
||||
all_buckets: &[BucketInfo],
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
) -> ScannerBucketScanScope {
|
||||
if !requested_scope.is_default() || !dirty_snapshot_complete {
|
||||
return requested_scope;
|
||||
}
|
||||
|
||||
let current_buckets = all_buckets.iter().map(|bucket| bucket.name.as_str()).collect::<HashSet<_>>();
|
||||
let selected_buckets = dirty_buckets
|
||||
.into_iter()
|
||||
.filter(|bucket| current_buckets.contains(bucket.as_str()))
|
||||
.collect::<HashSet<_>>();
|
||||
if selected_buckets.is_empty() {
|
||||
return requested_scope;
|
||||
}
|
||||
|
||||
let Some(baseline_scan_plan_digest) = complete_scanner_cache_baseline_plan_digest(baseline_proof) else {
|
||||
return requested_scope;
|
||||
};
|
||||
|
||||
ScannerBucketScanScope::from_dirty_buckets(selected_buckets, baseline_scan_plan_digest)
|
||||
}
|
||||
|
||||
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
|
||||
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
|
||||
}
|
||||
@@ -326,14 +209,6 @@ fn scanner_bucket_cache_digest(
|
||||
DataUsageScanPlanDigest(hasher.finalize().into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn checkpoint_fixture_bucket_digest(
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
dirty_generation: Option<u64>,
|
||||
) -> DataUsageScanPlanDigest {
|
||||
scanner_bucket_cache_digest(scan_plan_digest, dirty_generation)
|
||||
}
|
||||
|
||||
fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error>) -> Result<()> {
|
||||
if results.iter().any(|result| result.info.last_update.is_some()) {
|
||||
return Ok(());
|
||||
@@ -866,7 +741,7 @@ mod io_cache;
|
||||
mod io_cycle;
|
||||
#[cfg(test)]
|
||||
use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan};
|
||||
pub(crate) use io_cycle::{ScannerCycleRequest, nsscanner_with_storage_status_scoped};
|
||||
pub(crate) use io_cycle::nsscanner_with_storage_status;
|
||||
mod io_disk;
|
||||
#[cfg(test)]
|
||||
mod publish_gate_tests;
|
||||
@@ -883,9 +758,8 @@ pub(crate) use cache::{
|
||||
};
|
||||
pub use dirty_usage::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change,
|
||||
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
||||
|
||||
@@ -282,26 +282,9 @@ pub(super) fn completed_data_usage_info(
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect();
|
||||
let mut usage_snapshot_set_states = results
|
||||
.iter()
|
||||
.map(|result| {
|
||||
let source = result.info.source?;
|
||||
Some(DataUsageSnapshotSetState {
|
||||
pool_index: u64::try_from(source.pool_index).ok()?,
|
||||
set_index: u64::try_from(source.set_index).ok()?,
|
||||
scanner_cycle: Some(result.info.next_cycle),
|
||||
scanner_epoch: Some(result.info.leader_epoch),
|
||||
scan_plan_digest: Some(result.info.scan_plan_digest?.0),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
})
|
||||
})
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
usage_snapshot_set_states.sort_by_key(|state| (state.pool_index, state.set_index));
|
||||
let data_usage_info = DataUsageInfo {
|
||||
last_update: Some(merged_last_update),
|
||||
scanner_cycle: Some(results.first()?.info.next_cycle),
|
||||
scanner_epoch: Some(results.first()?.info.leader_epoch),
|
||||
objects_total_count: u64::try_from(total.objects).ok()?,
|
||||
versions_total_count: u64::try_from(total.versions).ok()?,
|
||||
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
|
||||
@@ -312,7 +295,6 @@ pub(super) fn completed_data_usage_info(
|
||||
bucket_sizes,
|
||||
buckets_usage,
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_set_states,
|
||||
..Default::default()
|
||||
};
|
||||
Some((data_usage_info, merged_last_update))
|
||||
|
||||
@@ -52,112 +52,6 @@ pub enum ScannerDirtyUsageAckError {
|
||||
ProcessChanged,
|
||||
#[error("scanner dirty usage generation cannot be acknowledged")]
|
||||
InvalidGeneration,
|
||||
#[error("scanner dirty usage bucket incarnation fence is unavailable")]
|
||||
IncarnationUnavailable,
|
||||
}
|
||||
|
||||
/// A scoped ACK requires storage-owned lifecycle and incarnation fences.
|
||||
/// Callers must only send ACKs backed by durable per-bucket publication.
|
||||
pub fn acknowledge_scoped_dirty_usage(
|
||||
instance_id: &str,
|
||||
entries: &[(&crate::storage_api::EcstoreBucketMetadataMutationGuard, u64)],
|
||||
probe_only: bool,
|
||||
) -> std::result::Result<u64, ScannerDirtyUsageAckError> {
|
||||
// Lock order: sorted bucket lifecycle/metadata fences (caller), then dirty map.
|
||||
// No await or storage operation occurs while the dirty map is locked.
|
||||
let (cleared, pending) = {
|
||||
let mut dirty = dirty_usage_buckets();
|
||||
let checked = entries
|
||||
.iter()
|
||||
.map(|(guard, generation)| {
|
||||
guard
|
||||
.checked_bucket_incarnation()
|
||||
.map(|(bucket, _)| (bucket, *generation))
|
||||
.map_err(|_| ScannerDirtyUsageAckError::IncarnationUnavailable)
|
||||
})
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?;
|
||||
let cleared = apply_scoped_dirty_usage_ack(
|
||||
instance_id,
|
||||
scanner_activity_epoch(),
|
||||
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire),
|
||||
&mut dirty,
|
||||
&checked,
|
||||
probe_only,
|
||||
)?;
|
||||
if cleared > 0 {
|
||||
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
|
||||
}
|
||||
(cleared, dirty.len())
|
||||
};
|
||||
if !probe_only {
|
||||
global_metrics().record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared), usize_to_u64_saturated(pending));
|
||||
}
|
||||
Ok(usize_to_u64_saturated(cleared))
|
||||
}
|
||||
|
||||
fn apply_scoped_dirty_usage_ack(
|
||||
instance_id: &str,
|
||||
current_instance: &str,
|
||||
current_generation: u64,
|
||||
dirty: &mut DirtyUsageBuckets,
|
||||
entries: &[(&str, u64)],
|
||||
probe_only: bool,
|
||||
) -> std::result::Result<usize, ScannerDirtyUsageAckError> {
|
||||
if instance_id != current_instance {
|
||||
return Err(ScannerDirtyUsageAckError::ProcessChanged);
|
||||
}
|
||||
if current_generation == u64::MAX
|
||||
|| entries
|
||||
.iter()
|
||||
.any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation)
|
||||
{
|
||||
return Err(ScannerDirtyUsageAckError::InvalidGeneration);
|
||||
}
|
||||
let mut cleared = 0;
|
||||
if !probe_only {
|
||||
for (bucket, generation) in entries {
|
||||
if dirty.get(*bucket) == Some(generation) {
|
||||
dirty.remove(*bucket);
|
||||
cleared += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(cleared)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod scoped_dirty_usage_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
|
||||
let mut dirty = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], true), Ok(0));
|
||||
assert_eq!(dirty.len(), 2);
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(1));
|
||||
assert_eq!(dirty.get("hot"), Some(&7));
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(0));
|
||||
dirty.insert("cold".to_string(), 9);
|
||||
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 9, &mut dirty, &[("cold", 8)], false), Ok(0));
|
||||
assert_eq!(dirty.get("cold"), Some(&9));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_rejects_restart_and_invalid_batch_before_clearing() {
|
||||
let original = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
|
||||
let mut dirty = original.clone();
|
||||
assert_eq!(
|
||||
apply_scoped_dirty_usage_ack("old", "new", 8, &mut dirty, &[("cold", 8)], false),
|
||||
Err(ScannerDirtyUsageAckError::ProcessChanged)
|
||||
);
|
||||
for generation in [0, 9, u64::MAX] {
|
||||
assert_eq!(
|
||||
apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8), ("hot", generation)], false),
|
||||
Err(ScannerDirtyUsageAckError::InvalidGeneration)
|
||||
);
|
||||
assert_eq!(dirty, original);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
||||
|
||||
@@ -71,7 +71,6 @@ where
|
||||
leader_epoch,
|
||||
scan_mode,
|
||||
scan_scope: ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline: None,
|
||||
};
|
||||
nsscanner_with_storage_status_scoped(store, request).await
|
||||
}
|
||||
@@ -84,79 +83,6 @@ pub(crate) struct ScannerCycleRequest {
|
||||
pub(crate) leader_epoch: u64,
|
||||
pub(crate) scan_mode: HealScanMode,
|
||||
pub(crate) scan_scope: ScannerBucketScanScope,
|
||||
pub(crate) persisted_usage_baseline: Option<Bytes>,
|
||||
}
|
||||
|
||||
struct ScannerBucketScopeResolution<'a> {
|
||||
requested_scope: ScannerBucketScanScope,
|
||||
baseline_proof: ScannerCacheBaselineProof<'a>,
|
||||
activity_before: &'a crate::scanner::ScannerActivitySnapshot,
|
||||
dirty_usage_snapshot: &'a DirtyUsageSnapshot,
|
||||
all_buckets: &'a [BucketInfo],
|
||||
}
|
||||
|
||||
async fn resolve_scanner_bucket_scan_scope<S>(
|
||||
store: &S,
|
||||
distributed: bool,
|
||||
resolution: ScannerBucketScopeResolution<'_>,
|
||||
) -> ScannerBucketScanScope
|
||||
where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
if !resolution.requested_scope.is_default()
|
||||
|| !resolution.dirty_usage_snapshot.covers_all_pending
|
||||
|| resolution.dirty_usage_snapshot.generation == u64::MAX
|
||||
|| resolution.dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES
|
||||
{
|
||||
return resolution.requested_scope;
|
||||
}
|
||||
|
||||
let mut dirty_buckets = resolution
|
||||
.dirty_usage_snapshot
|
||||
.buckets
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<HashSet<_>>();
|
||||
if distributed {
|
||||
let Some(notification_system) = store.scanner_notification_system() else {
|
||||
return resolution.requested_scope;
|
||||
};
|
||||
let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else {
|
||||
return resolution.requested_scope;
|
||||
};
|
||||
let mut expected_peers = HashMap::new();
|
||||
for (host, lease_instance_id, _) in crate::scanner::scanner_activity_publication_lease_targets(resolution.activity_before)
|
||||
{
|
||||
let Some((activity_instance_id, generation, pending)) =
|
||||
crate::scanner::scanner_activity_dirty_usage_state_for_host(resolution.activity_before, &host)
|
||||
else {
|
||||
return resolution.requested_scope;
|
||||
};
|
||||
if activity_instance_id != lease_instance_id || expected_peers.contains_key(&host) {
|
||||
return resolution.requested_scope;
|
||||
}
|
||||
expected_peers.insert(
|
||||
host,
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: activity_instance_id.to_string(),
|
||||
generation,
|
||||
pending,
|
||||
},
|
||||
);
|
||||
}
|
||||
let Some(remote_dirty_buckets) = verified_remote_dirty_usage_buckets(&expected_peers, peer_snapshots) else {
|
||||
return resolution.requested_scope;
|
||||
};
|
||||
dirty_buckets.extend(remote_dirty_buckets);
|
||||
}
|
||||
|
||||
scoped_scan_scope_from_dirty_buckets(
|
||||
resolution.requested_scope,
|
||||
dirty_buckets,
|
||||
true,
|
||||
resolution.all_buckets,
|
||||
resolution.baseline_proof,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
|
||||
@@ -171,7 +97,6 @@ where
|
||||
leader_epoch,
|
||||
scan_mode,
|
||||
scan_scope,
|
||||
persisted_usage_baseline,
|
||||
} = request;
|
||||
let child_token = ctx.child_token();
|
||||
let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch);
|
||||
@@ -261,26 +186,8 @@ where
|
||||
}
|
||||
bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
|
||||
let scan_plan_digest =
|
||||
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
|
||||
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before));
|
||||
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
|
||||
let scan_scope = resolve_scanner_bucket_scan_scope(
|
||||
store,
|
||||
distributed,
|
||||
ScannerBucketScopeResolution {
|
||||
requested_scope: scan_scope,
|
||||
baseline_proof: ScannerCacheBaselineProof {
|
||||
data: persisted_usage_baseline.as_ref(),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch,
|
||||
want_cycle,
|
||||
scan_plan_digest,
|
||||
},
|
||||
activity_before: &activity_before,
|
||||
dirty_usage_snapshot: &dirty_usage_snapshot,
|
||||
all_buckets: &all_buckets,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle));
|
||||
let tier_registry = runtime_tier_registry_for_cycle(want_cycle, leader_epoch).await;
|
||||
let tier_registry_generation = tier_registry.generation;
|
||||
|
||||
@@ -655,31 +655,9 @@ fn completed_data_usage_info_requires_every_set_before_publish() {
|
||||
.expect("all completed sets should produce a publishable data usage snapshot");
|
||||
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
|
||||
assert_eq!(data_usage_info.scanner_cycle, Some(0));
|
||||
assert_eq!(data_usage_info.scanner_epoch, Some(0));
|
||||
assert_eq!(data_usage_info.objects_total_count, 3);
|
||||
assert_eq!(data_usage_info.buckets_usage.len(), 3);
|
||||
assert!(data_usage_info.usage_snapshot_complete);
|
||||
assert_eq!(
|
||||
data_usage_info
|
||||
.usage_snapshot_set_states
|
||||
.iter()
|
||||
.map(|state| {
|
||||
(
|
||||
state.pool_index,
|
||||
state.set_index,
|
||||
state.scanner_cycle,
|
||||
state.scanner_epoch,
|
||||
state.scan_plan_digest,
|
||||
state.complete,
|
||||
state.tombstone,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
(0, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false),
|
||||
(1, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false),
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
data_usage_info
|
||||
.buckets_usage
|
||||
|
||||
@@ -17,7 +17,6 @@ use super::io_disk::tier_stats_template;
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||
use crate::scanner_folder::ScannerItem;
|
||||
use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
|
||||
use crate::storage_api::owner::{
|
||||
EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
|
||||
};
|
||||
@@ -797,195 +796,6 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa
|
||||
cache
|
||||
}
|
||||
|
||||
fn complete_usage_baseline(
|
||||
source: DataUsageCacheSource,
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
scanner_cycle: u64,
|
||||
scanner_epoch: u64,
|
||||
) -> bytes::Bytes {
|
||||
let baseline = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
|
||||
scanner_cycle: Some(scanner_cycle),
|
||||
scanner_epoch: Some(scanner_epoch),
|
||||
buckets_count: 1,
|
||||
buckets_usage: HashMap::from([("photos".to_string(), Default::default())]),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(true),
|
||||
usage_snapshot_set_states: vec![DataUsageSnapshotSetState {
|
||||
pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"),
|
||||
set_index: u64::try_from(source.set_index).expect("test set index should fit"),
|
||||
scanner_cycle: Some(scanner_cycle),
|
||||
scanner_epoch: Some(scanner_epoch),
|
||||
scan_plan_digest: Some(scan_plan_digest.0),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_scan_requires_a_converged_complete_baseline_with_exact_set_provenance() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let expected_sources = HashSet::from([source]);
|
||||
let scan_plan_digest = DataUsageScanPlanDigest([9; 32]);
|
||||
let baseline = complete_usage_baseline(source, scan_plan_digest, 7, 11);
|
||||
|
||||
assert_eq!(
|
||||
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
|
||||
data: Some(&baseline),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest,
|
||||
}),
|
||||
Some(scan_plan_digest)
|
||||
);
|
||||
|
||||
let mut incomplete = serde_json::from_slice::<DataUsageInfo>(&baseline).expect("test baseline should decode");
|
||||
incomplete.usage_snapshot_converged = Some(false);
|
||||
let incomplete = bytes::Bytes::from(serde_json::to_vec(&incomplete).expect("test baseline should encode"));
|
||||
assert_eq!(
|
||||
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
|
||||
data: Some(&incomplete),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest,
|
||||
}),
|
||||
None
|
||||
);
|
||||
|
||||
let mut wrong_provenance = serde_json::from_slice::<DataUsageInfo>(&baseline).expect("test baseline should decode");
|
||||
wrong_provenance.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32]);
|
||||
let wrong_provenance = bytes::Bytes::from(serde_json::to_vec(&wrong_provenance).expect("test baseline should encode"));
|
||||
assert_eq!(
|
||||
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
|
||||
data: Some(&wrong_provenance),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest,
|
||||
}),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_scan_selects_only_current_dirty_buckets_after_baseline_validation() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let expected_sources = HashSet::from([source]);
|
||||
let baseline_scan_plan_digest = DataUsageScanPlanDigest([4; 32]);
|
||||
let current_scan_plan_digest = DataUsageScanPlanDigest([5; 32]);
|
||||
let baseline = complete_usage_baseline(source, current_scan_plan_digest, 7, 11);
|
||||
let scope = scoped_scan_scope_from_dirty_buckets(
|
||||
ScannerBucketScanScope::default(),
|
||||
HashSet::from(["photos".to_string(), "deleted".to_string()]),
|
||||
true,
|
||||
&[bucket_info("photos")],
|
||||
ScannerCacheBaselineProof {
|
||||
data: Some(&baseline),
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest: current_scan_plan_digest,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(scope.baseline_scan_plan_digest, Some(current_scan_plan_digest));
|
||||
assert_eq!(
|
||||
scope
|
||||
.selected_buckets
|
||||
.as_deref()
|
||||
.expect("validated scope should select a bucket"),
|
||||
&HashSet::from(["photos".to_string()])
|
||||
);
|
||||
assert_ne!(scope.baseline_scan_plan_digest, Some(baseline_scan_plan_digest));
|
||||
}
|
||||
|
||||
fn peer_dirty_usage_snapshot(
|
||||
instance_id: &str,
|
||||
generation: u64,
|
||||
complete: bool,
|
||||
buckets: &[(&str, u64)],
|
||||
) -> EcstoreScannerPeerDirtyUsageSnapshot {
|
||||
EcstoreScannerPeerDirtyUsageSnapshot {
|
||||
instance_id: instance_id.to_string(),
|
||||
generation,
|
||||
pending_bucket_count: u64::try_from(buckets.len()).expect("test bucket count should fit"),
|
||||
protocol_version: crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
|
||||
complete,
|
||||
buckets: buckets
|
||||
.iter()
|
||||
.map(|(bucket, generation)| ((*bucket).to_string(), *generation))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots() {
|
||||
let expected_peers = HashMap::from([
|
||||
(
|
||||
"node-a:9000".to_string(),
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: "instance-a".to_string(),
|
||||
generation: 7,
|
||||
pending: true,
|
||||
},
|
||||
),
|
||||
(
|
||||
"node-b:9000".to_string(),
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: "instance-b".to_string(),
|
||||
generation: 3,
|
||||
pending: false,
|
||||
},
|
||||
),
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
verified_remote_dirty_usage_buckets(
|
||||
&expected_peers,
|
||||
vec![
|
||||
(
|
||||
"node-a:9000".to_string(),
|
||||
peer_dirty_usage_snapshot("instance-a", 7, true, &[("photos", 7)]),
|
||||
),
|
||||
(
|
||||
"node-b:9000".to_string(),
|
||||
peer_dirty_usage_snapshot("instance-b", 3, true, &[("archive", 3)]),
|
||||
),
|
||||
],
|
||||
),
|
||||
Some(HashSet::from(["photos".to_string(), "archive".to_string()]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state() {
|
||||
let expected_peers = HashMap::from([(
|
||||
"node-a:9000".to_string(),
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: "instance-a".to_string(),
|
||||
generation: 7,
|
||||
pending: true,
|
||||
},
|
||||
)]);
|
||||
|
||||
for snapshot in [
|
||||
peer_dirty_usage_snapshot("instance-a", 7, false, &[("photos", 7)]),
|
||||
peer_dirty_usage_snapshot("instance-a", 6, true, &[("photos", 6)]),
|
||||
peer_dirty_usage_snapshot("instance-b", 7, true, &[("photos", 7)]),
|
||||
peer_dirty_usage_snapshot("instance-a", 7, true, &[]),
|
||||
] {
|
||||
assert!(
|
||||
verified_remote_dirty_usage_buckets(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(),
|
||||
"incomplete, stale, mismatched, or empty pending peer state must fall back to a full scan"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
|
||||
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
|
||||
@@ -1238,27 +1048,6 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_superseded_is_distinct_from_partial_and_cancel() {
|
||||
for (budget, cancelled, bucket, expected) in [
|
||||
(false, false, ScannerBucketScanStatus::Complete, ScannerCycleStatus::Superseded),
|
||||
(true, false, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
|
||||
(false, true, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
|
||||
] {
|
||||
assert_eq!(
|
||||
classify_nsscanner_cycle(
|
||||
true,
|
||||
budget,
|
||||
cancelled,
|
||||
bucket,
|
||||
DirtyUsageSnapshotStatus::Changed,
|
||||
ScannerCycleActivityStatus::Unchanged
|
||||
),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unverified_activity_defers_partial_and_floor_cycles() {
|
||||
let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
||||
|
||||
@@ -38,8 +38,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::object_opts_fr
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard, get_lifecycle_config as ecstore_get_lifecycle_config,
|
||||
get_object_lock_config as ecstore_get_object_lock_config, get_replication_config as ecstore_get_replication_config,
|
||||
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
|
||||
get_replication_config as ecstore_get_replication_config,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::replication::{
|
||||
ReplicateObjectInfo, ReplicationConfig as EcstoreReplicationConfig,
|
||||
@@ -103,9 +103,7 @@ pub(crate) use rustfs_ecstore::api::rebalance::{
|
||||
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
|
||||
RebalanceStats as EcstoreRebalanceStats,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
ScannerBucketListing as EcstoreScannerBucketListing, ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::ScannerBucketListing as EcstoreScannerBucketListing;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
|
||||
pub(crate) use rustfs_ecstore::api::runtime::{
|
||||
|
||||
@@ -49,14 +49,5 @@ rustfs-rio.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
astral-tokio-tar = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
tar-codec = { workspace = true }
|
||||
tar-framing = { workspace = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
# minio-go Snowball fixtures
|
||||
|
||||
These request bodies are generated by
|
||||
`github.com/minio/minio-go/v7.Client.PutObjectsSnowball` at the version pinned
|
||||
in `generate/go.mod`. They cover the raw TAR and S2-compressed forms accepted by
|
||||
RustFS Snowball extraction.
|
||||
|
||||
The decoded TAR intentionally ends immediately after the final padded member
|
||||
body because minio-go flushes, rather than closes, its TAR writer. The
|
||||
compatibility test permits that shape only when the authenticated request body
|
||||
is complete at the exact member boundary; it does not make incomplete TAR
|
||||
terminators generally valid.
|
||||
|
||||
Regenerate them from this directory with Go 1.25:
|
||||
|
||||
```console
|
||||
cd generate
|
||||
go mod download
|
||||
go run . -out ..
|
||||
```
|
||||
|
||||
`manifest.json` records the input objects and SHA-256 digest of each captured
|
||||
request body. Review changes to the manifest and binary fixtures together when
|
||||
updating minio-go.
|
||||
@@ -1,26 +0,0 @@
|
||||
module rustfs.local/snowball-fixture
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require github.com/minio/minio-go/v7 v7.3.0
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.41.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.3 // indirect
|
||||
)
|
||||
@@ -1,59 +0,0 @@
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.3.0 h1:HM4pFCSQq/TK+j0/zmorSh5ddh81iDgRgU0BG0Vz/YU=
|
||||
github.com/minio/minio-go/v7 v7.3.0/go.mod h1:KUPWdecEO1LWyUz+sTGXAuf2jZHrPh5fCsRH86QbPfk=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
|
||||
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -1,193 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
const minioGoVersion = "v7.3.0"
|
||||
|
||||
type fixtureManifest struct {
|
||||
Generator string `json:"generator"`
|
||||
MinioGo string `json:"minio_go"`
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
Objects []fixtureObject `json:"objects"`
|
||||
Archives []fixtureArchive `json:"archives"`
|
||||
}
|
||||
|
||||
type fixtureObject struct {
|
||||
Key string `json:"key"`
|
||||
Body string `json:"body"`
|
||||
ModTime string `json:"mod_time"`
|
||||
VersionID string `json:"version_id,omitempty"`
|
||||
Headers map[string][]string `json:"headers,omitempty"`
|
||||
}
|
||||
|
||||
type fixtureArchive struct {
|
||||
File string `json:"file"`
|
||||
Compressed bool `json:"compressed"`
|
||||
Length int `json:"length"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
func objects() []fixtureObject {
|
||||
return []fixtureObject{
|
||||
{
|
||||
Key: "alpha.txt",
|
||||
Body: "alpha-body",
|
||||
ModTime: "2024-01-02T03:04:05Z",
|
||||
VersionID: "018cc251-f400-7c22-9e8d-8b1800000001",
|
||||
Headers: map[string][]string{
|
||||
"Content-Type": {"text/plain"},
|
||||
"X-Amz-Meta-Owner": {"snowball-fixture"},
|
||||
"X-Amz-Tagging": {"project=rustfs&source=minio-go"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Key: "nested/世界.txt",
|
||||
Body: "bravo-body",
|
||||
ModTime: "2024-01-02T03:05:05Z",
|
||||
Headers: map[string][]string{
|
||||
"Content-Language": {"zh-CN"},
|
||||
"X-Amz-Meta-Note": {"unicode-path"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func captureSnowball(compressed bool, specs []fixtureObject) ([]byte, error) {
|
||||
body := make(chan []byte, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
payload, err := io.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
http.Error(writer, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
body <- payload
|
||||
writer.Header().Set("ETag", `"snowball-fixture"`)
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := minio.New(strings.TrimPrefix(server.URL, "http://"), &minio.Options{
|
||||
// The S3 authentication layer removes AWS streaming-signature framing
|
||||
// before Snowball extraction sees the request body. Anonymous signing
|
||||
// captures those decoded archive bytes directly.
|
||||
Creds: credentials.NewStatic("", "", "", credentials.SignatureAnonymous),
|
||||
Secure: false,
|
||||
Region: "us-east-1",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("construct minio client: %w", err)
|
||||
}
|
||||
|
||||
input := make(chan minio.SnowballObject, len(specs))
|
||||
for _, spec := range specs {
|
||||
modTime, err := time.Parse(time.RFC3339, spec.ModTime)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse mod time for %q: %w", spec.Key, err)
|
||||
}
|
||||
headers := make(http.Header, len(spec.Headers))
|
||||
for name, values := range spec.Headers {
|
||||
headers[name] = append([]string(nil), values...)
|
||||
}
|
||||
input <- minio.SnowballObject{
|
||||
Key: spec.Key,
|
||||
Size: int64(len(spec.Body)),
|
||||
ModTime: modTime,
|
||||
Content: bytes.NewReader([]byte(spec.Body)),
|
||||
VersionID: spec.VersionID,
|
||||
Headers: headers,
|
||||
}
|
||||
}
|
||||
close(input)
|
||||
|
||||
err = client.PutObjectsSnowball(context.Background(), "fixture-bucket", minio.SnowballOptions{
|
||||
Opts: minio.PutObjectOptions{
|
||||
ContentType: "application/octet-stream",
|
||||
},
|
||||
InMemory: true,
|
||||
Compress: compressed,
|
||||
}, input)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate snowball request: %w", err)
|
||||
}
|
||||
return <-body, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
outDir := flag.String("out", "..", "fixture output directory")
|
||||
flag.Parse()
|
||||
|
||||
specs := objects()
|
||||
archives := make([]fixtureArchive, 0, 2)
|
||||
for _, fixture := range []struct {
|
||||
name string
|
||||
compressed bool
|
||||
}{
|
||||
{name: "snowball.tar"},
|
||||
{name: "snowball.tar.s2", compressed: true},
|
||||
} {
|
||||
payload, err := captureSnowball(fixture.compressed, specs)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
path := filepath.Join(*outDir, fixture.name)
|
||||
if err := os.WriteFile(path, payload, 0o644); err != nil {
|
||||
panic(fmt.Errorf("write %s: %w", path, err))
|
||||
}
|
||||
digest := sha256.Sum256(payload)
|
||||
archives = append(archives, fixtureArchive{
|
||||
File: fixture.name,
|
||||
Compressed: fixture.compressed,
|
||||
Length: len(payload),
|
||||
SHA256: hex.EncodeToString(digest[:]),
|
||||
})
|
||||
}
|
||||
|
||||
manifest := fixtureManifest{
|
||||
Generator: "github.com/minio/minio-go/v7.Client.PutObjectsSnowball",
|
||||
MinioGo: minioGoVersion,
|
||||
GeneratedAt: "2026-09-05T00:00:00Z",
|
||||
Objects: specs,
|
||||
Archives: archives,
|
||||
}
|
||||
payload, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
payload = append(payload, '\n')
|
||||
path := filepath.Join(*outDir, "manifest.json")
|
||||
if err := os.WriteFile(path, payload, 0o644); err != nil {
|
||||
panic(fmt.Errorf("write %s: %w", path, err))
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
{
|
||||
"generator": "github.com/minio/minio-go/v7.Client.PutObjectsSnowball",
|
||||
"minio_go": "v7.3.0",
|
||||
"generated_at": "2026-09-05T00:00:00Z",
|
||||
"objects": [
|
||||
{
|
||||
"key": "alpha.txt",
|
||||
"body": "alpha-body",
|
||||
"mod_time": "2024-01-02T03:04:05Z",
|
||||
"version_id": "018cc251-f400-7c22-9e8d-8b1800000001",
|
||||
"headers": {
|
||||
"Content-Type": [
|
||||
"text/plain"
|
||||
],
|
||||
"X-Amz-Meta-Owner": [
|
||||
"snowball-fixture"
|
||||
],
|
||||
"X-Amz-Tagging": [
|
||||
"project=rustfs\u0026source=minio-go"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "nested/世界.txt",
|
||||
"body": "bravo-body",
|
||||
"mod_time": "2024-01-02T03:05:05Z",
|
||||
"headers": {
|
||||
"Content-Language": [
|
||||
"zh-CN"
|
||||
],
|
||||
"X-Amz-Meta-Note": [
|
||||
"unicode-path"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"archives": [
|
||||
{
|
||||
"file": "snowball.tar",
|
||||
"compressed": false,
|
||||
"length": 4096,
|
||||
"sha256": "f00f2789dcb65b567f722f49cfdac9705e7bdac6c0badae75194327c32193d2e"
|
||||
},
|
||||
{
|
||||
"file": "snowball.tar.s2",
|
||||
"compressed": true,
|
||||
"length": 528,
|
||||
"sha256": "f8a9d9aa9b9ccdfae24ded1bff3741aacb935f1457a252efc9266674ff13c992"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user