mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-28 15:02:49 +00:00
Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d2cdf057cc | |||
| 717e8b9c87 | |||
| 5886368178 | |||
| db4b8290e7 | |||
| 5829c66f21 | |||
| 9c199aefd1 | |||
| 4c02ad4025 | |||
| e31a22febb | |||
| 91a6c5bad8 | |||
| a58ac62618 | |||
| 36c6c2e34b | |||
| d6729a1138 | |||
| 07a9d15ab2 | |||
| d9c3eeea0c | |||
| 892d3ba3f7 | |||
| 706ffdc9ac | |||
| f018628374 | |||
| cd10981132 | |||
| 5a353a6f6f | |||
| e3be247b40 | |||
| 14081c41a4 |
@@ -132,6 +132,12 @@ test-group = 'ecstore-serial-flaky'
|
||||
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
# The real object probe includes bucket creation and cleanup in its short
|
||||
# measurement budget. Keep competing storage fixtures outside that budget.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Serialize the durable manual-transition checkpoint test across nextest's
|
||||
# process boundary; it mutates bucket lifecycle metadata and is not quarantined.
|
||||
[[profile.default.overrides]]
|
||||
@@ -327,6 +333,10 @@ test-group = 'ecstore-serial-flaky'
|
||||
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Serialize the durable manual-transition checkpoint test under the ci profile
|
||||
# too. No retries: failures stay visible.
|
||||
[[profile.ci.overrides]]
|
||||
|
||||
@@ -4,13 +4,14 @@
|
||||
# behavior under drive and node loss against the erasure-coding contract and
|
||||
# snapshots health-endpoint responses at every tier.
|
||||
#
|
||||
# A single-node 4 drives (SNMD): hide 1/2/3 drives, restore
|
||||
# A single-node 4 drives (SNMD): revoke access to 1/2/3 drives, restore
|
||||
# B multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore
|
||||
# C multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary),
|
||||
# stop 2 nodes, restore
|
||||
# C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads
|
||||
# satisfy the EC read quorum while the lock majority is broken (the
|
||||
# reported divergence window: reads 503 with lock_quorum_unavailable)
|
||||
# D multi-node 4x1: stop 1/2 nodes, then restore
|
||||
#
|
||||
# Expectations come from product source (default_parity_count, erasure set
|
||||
# sizing). By default a "reads refused although the read quorum is met"
|
||||
@@ -32,6 +33,11 @@ on:
|
||||
description: 'Direct .deb URL. Required unless the nightly default is wanted.'
|
||||
required: false
|
||||
type: string
|
||||
auto_testing_ref:
|
||||
description: 'auto-testing branch, tag, or SHA (defaults to main)'
|
||||
required: false
|
||||
default: main
|
||||
type: string
|
||||
strict:
|
||||
description: 'Fail the suite when reads are refused despite a met read quorum'
|
||||
type: boolean
|
||||
@@ -52,7 +58,7 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The suite stops services and hides drive dirs on the shared fleet; only one
|
||||
# The suite stops services and revokes drive access on the shared fleet; only one
|
||||
# functional suite may touch the environment at a time.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests-v2
|
||||
@@ -92,6 +98,7 @@ jobs:
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
@@ -109,16 +116,24 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
|
||||
ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Record auto-testing revision
|
||||
run: |
|
||||
set -euo pipefail
|
||||
AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)"
|
||||
printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}"
|
||||
echo "auto-testing revision: ${AUTO_TESTING_SHA}"
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
aws --version
|
||||
echo "auto-testing: ${AUTO_TESTING_SHA}"
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
@@ -134,14 +149,14 @@ jobs:
|
||||
CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
|
||||
run: python3 scripts/prepare_functional_package.py prepare
|
||||
|
||||
- name: Run fault-tolerance scenarios (A, B, C, C2)
|
||||
- name: Run fault-tolerance scenarios (A, B, C, C2, D)
|
||||
timeout-minutes: 45
|
||||
id: test
|
||||
# Standalone case failures are reported by the backlog manager.
|
||||
# Chain evidence separately requires complete passing cases.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
|
||||
ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}")
|
||||
if [ "${{ inputs.strict }}" = "true" ]; then
|
||||
ARGS+=(--strict)
|
||||
fi
|
||||
@@ -158,12 +173,13 @@ jobs:
|
||||
echo ""
|
||||
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo "- auto-testing: \`${AUTO_TESTING_SHA}\`"
|
||||
echo "- Strict mode: ${{ inputs.strict || 'false' }}"
|
||||
echo ""
|
||||
echo "## Per-probe results"
|
||||
echo ""
|
||||
echo '```'
|
||||
grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
|
||||
grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
|
||||
echo '```'
|
||||
echo ""
|
||||
echo "## Health snapshots"
|
||||
@@ -174,19 +190,30 @@ jobs:
|
||||
done
|
||||
} > "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
# Product gate: failing FT cases keep the run green — the report above and
|
||||
# the backlog issue manager carry the signal. Only harness/environment
|
||||
# breakdowns (no case verdicts at all) turn the workflow red.
|
||||
FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)"
|
||||
FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} ))
|
||||
HARNESS_OK=0
|
||||
if [ "${{ steps.test.outcome }}" = "success" ]; then
|
||||
HARNESS_OK=1
|
||||
elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then
|
||||
HARNESS_OK=1
|
||||
fi
|
||||
[ "${HARNESS_OK}" = "1" ]
|
||||
printf 'complete=false\n' >> "${GITHUB_OUTPUT}"
|
||||
jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
|
||||
.schema_version == 1 and
|
||||
.auto_testing_sha == $testing_sha and
|
||||
.selected_scenarios == ["A", "B", "C", "C2", "D"] and
|
||||
.complete == true and
|
||||
.counts.expected == 38 and
|
||||
.counts.seen == 38 and
|
||||
.counts.harness_errors == 0 and
|
||||
(.expected_cases | length) == 38 and
|
||||
(.expected_cases | unique | length) == 38 and
|
||||
(.seen_cases | length) == 38 and
|
||||
(.seen_cases | unique | length) == 38 and
|
||||
(.seen_cases | sort) == (.expected_cases | sort) and
|
||||
(.missing_cases | length) == 0 and
|
||||
(.harness_errors | length) == 0 and
|
||||
(.cases | length) == 38 and
|
||||
([.cases[].case_id] | sort) == (.expected_cases | sort) and
|
||||
([.cases[].case_id] | unique | length) == 38 and
|
||||
all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
|
||||
.counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
|
||||
.counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
|
||||
' "${RESULTS_FILE}" >/dev/null
|
||||
printf 'complete=true\n' >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
timeout-minutes: 2
|
||||
@@ -196,7 +223,7 @@ jobs:
|
||||
# green run. Never acts on cancelled runs. Logic lives in
|
||||
# auto-testing/scripts/issue_manager.py, which parses the
|
||||
# FT-CASE verdict lines from the suite log.
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' && steps.chain_report.outputs.complete == 'true' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -272,6 +299,7 @@ jobs:
|
||||
name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
|
||||
if-no-files-found: warn
|
||||
|
||||
@@ -61,10 +61,13 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The default performance nodes overlap the other suites' remote VMs, even
|
||||
# though the runner differs. Hold the shared lock through cleanup as well.
|
||||
# Workflow-level so the lock is held through cleanup as well.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests-v2
|
||||
# Own group: the performance suite runs on the pf-testing fleet - a fully
|
||||
# separate environment from the smoke-testing fleet the functional suites
|
||||
# use. No shared node, no shared runner: it must never queue behind them
|
||||
# (or block them).
|
||||
group: rustfs-performance-suite
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
|
||||
Generated
+106
-63
@@ -685,9 +685,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.4.47"
|
||||
version = "0.4.48"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef217a77a86a6e3dab9a5b3c81dc445b603fe743a90c1cb10a2f2144628d8cfa"
|
||||
checksum = "fb61aea1a7def73ee7c350a184f0e70b32c182344e2e75bf70c9b621b83417fd"
|
||||
dependencies = [
|
||||
"compression-codecs",
|
||||
"compression-core",
|
||||
@@ -812,7 +812,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -961,9 +961,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-runtime"
|
||||
version = "1.9.2"
|
||||
version = "1.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657"
|
||||
checksum = "25b43ad47adc2517efe3d706559d94b97e50e80e0321b3cadbc9f77cee88adcd"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-sigv4",
|
||||
@@ -989,9 +989,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-kms"
|
||||
version = "1.118.0"
|
||||
version = "1.120.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c243864bc3754be9f0001414e0162fdf1370fa62c70b0cfbe1da6a6221d553c7"
|
||||
checksum = "d6b0fe38fee2ba5b6cd24d32d08365b314ae1adea123649e061a7eb6300b6f5b"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1015,9 +1015,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.146.1"
|
||||
version = "1.148.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2cd651b4400d4011b8927b83a9552bf90ff11e6e5da0b9f0a7583247aceec971"
|
||||
checksum = "8a61c1db3987ab6c8740fb87f248864687d50549401115244a84f02c3047b2d5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1052,9 +1052,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sso"
|
||||
version = "1.109.0"
|
||||
version = "1.111.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab"
|
||||
checksum = "65700d75f6da78c89a61a49f1539979400a4609d3329376b1dc2133d7a1b2b90"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1078,9 +1078,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-ssooidc"
|
||||
version = "1.111.0"
|
||||
version = "1.113.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161"
|
||||
checksum = "eb1147c254af7b6cee3ace90ae7da6f21366ce9a9b899035fa8a0fdf586eb75a"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1104,9 +1104,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sts"
|
||||
version = "1.114.0"
|
||||
version = "1.116.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a"
|
||||
checksum = "035c7392e751d20bf1060d625d023587eb14099646110a9cff7c697bfe4b253a"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1131,9 +1131,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sigv4"
|
||||
version = "1.5.1"
|
||||
version = "1.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f"
|
||||
checksum = "31d955e76ff96acd555bf06fa0fa6d5bf9335fa84ae7c64481b20ae61d231f70"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-eventstream",
|
||||
@@ -1737,7 +1737,7 @@ dependencies = [
|
||||
"prettyplease 0.3.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1996,9 +1996,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
|
||||
|
||||
[[package]]
|
||||
name = "cfg_aliases"
|
||||
@@ -2141,7 +2141,7 @@ dependencies = [
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2222,9 +2222,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "compression-codecs"
|
||||
version = "0.4.42"
|
||||
version = "0.4.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "257c7085cbb71be72d8fb97edff08b03d86d5d9f2222b9cc34a6bec87093bc10"
|
||||
checksum = "bef16c47ba2797aa6a909cc37d39911f3a6743811fe7408ac0b0cc0276b656e9"
|
||||
dependencies = [
|
||||
"brotli 8.0.4",
|
||||
"bzip2",
|
||||
@@ -2798,7 +2798,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2842,7 +2842,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
|
||||
dependencies = [
|
||||
"darling_core 0.24.1",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3343,7 +3343,7 @@ checksum = "3a711bd5a06ec85d9683e142d28a98ad0679a7220669f5c1b8589e57e9b29e46"
|
||||
dependencies = [
|
||||
"datafusion-doc",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3715,7 +3715,7 @@ checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3790,6 +3790,23 @@ dependencies = [
|
||||
"cipher 0.5.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dial9"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49b11bce55173f77d9003cce8f129b4af889a4639005c53c3498cef933bc895c"
|
||||
dependencies = [
|
||||
"bon",
|
||||
"dial9-core",
|
||||
"dial9-macro",
|
||||
"dial9-metrique",
|
||||
"dial9-perf-self-profile",
|
||||
"dial9-tokio-telemetry",
|
||||
"dial9-trace-format",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dial9-core"
|
||||
version = "0.5.0"
|
||||
@@ -3812,6 +3829,31 @@ dependencies = [
|
||||
"ulid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dial9-macro"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4177b8f8bf95d8e3769c1aad596f5d6fb231953a45c4c3d2a5946fe8e4ec327b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dial9-metrique"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d42fe0e3e14e8780c4ecf9ee14c183dbcb9b1ae3612cc9a689e7dc83b13adef1"
|
||||
dependencies = [
|
||||
"dial9-core",
|
||||
"dial9-trace-format",
|
||||
"metrique",
|
||||
"metrique-writer",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dial9-perf-self-profile"
|
||||
version = "0.5.0"
|
||||
@@ -3827,6 +3869,7 @@ dependencies = [
|
||||
"libc",
|
||||
"perf-event-data",
|
||||
"perf-event-open-sys2",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -3948,7 +3991,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4554,7 +4597,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5263,9 +5306,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.25.1"
|
||||
version = "0.26.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "366c5db5a8e4643bdde121a1d6f35cfa035e7850b8b8ea94c6960a139d44e93a"
|
||||
checksum = "f37234f85a8d1bda6b84d486245dd0847c5b33cca10d5f3691375e66704de0f8"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5275,7 +5318,7 @@ dependencies = [
|
||||
"flate2",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"futures-sink",
|
||||
"hdrhistogram",
|
||||
"hotpath-macros",
|
||||
"http 1.5.0",
|
||||
@@ -5296,9 +5339,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.25.1"
|
||||
version = "0.26.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "846bde0d9600d98434e1aac376977d7718bfe3d2f5312a041b7c59a6a466c51a"
|
||||
checksum = "2ac39b5517f540cab511bdbd5a4dc8cfc62858d6cd26a095463c1d2acfb3f290"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -6708,7 +6751,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustversion",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8177,7 +8220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9011,7 +9054,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10345,7 +10388,7 @@ dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
"dial9-tokio-telemetry",
|
||||
"dial9",
|
||||
"flate2",
|
||||
"futures-util",
|
||||
"glob",
|
||||
@@ -11068,9 +11111,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.4"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
|
||||
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"errno",
|
||||
@@ -11215,9 +11258,9 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.16.0"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d29a4db6a51f1cd3269a36354ce4aa7db2be8816cee42a642b9c85cd3769577"
|
||||
checksum = "87d22507a8ebd0ea64972c76dfedbc477418dc70afd98bfce928263894067d2f"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -11274,9 +11317,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "s3s-rfc2047"
|
||||
version = "0.16.0"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9370a2f353a7929b79ae7b5f692b4758c59aa19526b18feff6a977e8ff17da5a"
|
||||
checksum = "f9d3e6b36be7cca3da377b4cabcd668ffe5b16c3b73a83f7a8f7b63ff6a178ee"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"thiserror 2.0.20",
|
||||
@@ -11284,9 +11327,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "s3s-sigv2"
|
||||
version = "0.16.0"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f277b63ffc9a2135c49d52726199b7436c2f86746d625496984f5bb5cabef050"
|
||||
checksum = "d96880b5ad8d639c9224155126741708aeb62ad3682e0476d2fd14290af808ef"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -11298,9 +11341,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "s3s-sigv4"
|
||||
version = "0.16.0"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a979a401f6ae4035bc22e4ac0f7a9a5923ba93841c931e8e28652fe211d991bc"
|
||||
checksum = "fed605cbaaa1cf4d82e8ee15098fb3e42d49c92bb6215eb1359f63bfc0c6b020"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"base64-simd",
|
||||
@@ -11532,7 +11575,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11596,7 +11639,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11641,7 +11684,7 @@ dependencies = [
|
||||
"darling 0.24.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11676,7 +11719,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12273,9 +12316,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.5"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
|
||||
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -12322,7 +12365,7 @@ checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12516,7 +12559,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12653,7 +12696,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13169,9 +13212,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
@@ -13413,7 +13456,7 @@ dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
@@ -13988,7 +14031,7 @@ checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
"synstructure 0.14.0",
|
||||
]
|
||||
|
||||
@@ -14029,7 +14072,7 @@ checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
"synstructure 0.14.0",
|
||||
]
|
||||
|
||||
@@ -14083,7 +14126,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+9
-9
@@ -145,7 +145,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.1" }
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.47" }
|
||||
async-compression = { version = "0.4.48" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
@@ -244,9 +244,9 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.12.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.118.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.146.1" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.114.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.120.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.148.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.116.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.2" }
|
||||
aws-smithy-runtime-api = { version = "1.16.2" }
|
||||
@@ -310,11 +310,11 @@ regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.0" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rustix = { version = "1.1.5" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { version = "0.16.0", features = ["minio"] }
|
||||
s3s-sigv4 = { version = "0.16.0" }
|
||||
s3s = { version = "0.16.1", features = ["minio"] }
|
||||
s3s-sigv4 = { version = "0.16.1" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
@@ -351,7 +351,7 @@ zstd = "0.14.0"
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9-tokio-telemetry = "0.5.0"
|
||||
dial9 = { version = "0.5.0", default-features = false }
|
||||
opentelemetry = { version = "0.32.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.32.0" }
|
||||
opentelemetry-otlp = { version = "0.32.0" }
|
||||
@@ -375,7 +375,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.5.4" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.25.1", default-features = false }
|
||||
hotpath = { version = "0.26.0", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -663,6 +663,15 @@ pub const ENV_OBJECT_LOCK_RPC_DETACHED_LIMIT: &str = "RUSTFS_OBJECT_LOCK_RPC_DET
|
||||
/// Default per-peer budget of detached (timed-out but still running) lock RPCs: 256.
|
||||
pub const DEFAULT_OBJECT_LOCK_RPC_DETACHED_LIMIT: usize = 256;
|
||||
|
||||
/// Environment variable for the maximum number of in-flight lock acquisition
|
||||
/// RPCs admitted to one peer. Requests beyond this bound fail fast as
|
||||
/// retryable contention so a slow endpoint cannot accumulate an unbounded
|
||||
/// queue while its channel remains healthy.
|
||||
pub const ENV_OBJECT_LOCK_RPC_REQUEST_LIMIT: &str = "RUSTFS_OBJECT_LOCK_RPC_REQUEST_LIMIT";
|
||||
|
||||
/// Default per-peer in-flight lock acquisition RPC admission limit.
|
||||
pub const DEFAULT_OBJECT_LOCK_RPC_REQUEST_LIMIT: usize = 128;
|
||||
|
||||
/// Environment variable to enable object namespace lock diagnostics.
|
||||
///
|
||||
/// When enabled, RustFS emits slow lock acquisition and long lock hold
|
||||
|
||||
@@ -39,7 +39,7 @@ pub const ENV_RUNTIME_DIAL9_S3_BUCKET: &str = "RUSTFS_RUNTIME_DIAL9_S3_BUCKET";
|
||||
/// Accepted but not honoured; see [`ENV_RUNTIME_DIAL9_S3_BUCKET`].
|
||||
pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
|
||||
// Note: there are deliberately no task-dump knobs. dial9 only captures task
|
||||
// dumps for futures spawned through `dial9_tokio_telemetry::spawn`, and RustFS
|
||||
// dumps for futures spawned through `dial9::spawn`, and RustFS
|
||||
// spawns with `tokio::spawn` throughout, so the switch could never do anything.
|
||||
// Measured: 0 dumps via tokio::spawn vs 14709 via dial9::spawn on an identical
|
||||
// workload. See rustfs/backlog#1157 (D9-16).
|
||||
|
||||
@@ -400,17 +400,36 @@ async fn verify_write_observations_during_peer_failure(dist: &DistCluster, bucke
|
||||
let observations = storage["info"]["observations"]
|
||||
.as_array()
|
||||
.ok_or("recovery omitted observations")?;
|
||||
let disks = storage["info"]["disks"].as_array().ok_or("recovery omitted disks")?;
|
||||
Ok(observations.len() == 4
|
||||
&& observations
|
||||
.iter()
|
||||
.all(|item| item["status"] == "succeeded" && item["cached"] == false))
|
||||
.all(|item| item["status"] == "succeeded" && item["cached"] == false)
|
||||
&& disks.len() == 16
|
||||
&& disks.iter().all(|disk| {
|
||||
disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
|
||||
&& disk["runtimeState"]
|
||||
.as_str()
|
||||
.is_some_and(|state| state.eq_ignore_ascii_case("online"))
|
||||
}))
|
||||
},
|
||||
"peer probes recover to fresh successful observations",
|
||||
"peer probes and remote disks recover to fresh successful observations",
|
||||
)
|
||||
.await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
assert!(observed_put(dist, 0, bucket, "recovered").await?.is_success());
|
||||
assert_eq!(http_put_counts(dist, 0).await?, [baseline[0][0] + 1, baseline[0][1] + 2]);
|
||||
let recovered_before = http_put_counts(dist, 0).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(90),
|
||||
|| async { Ok(observed_put(dist, 0, bucket, "recovered").await?.is_success()) },
|
||||
"recovered PUT after remote disk and lock convergence",
|
||||
)
|
||||
.await?;
|
||||
let recovered_after = http_put_counts(dist, 0).await?;
|
||||
assert_eq!(recovered_after[0], recovered_before[0] + 1);
|
||||
assert!(
|
||||
recovered_after[1] >= baseline[0][1] + 2,
|
||||
"recovery retries must retain the two intentional sub-quorum failures: before={recovered_before:?}, after={recovered_after:?}"
|
||||
);
|
||||
for node in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node)?, bucket, "healthy-0", b"write-observation").await?;
|
||||
assert_object_bytes(&dist.client(node)?, bucket, "recovered", b"write-observation").await?;
|
||||
|
||||
@@ -391,8 +391,9 @@ pub mod data_usage {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_movement::scanner_backlog::test_util::NativeScannerPauseBacklogWriteFault;
|
||||
pub use crate::data_movement::scanner_backlog::{
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementPlanner,
|
||||
ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan,
|
||||
ScannerPauseBacklogRetirementPlanner, ScannerPauseBacklogRetirementReplica,
|
||||
register_scanner_pause_backlog_retirement_planner,
|
||||
};
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
|
||||
@@ -515,20 +515,26 @@ impl ExpiryStats {
|
||||
Self::add_nonnegative(&self.missed_tier_journal_tasks, 1);
|
||||
}
|
||||
|
||||
// The pending and active gauges are balanced by design: every increment
|
||||
// has exactly one matching decrement. They must not saturate at zero on
|
||||
// update, because a worker can dequeue (and decrement) before the
|
||||
// enqueuing side has recorded its increment. Clamping that transient -1
|
||||
// to 0 turns the later +1 into a phantom task that never drains
|
||||
// (rustfs#7921). Readers clamp negative snapshots instead.
|
||||
fn increment_pending_tasks(&self) {
|
||||
Self::add_nonnegative(&self.pending_tasks, 1);
|
||||
self.pending_tasks.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
fn decrement_pending_tasks(&self) {
|
||||
Self::add_nonnegative(&self.pending_tasks, -1);
|
||||
self.pending_tasks.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
fn increment_active_tasks(&self) {
|
||||
Self::add_nonnegative(&self.active_tasks, 1);
|
||||
self.active_tasks.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
fn decrement_active_tasks(&self) {
|
||||
Self::add_nonnegative(&self.active_tasks, -1);
|
||||
self.active_tasks.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
fn increment_workers(&self) {
|
||||
@@ -1074,9 +1080,13 @@ impl ExpiryState {
|
||||
}
|
||||
|
||||
fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
|
||||
// Account for the task before a worker can observe it. The worker
|
||||
// decrements on dequeue, so incrementing after `try_send` would let a
|
||||
// fast dequeue run the gauge through zero first.
|
||||
self.stats.increment_pending_tasks();
|
||||
let queued = wrkr.try_send(Some(task)).is_ok();
|
||||
if queued {
|
||||
self.stats.increment_pending_tasks();
|
||||
if !queued {
|
||||
self.stats.decrement_pending_tasks();
|
||||
}
|
||||
queued
|
||||
}
|
||||
@@ -1444,11 +1454,14 @@ async fn enqueue_recovered_free_version_with_state(state: &Arc<RwLock<ExpiryStat
|
||||
return false;
|
||||
};
|
||||
|
||||
// Same ordering rule as `ExpiryState::send_expiry_task`: count first, so
|
||||
// a worker that dequeues immediately cannot decrement before this
|
||||
// increment lands.
|
||||
stats.increment_pending_tasks();
|
||||
let queued = wrkr.try_send(Some(Box::new(task))).is_ok();
|
||||
if !queued {
|
||||
stats.decrement_pending_tasks();
|
||||
stats.increment_missed_freevers_tasks();
|
||||
} else {
|
||||
stats.increment_pending_tasks();
|
||||
}
|
||||
stats.record_scanner_expiry_state();
|
||||
queued
|
||||
@@ -3757,11 +3770,8 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
let Some(api) = runtime_sources::object_store_handle() else {
|
||||
return;
|
||||
};
|
||||
let configs = match metadata_boundary::get_expiry_configs(&api, &oi.bucket).await {
|
||||
pub(crate) async fn enqueue_immediate_expiry(api: Arc<ECStore>, oi: &ObjectInfo, src: LcEventSrc, opts: &ObjectOptions) {
|
||||
let configs = match metadata_boundary::get_expiry_configs_for_options(&api, &oi.bucket, opts).await {
|
||||
Ok(configs) => configs,
|
||||
Err(err) => {
|
||||
observe_lifecycle_observability_event(EVENT_LIFECYCLE_EVALUATION_FAILED, "failed", Some("metadata_unavailable"));
|
||||
@@ -7572,6 +7582,65 @@ mod tests {
|
||||
assert!(recovery_notify.notified().now_or_never().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn expiry_pending_gauge_survives_dequeue_landing_before_enqueue_accounting() {
|
||||
// A worker may dequeue and decrement before the enqueuing side records
|
||||
// its increment. The gauge must return to zero afterwards instead of
|
||||
// clamping the transient -1 away and reporting a phantom pending task
|
||||
// that no idle check can ever drain (rustfs#7921).
|
||||
let state = ExpiryState::new();
|
||||
let stats = Arc::clone(&state.read().await.stats);
|
||||
|
||||
stats.decrement_pending_tasks();
|
||||
stats.increment_pending_tasks();
|
||||
assert_eq!(stats.pending_tasks(), 0);
|
||||
assert_eq!(state.read().await.pending_tasks(), 0);
|
||||
|
||||
stats.decrement_active_tasks();
|
||||
stats.increment_active_tasks();
|
||||
assert_eq!(stats.active_tasks(), 0);
|
||||
assert_eq!(state.read().await.active_tasks(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn free_version_enqueue_rolls_back_pending_when_queue_full() {
|
||||
// Single-threaded, so this cannot observe the count-before-publish
|
||||
// ordering itself; it pins the rollback that ordering requires: a
|
||||
// rejected send must not leave its speculative increment behind.
|
||||
let state = ExpiryState::new_with_unconsumed_worker_channel(1);
|
||||
let oi = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: "remote-version".to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
free_version: true,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(state.read().await.enqueue_free_version(oi.clone()));
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
|
||||
// The single-slot queue is full for both enqueue paths.
|
||||
assert!(!enqueue_recovered_free_version_with_state(&state, oi.clone()).await);
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 1);
|
||||
|
||||
assert!(!state.read().await.enqueue_free_version(oi));
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1);
|
||||
assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 2);
|
||||
|
||||
// Draining the one real task returns the gauge to zero.
|
||||
let receiver = state.read().await.tasks_rx[0].clone();
|
||||
let task = receiver.lock().await.recv().await.expect("queued task");
|
||||
assert!(task.is_some());
|
||||
state.read().await.stats.decrement_pending_tasks();
|
||||
assert_eq!(state.read().await.pending_tasks(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enqueue_recovered_free_version_reports_false_without_worker_channel() {
|
||||
let state = ExpiryState::new();
|
||||
@@ -12703,7 +12772,8 @@ mod tests {
|
||||
.push((event, state, reason));
|
||||
});
|
||||
|
||||
super::enqueue_immediate_expiry(&object_info, LcEventSrc::S3PutObject).await;
|
||||
super::enqueue_immediate_expiry(Arc::clone(&ecstore), &object_info, LcEventSrc::S3PutObject, &ObjectOptions::default())
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
observed.lock().expect("observed events should not poison").contains(&(
|
||||
|
||||
@@ -55,7 +55,15 @@ pub(crate) async fn lifecycle_expiry_allowed(
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
get_expiry_configs_for_options(api, bucket, &crate::object_api::ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs_for_options(
|
||||
api: &crate::store::ECStore,
|
||||
bucket: &str,
|
||||
opts: &crate::object_api::ObjectOptions,
|
||||
) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?;
|
||||
let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
|
||||
let table_bucket_enabled = metadata.table_bucket_enabled();
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_p
|
||||
use crate::bucket::utils::is_meta_bucketname;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
@@ -490,6 +491,18 @@ pub(crate) async fn get_bucket_incarnation_id_in(ctx: &crate::runtime::instance:
|
||||
sys.get_bucket_incarnation_id_from_disk(bucket).await
|
||||
}
|
||||
|
||||
/// Validate against disk while retaining an already-held Object Lock fence.
|
||||
pub(crate) async fn get_bucket_incarnation_id_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<Uuid> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
sys.get_bucket_incarnation_id_under_transaction_lock(bucket, &guard).await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_cached_bucket_incarnation_id_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
@@ -1055,6 +1068,24 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in(
|
||||
Ok(lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
|
||||
}
|
||||
|
||||
/// Readers already holding an Object Lock snapshot must share its guard:
|
||||
/// a fresh read acquisition can queue behind a writer waiting for that snapshot.
|
||||
pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
|
||||
let store = object_store_in(ctx).await?;
|
||||
return snapshot
|
||||
.metadata_transaction_guard_for(store.id, bucket, opts.expected_bucket_incarnation_id)
|
||||
.ok_or_else(|| {
|
||||
Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
|
||||
});
|
||||
}
|
||||
Ok(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?))
|
||||
}
|
||||
|
||||
async fn acquire_transaction_lock_with_sys(
|
||||
sys: &Arc<RwLock<BucketMetadataSys>>,
|
||||
bucket: &str,
|
||||
@@ -2357,8 +2388,17 @@ impl BucketMetadataSys {
|
||||
let _transaction_guard = transaction_lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await?;
|
||||
self.get_bucket_incarnation_id_under_transaction_lock(bucket, &_transaction_guard)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_bucket_incarnation_id_under_transaction_lock(
|
||||
&self,
|
||||
bucket: &str,
|
||||
transaction_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<Uuid> {
|
||||
let incarnation_id = load_bucket_incarnation(self.object_store(), bucket).await?;
|
||||
if _transaction_guard.is_lock_lost() {
|
||||
if transaction_guard.is_lock_lost() {
|
||||
return Err(Error::other(format!("bucket incarnation metadata transaction lock was lost: {bucket}")));
|
||||
}
|
||||
match incarnation_id {
|
||||
|
||||
@@ -38,6 +38,7 @@ const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
|
||||
const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
|
||||
const ORPHAN_PROBE_CONCURRENCY: usize = 32;
|
||||
const EVENT_QUOTA_LEDGER_SETTLEMENT: &str = "quota_ledger_settlement";
|
||||
const EVENT_QUOTA_ADMISSION: &str = "quota_admission";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_QUOTA: &str = "quota";
|
||||
|
||||
@@ -46,6 +47,8 @@ static FAIL_NEXT_LEDGER_SAVE: std::sync::atomic::AtomicBool = std::sync::atomic:
|
||||
|
||||
// Lock order: caller-held destination object/upload, bucket metadata
|
||||
// transaction (read), operation reservation, then quota ledger.
|
||||
// When Object Lock already holds the metadata transaction read lock, reuse
|
||||
// that guard: reacquiring it behind a waiting metadata writer would deadlock.
|
||||
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
const ORPHAN_MIN_AGE_SECONDS: i64 = 30;
|
||||
@@ -210,7 +213,7 @@ pub(crate) struct QuotaContext {
|
||||
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
snapshot_admission: Option<QuotaAdmission>,
|
||||
legacy_data_movement: bool,
|
||||
metadata_guard: Option<NamespaceLockGuard>,
|
||||
metadata_guard: Option<Arc<NamespaceLockGuard>>,
|
||||
pool_index: Option<usize>,
|
||||
set_index: Option<usize>,
|
||||
}
|
||||
@@ -295,7 +298,7 @@ impl QuotaContext {
|
||||
limit: quota_limit,
|
||||
});
|
||||
}
|
||||
if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) {
|
||||
if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
bucket: ledger_data.bucket.clone(),
|
||||
@@ -332,7 +335,7 @@ struct LedgerReservationData {
|
||||
pub(crate) struct QuotaReservation {
|
||||
ledger: Option<LedgerReservationData>,
|
||||
operation_guard: Option<NamespaceLockGuard>,
|
||||
metadata_guard: Option<NamespaceLockGuard>,
|
||||
metadata_guard: Option<Arc<NamespaceLockGuard>>,
|
||||
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
state: ReservationState,
|
||||
}
|
||||
@@ -346,7 +349,7 @@ enum ReservationState {
|
||||
}
|
||||
|
||||
impl QuotaReservation {
|
||||
fn unlimited(metadata_guard: Option<NamespaceLockGuard>) -> Self {
|
||||
fn unlimited(metadata_guard: Option<Arc<NamespaceLockGuard>>) -> Self {
|
||||
Self {
|
||||
ledger: None,
|
||||
operation_guard: None,
|
||||
@@ -358,7 +361,7 @@ impl QuotaReservation {
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
self.operation_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
|| self.metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|
||||
|| self.metadata_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
}
|
||||
|
||||
pub(crate) fn capability_proof_matches(&self) -> bool {
|
||||
@@ -438,11 +441,12 @@ pub(crate) async fn begin(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
snapshot_admission: Option<QuotaAdmission>,
|
||||
data_movement: bool,
|
||||
opts: &ObjectOptions,
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
) -> Result<QuotaContext> {
|
||||
let snapshot_admission = opts.quota_admission;
|
||||
let data_movement = opts.data_movement;
|
||||
if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
return Ok(QuotaContext {
|
||||
store: None,
|
||||
@@ -497,7 +501,7 @@ pub(crate) async fn begin(
|
||||
});
|
||||
}
|
||||
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?;
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
let (quota, bucket_incarnation, quota_revision) =
|
||||
metadata_sys::get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await?;
|
||||
if metadata_guard.is_lock_lost() {
|
||||
@@ -513,6 +517,7 @@ pub(crate) async fn begin(
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.has_unsupported_reservation_protocol())
|
||||
{
|
||||
log_admission_rejected(bucket, object, "unsupported_reservation_protocol");
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations());
|
||||
@@ -529,7 +534,16 @@ pub(crate) async fn begin(
|
||||
Some(quota) => match (quota.quota, snapshot_admission) {
|
||||
(Some(limit), Some(admission)) if admission.quota_limit() == limit => Some(admission),
|
||||
(Some(_), None) if data_movement => None,
|
||||
(Some(_), _) => return Err(StorageError::PartMissingOrCorrupt),
|
||||
(Some(_), _) => {
|
||||
// A snapshot-protocol quota requires the request handler's
|
||||
// admission on every write. Missing or mismatched admission
|
||||
// means a caller rebuilt `ObjectOptions` without carrying it
|
||||
// over (rustfs/rustfs#7674 lost it on CopyObject); fail closed
|
||||
// but leave a diagnosable trace, because the storage error is
|
||||
// the generic `PartMissingOrCorrupt`.
|
||||
log_admission_rejected(bucket, object, "snapshot_quota_admission_missing");
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
(None, _) => None,
|
||||
},
|
||||
None => None,
|
||||
@@ -904,6 +918,18 @@ pub fn fail_next_quota_ledger_save_for_test() {
|
||||
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
|
||||
fn log_admission_rejected(bucket: &str, object: &str, state: &'static str) {
|
||||
warn!(
|
||||
event = EVENT_QUOTA_ADMISSION,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_QUOTA,
|
||||
state,
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
"quota admission rejected the write before commit"
|
||||
);
|
||||
}
|
||||
|
||||
fn log_deferred_settlement(data: &LedgerReservationData, state: &'static str, err: &StorageError) {
|
||||
warn!(
|
||||
event = EVENT_QUOTA_LEDGER_SETTLEMENT,
|
||||
|
||||
@@ -1170,6 +1170,11 @@ impl RemoteDisk {
|
||||
self.health.force_runtime_state_for_test(state);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
self.health.force_offline_for_test();
|
||||
}
|
||||
|
||||
/// Same as [`DiskHealthTracker::reset_for_store_init_retry`]: undo a transient faulty mark before another format load attempt.
|
||||
pub fn reset_health_for_store_init_retry(&self) {
|
||||
self.health.reset_for_store_init_retry(&self.endpoint);
|
||||
|
||||
@@ -47,10 +47,37 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
|
||||
|
||||
/// Work to run if an RPC that already timed out for its caller completes later.
|
||||
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
|
||||
type LateFailure = LateCompletion<()>;
|
||||
|
||||
struct RpcCleanup<T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
}
|
||||
|
||||
impl<T> RpcCleanup<T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
|
||||
Self {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The liveness window is this many RPC deadlines: a peer that completed a
|
||||
/// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
|
||||
const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
|
||||
const LOCK_RPC_REQUEST_BACKOFF_THRESHOLD: u32 = 3;
|
||||
const LOCK_RPC_REQUEST_BACKOFF_MAX: Duration = Duration::from_secs(30);
|
||||
const LATE_RELEASE_RETRY_DELAYS: [Duration; 3] = [Duration::from_millis(100), Duration::from_millis(500), Duration::from_secs(1)];
|
||||
const LATE_RELEASE_IN_FLIGHT_LIMIT: usize = 32;
|
||||
const TIMEOUT_LOG_STATE_MAX: usize = 1024;
|
||||
const TIMEOUT_LOG_STATE_RETENTION: Duration = Duration::from_secs(600);
|
||||
|
||||
fn is_request_breaker_operation(op: &'static str) -> bool {
|
||||
matches!(op, "lock" | "lock_batch")
|
||||
}
|
||||
|
||||
/// Recent history of the shared lock channel to one peer (issue #7363).
|
||||
///
|
||||
@@ -64,15 +91,44 @@ struct LockPeerChannelHealth {
|
||||
last_success: Option<Instant>,
|
||||
last_eviction: Option<Instant>,
|
||||
consecutive_timeouts: u32,
|
||||
request_consecutive_timeouts: u32,
|
||||
request_backoff_until: Option<Instant>,
|
||||
request_backoff: Duration,
|
||||
request_probe_generation: Option<u64>,
|
||||
next_probe_generation: u64,
|
||||
request_in_flight: usize,
|
||||
late_release_in_flight: usize,
|
||||
/// Timed-out RPCs still running in the background for this peer.
|
||||
detached_rpcs: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
struct TimeoutLogHealth {
|
||||
last_logged: Option<Instant>,
|
||||
suppressed: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct LockRequestAdmission {
|
||||
probe_generation: Option<u64>,
|
||||
}
|
||||
|
||||
impl LockRequestAdmission {
|
||||
fn is_probe(self) -> bool {
|
||||
self.probe_generation.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_peer_channel_health() -> &'static Mutex<HashMap<String, LockPeerChannelHealth>> {
|
||||
static HEALTH: OnceLock<Mutex<HashMap<String, LockPeerChannelHealth>>> = OnceLock::new();
|
||||
HEALTH.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
fn timeout_log_health() -> &'static Mutex<HashMap<(String, &'static str), TimeoutLogHealth>> {
|
||||
static HEALTH: OnceLock<Mutex<HashMap<(String, &'static str), TimeoutLogHealth>>> = OnceLock::new();
|
||||
HEALTH.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannelHealth) -> R) -> R {
|
||||
let mut peers = lock_peer_channel_health()
|
||||
.lock()
|
||||
@@ -343,17 +399,163 @@ impl RemoteClient {
|
||||
)
|
||||
}
|
||||
|
||||
fn request_limit() -> usize {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_LOCK_RPC_REQUEST_LIMIT,
|
||||
rustfs_config::DEFAULT_OBJECT_LOCK_RPC_REQUEST_LIMIT,
|
||||
)
|
||||
.max(1)
|
||||
}
|
||||
|
||||
fn liveness_window(deadline: Duration) -> Duration {
|
||||
deadline.saturating_mul(LOCK_RPC_LIVENESS_WINDOW_DEADLINES)
|
||||
}
|
||||
|
||||
fn record_rpc_success(&self) {
|
||||
fn record_rpc_success(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
health.consecutive_timeouts = 0;
|
||||
if is_request_breaker_operation(op) {
|
||||
let is_probe = admission.is_some_and(LockRequestAdmission::is_probe);
|
||||
if health.request_backoff_until.is_none() || is_probe {
|
||||
health.request_consecutive_timeouts = 0;
|
||||
health.request_backoff_until = None;
|
||||
health.request_backoff = Duration::ZERO;
|
||||
}
|
||||
if is_probe && admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
|
||||
health.request_probe_generation = None;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn admit_late_release(&self) -> bool {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
if health.late_release_in_flight >= LATE_RELEASE_IN_FLIGHT_LIMIT {
|
||||
false
|
||||
} else {
|
||||
health.late_release_in_flight += 1;
|
||||
true
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn release_late_release(&self) {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.late_release_in_flight = health.late_release_in_flight.saturating_sub(1);
|
||||
});
|
||||
}
|
||||
|
||||
/// Admit a new lock acquisition unless this peer is in the request-level
|
||||
/// backoff window. A single half-open probe is allowed after the window;
|
||||
/// channel liveness and request admission are deliberately separate.
|
||||
fn admit_lock_request(&self, op: &'static str) -> Option<LockRequestAdmission> {
|
||||
if !is_request_breaker_operation(op) {
|
||||
return Some(LockRequestAdmission { probe_generation: None });
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
let (admission, suppression_reason) = with_lock_peer_health(&self.addr, |health| {
|
||||
let degraded = health.request_backoff_until.is_some() || health.request_consecutive_timeouts > 0;
|
||||
if degraded && health.request_in_flight >= Self::request_limit() {
|
||||
return (None, "in_flight_limit");
|
||||
}
|
||||
let Some(until) = health.request_backoff_until else {
|
||||
health.request_in_flight += 1;
|
||||
return (Some(LockRequestAdmission { probe_generation: None }), "none");
|
||||
};
|
||||
if now < until || health.request_probe_generation.is_some() {
|
||||
return (None, "breaker_open");
|
||||
}
|
||||
health.next_probe_generation = health.next_probe_generation.saturating_add(1);
|
||||
let generation = health.next_probe_generation;
|
||||
health.request_probe_generation = Some(generation);
|
||||
health.request_in_flight += 1;
|
||||
(
|
||||
Some(LockRequestAdmission {
|
||||
probe_generation: Some(generation),
|
||||
}),
|
||||
"none",
|
||||
)
|
||||
});
|
||||
if admission.is_none() {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_request_suppressed(&self.addr, op, suppression_reason);
|
||||
}
|
||||
admission
|
||||
}
|
||||
|
||||
fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.request_in_flight = health.request_in_flight.saturating_sub(1);
|
||||
if admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
|
||||
health.request_probe_generation = None;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn record_request_timeout(&self, op: &'static str, deadline: Duration) {
|
||||
if !is_request_breaker_operation(op) {
|
||||
return;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
health.request_consecutive_timeouts = health.request_consecutive_timeouts.saturating_add(1);
|
||||
if health.request_consecutive_timeouts < LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
|
||||
return;
|
||||
}
|
||||
|
||||
let initial = deadline / 2;
|
||||
let next = if health.request_backoff.is_zero() {
|
||||
initial.max(Duration::from_millis(1))
|
||||
} else {
|
||||
health.request_backoff.saturating_mul(2)
|
||||
};
|
||||
health.request_backoff = next.min(LOCK_RPC_REQUEST_BACKOFF_MAX);
|
||||
health.request_backoff_until = Some(now + health.request_backoff);
|
||||
});
|
||||
}
|
||||
|
||||
fn timeout_log_decision(&self, op: &'static str, interval: Duration) -> (bool, u64) {
|
||||
let now = Instant::now();
|
||||
let mut peers = timeout_log_health().lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if peers.len() >= TIMEOUT_LOG_STATE_MAX {
|
||||
peers.retain(|_, state| {
|
||||
state
|
||||
.last_logged
|
||||
.is_some_and(|at| now.saturating_duration_since(at) < TIMEOUT_LOG_STATE_RETENTION)
|
||||
});
|
||||
}
|
||||
let key = (self.addr.clone(), op);
|
||||
if peers.len() >= TIMEOUT_LOG_STATE_MAX
|
||||
&& !peers.contains_key(&key)
|
||||
&& let Some(eviction_key) = peers.keys().next().cloned()
|
||||
{
|
||||
peers.remove(&eviction_key);
|
||||
}
|
||||
let state = peers.entry(key).or_default();
|
||||
if state
|
||||
.last_logged
|
||||
.is_some_and(|at| now.saturating_duration_since(at) < interval)
|
||||
{
|
||||
state.suppressed = state.suppressed.saturating_add(1);
|
||||
return (false, state.suppressed);
|
||||
}
|
||||
|
||||
let suppressed = state.suppressed;
|
||||
state.last_logged = Some(now);
|
||||
state.suppressed = 0;
|
||||
(true, suppressed)
|
||||
}
|
||||
|
||||
fn bounded_acquire_request(request: &LockRequest, deadline: Duration) -> LockRequest {
|
||||
let mut bounded = request.clone();
|
||||
bounded.acquire_timeout = bounded.acquire_timeout.min(deadline);
|
||||
bounded
|
||||
}
|
||||
|
||||
/// Apply the per-peer eviction policy after a failed RPC.
|
||||
async fn maybe_evict_connection(
|
||||
&self,
|
||||
@@ -406,6 +608,8 @@ impl RemoteClient {
|
||||
resource_summary: &str,
|
||||
handle: JoinHandle<std::result::Result<T, tonic::Status>>,
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
) {
|
||||
let limit = Self::detached_rpc_limit();
|
||||
let admitted = with_lock_peer_health(&self.addr, |health| {
|
||||
@@ -418,6 +622,10 @@ impl RemoteClient {
|
||||
});
|
||||
if !admitted {
|
||||
handle.abort();
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -432,12 +640,19 @@ impl RemoteClient {
|
||||
let addr = self.addr.clone();
|
||||
tokio::spawn(async move {
|
||||
let outcome = handle.await;
|
||||
with_lock_peer_health(&addr, |health| health.detached_rpcs = health.detached_rpcs.saturating_sub(1));
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
health.request_in_flight = health.request_in_flight.saturating_sub(1);
|
||||
if admission.is_some_and(|admission| health.request_probe_generation == admission.probe_generation) {
|
||||
health.request_probe_generation = None;
|
||||
}
|
||||
}
|
||||
});
|
||||
match outcome {
|
||||
Ok(Ok(response)) => {
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
health.consecutive_timeouts = 0;
|
||||
});
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
|
||||
if let Some(late) = late {
|
||||
@@ -445,6 +660,9 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
Ok(Err(status)) => {
|
||||
if let Some(late_failure) = late_failure {
|
||||
late_failure(()).await;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
|
||||
debug!(
|
||||
addr = %addr,
|
||||
@@ -455,6 +673,9 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
Err(join_error) => {
|
||||
if let Some(late_failure) = late_failure {
|
||||
late_failure(()).await;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
|
||||
debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
|
||||
}
|
||||
@@ -485,24 +706,79 @@ impl RemoteClient {
|
||||
}))
|
||||
}
|
||||
|
||||
fn late_release_failure_hook(&self, lock_ids: Vec<LockId>) -> LateFailure {
|
||||
let client = self.clone();
|
||||
Some(Box::new(move |_| {
|
||||
Box::pin(async move {
|
||||
client.release_late_acquisitions(lock_ids).await;
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
/// A lock granted after its caller stopped waiting is an orphan until its
|
||||
/// lease expires; hand it back right away, best effort.
|
||||
async fn release_late_acquisitions(&self, lock_ids: Vec<LockId>) {
|
||||
let outcome = match self.release_locks_batch(&lock_ids).await {
|
||||
Ok(released) if released.iter().all(|released| *released) => "released",
|
||||
Ok(_) => "partial",
|
||||
Err(_) => "failed",
|
||||
};
|
||||
async fn release_late_acquisitions(&self, mut lock_ids: Vec<LockId>) {
|
||||
if !self.admit_late_release() {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release("failed");
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
count = lock_ids.len(),
|
||||
outcome = "budget_exhausted",
|
||||
suppressed_timeouts,
|
||||
"Skipped late remote lock release because the cleanup budget is exhausted"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let original_count = lock_ids.len();
|
||||
let mut outcome = "failed";
|
||||
for (attempt, delay) in std::iter::once(Duration::ZERO).chain(LATE_RELEASE_RETRY_DELAYS).enumerate() {
|
||||
if !delay.is_zero() {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
|
||||
match self.release_locks_batch(&lock_ids).await {
|
||||
Ok(released) => {
|
||||
lock_ids = lock_ids
|
||||
.into_iter()
|
||||
.zip(released)
|
||||
.filter_map(|(lock_id, released)| (!released).then_some(lock_id))
|
||||
.collect();
|
||||
if lock_ids.is_empty() {
|
||||
outcome = "released";
|
||||
break;
|
||||
}
|
||||
outcome = if attempt == LATE_RELEASE_RETRY_DELAYS.len() {
|
||||
"partial"
|
||||
} else {
|
||||
"failed"
|
||||
};
|
||||
}
|
||||
Err(_) => {
|
||||
outcome = "failed";
|
||||
}
|
||||
}
|
||||
}
|
||||
self.release_late_release();
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
|
||||
if outcome == "released" {
|
||||
debug!(addr = %self.addr, count = lock_ids.len(), "Released remote locks granted after their caller timed out");
|
||||
debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
|
||||
} else {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
count = lock_ids.len(),
|
||||
outcome,
|
||||
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
|
||||
);
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
outcome,
|
||||
suppressed_timeouts,
|
||||
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
|
||||
);
|
||||
} else {
|
||||
debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -512,26 +788,74 @@ impl RemoteClient {
|
||||
resource_summary: &str,
|
||||
deadline: Duration,
|
||||
future: Fut,
|
||||
late: LateCompletion<T>,
|
||||
cleanup: RpcCleanup<T>,
|
||||
) -> std::result::Result<T, LockError>
|
||||
where
|
||||
Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let RpcCleanup {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
} = cleanup;
|
||||
let mut handle = tokio::spawn(future);
|
||||
match timeout(deadline, &mut handle).await {
|
||||
Ok(Ok(Ok(response))) => {
|
||||
self.record_rpc_success();
|
||||
self.record_rpc_success(op, admission);
|
||||
self.release_lock_request(op, admission);
|
||||
Ok(response)
|
||||
}
|
||||
Ok(Ok(Err(err))) => {
|
||||
let reason = err.to_string();
|
||||
if err.code() == tonic::Code::DeadlineExceeded {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
self.record_request_timeout(op, deadline);
|
||||
let (log_timeout, suppressed_timeouts) = self.timeout_log_decision(op, deadline);
|
||||
if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
"Remote lock RPC returned deadline exceeded for scanner leader lock"
|
||||
);
|
||||
} else if log_timeout {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Remote lock RPC returned deadline exceeded"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Suppressed repeated remote lock RPC deadline"
|
||||
);
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
}
|
||||
// Only evict (and re-dial) the cached channel when the failure is a genuine
|
||||
// transport problem. A server-produced application status (auth denied, peer
|
||||
// lock service not ready, invalid args, ...) arrives on a perfectly healthy
|
||||
// channel; evicting it just churns the connection and pushes the peer toward
|
||||
// the offline threshold for no benefit. See issue #4567.
|
||||
let transport_failure = Self::is_transport_failure(&err);
|
||||
if transport_failure {
|
||||
self.record_request_timeout(op, deadline);
|
||||
}
|
||||
if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -543,6 +867,31 @@ impl RemoteClient {
|
||||
transport_failure,
|
||||
"Remote lock RPC returned tonic error for scanner leader lock"
|
||||
);
|
||||
} else if transport_failure {
|
||||
let (log_failure, suppressed_failures) = self.timeout_log_decision(op, deadline);
|
||||
if log_failure {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
tonic_code = ?err.code(),
|
||||
tonic_message = err.message(),
|
||||
transport_failure,
|
||||
suppressed_failures,
|
||||
"Remote lock RPC returned transport error"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
tonic_code = ?err.code(),
|
||||
suppressed_failures,
|
||||
"Suppressed repeated remote lock transport error"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
@@ -559,6 +908,10 @@ impl RemoteClient {
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
|
||||
.await;
|
||||
}
|
||||
self.release_lock_request(op, admission);
|
||||
if transport_failure && let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC failed: {reason}")))
|
||||
}
|
||||
Ok(Err(join_error)) => {
|
||||
@@ -569,11 +922,17 @@ impl RemoteClient {
|
||||
error = %join_error,
|
||||
"Remote lock RPC task ended abnormally"
|
||||
);
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
|
||||
}
|
||||
Err(_) => {
|
||||
let reason = format!("RPC timed out after {deadline:?}");
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
self.record_request_timeout(op, deadline);
|
||||
let (log_timeout, suppressed_timeouts) = self.timeout_log_decision(op, deadline);
|
||||
if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -582,18 +941,28 @@ impl RemoteClient {
|
||||
resource_summary,
|
||||
"Remote lock RPC timed out for scanner leader lock"
|
||||
);
|
||||
} else {
|
||||
} else if log_timeout {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Remote lock RPC timed out"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = deadline.as_millis(),
|
||||
resource_summary,
|
||||
suppressed_timeouts,
|
||||
"Suppressed repeated remote lock RPC timeout"
|
||||
);
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.detach_timed_out_rpc(op, resource_summary, handle, late);
|
||||
self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
|
||||
Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
|
||||
}
|
||||
}
|
||||
@@ -621,6 +990,14 @@ impl RemoteClient {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn rpc_backoff_failure_response(request: &LockRequest) -> LockResponse {
|
||||
LockResponse::failure("Remote lock RPC timed out: peer request breaker is open", request.acquire_timeout)
|
||||
}
|
||||
|
||||
fn rpc_backoff_failure_batch(requests: &[LockRequest]) -> Vec<LockResponse> {
|
||||
requests.iter().map(Self::rpc_backoff_failure_response).collect()
|
||||
}
|
||||
|
||||
fn build_lock_info(request: &LockRequest, lock_info_json: Option<String>) -> LockInfo {
|
||||
if let Some(lock_info_json) = lock_info_json {
|
||||
match serde_json::from_str::<LockInfo>(&lock_info_json) {
|
||||
@@ -680,22 +1057,45 @@ impl RemoteClient {
|
||||
impl LockClient for RemoteClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
info!("remote acquire_exclusive for {}", request.resource);
|
||||
let mut client = self.get_client().await?;
|
||||
let admission = match self.admit_lock_request("lock") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_response(request)),
|
||||
};
|
||||
|
||||
let rpc_timeout = Self::rpc_timeout();
|
||||
let bounded_request = Self::bounded_acquire_request(request, rpc_timeout);
|
||||
let mut client = match self.get_client().await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock", rpc_timeout);
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let args = match serde_json::to_string(&bounded_request) {
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(LockError::internal(format!("Failed to serialize request: {err}")));
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(GenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let late = self.late_release_hook(request.lock_id.clone());
|
||||
let late_failure = self.late_release_failure_hook(vec![request.lock_id.clone()]);
|
||||
|
||||
let resp = match self
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
rpc_timeout,
|
||||
async move { client.lock(req).await },
|
||||
late,
|
||||
RpcCleanup::new(late, late_failure, Some(admission)),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -725,26 +1125,54 @@ impl LockClient for RemoteClient {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut client = self.get_client().await?;
|
||||
let admission = match self.admit_lock_request("lock_batch") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_batch(requests)),
|
||||
};
|
||||
|
||||
let rpc_timeout = Self::rpc_timeout();
|
||||
let bounded_requests = requests
|
||||
.iter()
|
||||
.map(|request| Self::bounded_acquire_request(request, rpc_timeout))
|
||||
.collect::<Vec<_>>();
|
||||
let mut client = match self.get_client().await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock_batch", rpc_timeout);
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let resource_summary = Self::summarize_resources(requests);
|
||||
let mut req = Request::new(BatchGenerallyLockRequest {
|
||||
args: requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let args = match bounded_requests
|
||||
.iter()
|
||||
.map(|request| {
|
||||
serde_json::to_string(request).map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()
|
||||
{
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(BatchGenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
|
||||
let late_failure = self.late_release_failure_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
|
||||
|
||||
let resp = match self
|
||||
.execute_rpc(
|
||||
"lock_batch",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
rpc_timeout,
|
||||
async move { client.lock_batch(req).await },
|
||||
late,
|
||||
RpcCleanup::new(late, late_failure, Some(admission)),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -791,7 +1219,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock(req).await },
|
||||
None,
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -825,7 +1253,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock_batch(req).await },
|
||||
None,
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -853,7 +1281,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
None,
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -879,7 +1307,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.force_un_lock(req).await },
|
||||
None,
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
@@ -894,26 +1322,31 @@ impl LockClient for RemoteClient {
|
||||
|
||||
// Since there's no direct status query in the gRPC service,
|
||||
// we attempt a non-blocking lock acquisition to check if the resource is available
|
||||
let status_request = Self::create_unlock_request(lock_id);
|
||||
let probe_lock_id = LockId::new_unique(&lock_id.resource);
|
||||
let rpc_timeout = Self::rpc_timeout();
|
||||
let status_request = Self::create_unlock_request(&probe_lock_id).with_ttl(rpc_timeout);
|
||||
let bounded_status_request = Self::bounded_acquire_request(&status_request, rpc_timeout);
|
||||
let resource_summary = status_request.resource.to_string();
|
||||
let mut client = self.get_client().await?;
|
||||
let args = serde_json::to_string(&status_request)
|
||||
let args = serde_json::to_string(&bounded_status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
|
||||
// Try to acquire a very short-lived lock to test availability
|
||||
let mut req = Request::new(GenerallyLockRequest { args: args.clone() });
|
||||
req.set_timeout(rpc_timeout);
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
// A probe lock granted after the deadline must not linger on the peer.
|
||||
let late = self.late_release_hook(lock_id.clone());
|
||||
let late = self.late_release_hook(probe_lock_id.clone());
|
||||
let late_failure = self.late_release_failure_hook(vec![probe_lock_id]);
|
||||
|
||||
// Try exclusive lock first with very short timeout
|
||||
let resp = match self
|
||||
.execute_rpc(
|
||||
"check_status",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
rpc_timeout,
|
||||
async move { client.lock(req).await },
|
||||
late,
|
||||
RpcCleanup::new(late, late_failure, None),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -925,6 +1358,7 @@ impl LockClient for RemoteClient {
|
||||
// If we successfully acquired the lock, the resource was free.
|
||||
// Immediately release it on a best-effort basis.
|
||||
let mut release_req = Request::new(GenerallyLockRequest { args });
|
||||
release_req.set_timeout(rpc_timeout);
|
||||
attach_lock_mutation_body_digest(&mut release_req)?;
|
||||
if let Ok(mut client) = self.get_client().await {
|
||||
let _ = self
|
||||
@@ -933,7 +1367,7 @@ impl LockClient for RemoteClient {
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.un_lock(release_req).await },
|
||||
None,
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -998,7 +1432,7 @@ impl LockClient for RemoteClient {
|
||||
Self::ONLINE_CHECK_RESOURCE,
|
||||
online_timeout,
|
||||
async move { client.ping(ping_req).await },
|
||||
None,
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1133,6 +1567,94 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_breaker_opens_after_three_timeouts_and_allows_one_probe() {
|
||||
let addr = "http://breaker-test";
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
|
||||
for _ in 0..LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
|
||||
client.record_request_timeout("lock", Duration::from_millis(100));
|
||||
}
|
||||
|
||||
assert!(
|
||||
client.admit_lock_request("lock").is_none(),
|
||||
"open breaker must reject requests during backoff"
|
||||
);
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
health.request_probe_generation = None;
|
||||
health.request_in_flight = 0;
|
||||
});
|
||||
|
||||
let probe = client
|
||||
.admit_lock_request("lock")
|
||||
.expect("half-open breaker must admit one probe");
|
||||
assert!(probe.is_probe());
|
||||
assert!(client.admit_lock_request("lock").is_none(), "only one half-open probe may run");
|
||||
client.release_lock_request("lock", Some(probe));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_none());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_request_completion_cannot_clear_a_new_probe() {
|
||||
let addr = "http://breaker-token-test";
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
health.request_probe_generation = None;
|
||||
health.request_in_flight = 1;
|
||||
});
|
||||
|
||||
let probe = client.admit_lock_request("lock").expect("probe should be admitted");
|
||||
client.release_lock_request("lock", Some(LockRequestAdmission { probe_generation: None }));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_some());
|
||||
client.record_rpc_success("lock", Some(LockRequestAdmission { probe_generation: None }));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_some());
|
||||
client.release_lock_request("lock", Some(probe));
|
||||
assert!(lock_peer_health_for_test(addr).request_probe_generation.is_none());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_probe_completion_cannot_clear_a_later_generation() {
|
||||
let addr = "http://breaker-generation-test";
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
});
|
||||
|
||||
let first = client.admit_lock_request("lock").expect("first probe should be admitted");
|
||||
client.record_request_timeout("lock", Duration::from_millis(10));
|
||||
with_lock_peer_health(addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_millis(1));
|
||||
health.request_in_flight = 0;
|
||||
health.request_probe_generation = None;
|
||||
health.request_consecutive_timeouts = 0;
|
||||
});
|
||||
let second = client.admit_lock_request("lock").expect("second probe should be admitted");
|
||||
assert_ne!(first.probe_generation, second.probe_generation);
|
||||
|
||||
client.release_lock_request("lock", Some(first));
|
||||
assert_eq!(lock_peer_health_for_test(addr).request_probe_generation, second.probe_generation);
|
||||
client.release_lock_request("lock", Some(second));
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_acquire_request_matches_transport_deadline() {
|
||||
let request = test_lock_request(Duration::from_secs(30));
|
||||
let bounded = RemoteClient::bounded_acquire_request(&request, Duration::from_secs(3));
|
||||
assert_eq!(bounded.acquire_timeout, Duration::from_secs(3));
|
||||
|
||||
let request = test_lock_request(Duration::from_secs(1));
|
||||
let bounded = RemoteClient::bounded_acquire_request(&request, Duration::from_secs(3));
|
||||
assert_eq!(bounded.acquire_timeout, Duration::from_secs(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acquired_lock_ids_picks_only_granted_batch_entries() {
|
||||
let lock_ids = vec![
|
||||
|
||||
@@ -147,6 +147,11 @@ const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
pub(crate) const DECOMMISSION_ENTRY_MAX_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS: usize = 12;
|
||||
// A scanner pause backlog handoff conflict is a transient membership
|
||||
// transition: the native authority is expected to converge once surviving
|
||||
// replicas advance their durable commit. Bound the wait so a genuinely
|
||||
// divergent authority still reaches a terminal, attributable failure.
|
||||
const DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS: usize = 12;
|
||||
const DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(100);
|
||||
pub(crate) const DECOMMISSION_VERSION_COPY_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50);
|
||||
@@ -996,6 +1001,30 @@ fn decommission_capacity_retry_kind(err: &Error, intent_conflict_attempt: usize)
|
||||
.then_some(DecommissionCapacityRetryKind::IntentConflict)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum DecommissionScannerBacklogRetryKind {
|
||||
HandoffConflict,
|
||||
}
|
||||
|
||||
fn is_decommission_scanner_backlog_handoff_conflict(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<data_movement::scanner_backlog::ScannerPauseBacklogRetirementError>(err)
|
||||
.is_some_and(data_movement::scanner_backlog::ScannerPauseBacklogRetirementError::is_retryable_handoff_conflict)
|
||||
}
|
||||
|
||||
/// Classify a retryable scanner pause backlog handoff conflict.
|
||||
///
|
||||
/// The typed planner error is the only signal consulted: string matching would
|
||||
/// turn unrelated retirement failures into retries. `handoff_conflict_attempt`
|
||||
/// is the number of retries already consumed by this entry.
|
||||
fn decommission_scanner_backlog_retry_kind(
|
||||
err: &Error,
|
||||
handoff_conflict_attempt: usize,
|
||||
) -> Option<DecommissionScannerBacklogRetryKind> {
|
||||
(handoff_conflict_attempt < DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS
|
||||
&& is_decommission_scanner_backlog_handoff_conflict(err))
|
||||
.then_some(DecommissionScannerBacklogRetryKind::HandoffConflict)
|
||||
}
|
||||
|
||||
fn ensure_decommission_capacity_target_fence(
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
target_pool_index: usize,
|
||||
@@ -14058,6 +14087,7 @@ impl ECStore {
|
||||
for entry_attempt in 1..=DECOMMISSION_ENTRY_MAX_ATTEMPTS {
|
||||
let attempt_result = {
|
||||
let mut conflict_attempt = 0;
|
||||
let mut scanner_backlog_handoff_attempt = 0;
|
||||
loop {
|
||||
let result = self
|
||||
.decommission_entry_attempt(
|
||||
@@ -14072,6 +14102,7 @@ impl ECStore {
|
||||
replication_config.clone(),
|
||||
expected_bucket_incarnation_id,
|
||||
entry_attempt,
|
||||
scanner_backlog_handoff_attempt,
|
||||
source_changed_exhaustions.as_ref(),
|
||||
&mut counted_versions,
|
||||
)
|
||||
@@ -14080,17 +14111,48 @@ impl ECStore {
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|err| decommission_capacity_retry_kind(err, conflict_attempt));
|
||||
let retry_attempt = match retry {
|
||||
Some(DecommissionCapacityRetryKind::IntentConflict) => {
|
||||
conflict_attempt += 1;
|
||||
conflict_attempt
|
||||
if let Some(DecommissionCapacityRetryKind::IntentConflict) = retry {
|
||||
conflict_attempt += 1;
|
||||
let retry_delay =
|
||||
decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, conflict_attempt);
|
||||
if wait_decommission_retry_backoff(&rx, retry_delay).await {
|
||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||
}
|
||||
None => break result,
|
||||
};
|
||||
let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, retry_attempt);
|
||||
if wait_decommission_retry_backoff(&rx, retry_delay).await {
|
||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||
continue;
|
||||
}
|
||||
// Scanner pause backlog handoff conflicts resolve when the
|
||||
// surviving membership converges. Retry on a bounded
|
||||
// backoff that never writes failure state, clears
|
||||
// `start_time`, or releases the capacity reservation.
|
||||
let scanner_retry = result
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|err| decommission_scanner_backlog_retry_kind(err, scanner_backlog_handoff_attempt));
|
||||
if let Some(DecommissionScannerBacklogRetryKind::HandoffConflict) = scanner_retry {
|
||||
scanner_backlog_handoff_attempt += 1;
|
||||
let retry_delay = decommission_retry_backoff_delay(
|
||||
DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY,
|
||||
scanner_backlog_handoff_attempt,
|
||||
);
|
||||
info!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
state = "scanner_backlog_handoff_retry",
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
attempt = scanner_backlog_handoff_attempt,
|
||||
max_attempts = DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS,
|
||||
retry_delay_ms = retry_delay.as_millis(),
|
||||
"Decommission scanner pause backlog handoff conflict; retrying entry"
|
||||
);
|
||||
if wait_decommission_retry_backoff(&rx, retry_delay).await {
|
||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
break result;
|
||||
}
|
||||
};
|
||||
match attempt_result {
|
||||
@@ -14138,6 +14200,7 @@ impl ECStore {
|
||||
replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
entry_attempt: usize,
|
||||
scanner_backlog_handoff_attempt: usize,
|
||||
source_changed_exhaustions: &AtomicUsize,
|
||||
counted_versions: &mut HashSet<(Option<uuid::Uuid>, bool)>,
|
||||
) -> Result<DecommissionEntryAttemptOutcome> {
|
||||
@@ -14197,27 +14260,56 @@ impl ECStore {
|
||||
if data_movement::scanner_backlog::is_scanner_pause_backlog(&bucket, &entry.name) {
|
||||
let outcome = self
|
||||
.retire_scanner_pause_backlog_entry(rx, idx, generation, Arc::clone(&set), fivs.clone(), capacity_owner)
|
||||
.await?;
|
||||
if matches!(outcome, DecommissionEntryAttemptOutcome::Complete) {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
ensure_decommission_generation(&pool_meta, idx, generation)?;
|
||||
if let Some(version) = fivs.versions.first()
|
||||
&& counted_versions.insert((version.version_id, false))
|
||||
{
|
||||
count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), false)?;
|
||||
.await;
|
||||
return match outcome {
|
||||
Ok(DecommissionEntryAttemptOutcome::Complete) => {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
ensure_decommission_generation(&pool_meta, idx, generation)?;
|
||||
if let Some(version) = fivs.versions.first()
|
||||
&& counted_versions.insert((version.version_id, false))
|
||||
{
|
||||
count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), false)?;
|
||||
}
|
||||
track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
|
||||
drop(pool_meta);
|
||||
self.track_decommission_entry_progress_stage(
|
||||
idx,
|
||||
generation,
|
||||
&bucket,
|
||||
&entry.name,
|
||||
DECOMMISSION_STAGE_ENTRY_FINISHED,
|
||||
)
|
||||
.await?;
|
||||
Ok(DecommissionEntryAttemptOutcome::Complete)
|
||||
}
|
||||
track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
|
||||
drop(pool_meta);
|
||||
self.track_decommission_entry_progress_stage(
|
||||
idx,
|
||||
generation,
|
||||
&bucket,
|
||||
&entry.name,
|
||||
DECOMMISSION_STAGE_ENTRY_FINISHED,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
return Ok(outcome);
|
||||
Ok(outcome) => Ok(outcome),
|
||||
Err(err) => {
|
||||
// The retry loop owns the handoff-conflict budget, so a
|
||||
// retryable conflict must not be attributed here. Once that
|
||||
// budget is exhausted the failure is terminal and belongs to
|
||||
// this entry rather than to whichever object was processed
|
||||
// last.
|
||||
let retryable_handoff = data_movement::data_movement_stage_source_as::<
|
||||
data_movement::scanner_backlog::ScannerPauseBacklogRetirementError,
|
||||
>(&err)
|
||||
.is_some_and(
|
||||
data_movement::scanner_backlog::ScannerPauseBacklogRetirementError::is_retryable_handoff_conflict,
|
||||
);
|
||||
let terminal = !retryable_handoff
|
||||
|| scanner_backlog_handoff_attempt >= DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS;
|
||||
if terminal {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
ensure_decommission_generation(&pool_meta, idx, generation)?;
|
||||
if let Some(version) = fivs.versions.first()
|
||||
&& counted_versions.insert((version.version_id, true))
|
||||
{
|
||||
count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), true)?;
|
||||
}
|
||||
track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let pending_mutations = if let Some(owner) = capacity_owner {
|
||||
@@ -21687,6 +21779,45 @@ mod tests {
|
||||
assert!(!is_decommission_target_capacity_error(&Error::SlowDown));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_scanner_backlog_handoff_retry_is_typed_and_bounded() {
|
||||
use data_movement::scanner_backlog::ScannerPauseBacklogRetirementError;
|
||||
|
||||
let handoff = data_movement::data_movement_context_error(
|
||||
"scanner pause backlog retirement planner failed: competing proofs".to_string(),
|
||||
ScannerPauseBacklogRetirementError::HandoffConflict {
|
||||
reason: "competing maximum-membership commit proofs".to_string(),
|
||||
},
|
||||
);
|
||||
assert!(is_decommission_scanner_backlog_handoff_conflict(&handoff));
|
||||
assert_eq!(
|
||||
decommission_scanner_backlog_retry_kind(&handoff, DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS - 1),
|
||||
Some(DecommissionScannerBacklogRetryKind::HandoffConflict)
|
||||
);
|
||||
assert_eq!(
|
||||
decommission_scanner_backlog_retry_kind(&handoff, DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS),
|
||||
None,
|
||||
"a divergent handoff must not retry forever"
|
||||
);
|
||||
|
||||
for non_retryable in [
|
||||
ScannerPauseBacklogRetirementError::AuthorityConflict {
|
||||
reason: "larger source authority".to_string(),
|
||||
},
|
||||
ScannerPauseBacklogRetirementError::InvalidRecord {
|
||||
reason: "unread member".to_string(),
|
||||
},
|
||||
] {
|
||||
let err = data_movement::data_movement_context_error("planner failed".to_string(), non_retryable);
|
||||
assert!(!is_decommission_scanner_backlog_handoff_conflict(&err));
|
||||
assert_eq!(
|
||||
decommission_scanner_backlog_retry_kind(&err, 0),
|
||||
None,
|
||||
"only a handoff conflict may be retried"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_skip_decommission_delete_marker_characterizes_empty_marker_without_replication() {
|
||||
let version = rustfs_filemeta::FileInfo {
|
||||
|
||||
@@ -481,6 +481,9 @@ mod decommission_lock_order_tests {
|
||||
if let Some(deployment_id) = other_store.ctx.deployment_id() {
|
||||
ctx.set_deployment_id(deployment_id);
|
||||
}
|
||||
for pool in &mut pools {
|
||||
Arc::make_mut(pool).set_instance_ctx_for_test(Arc::clone(&ctx));
|
||||
}
|
||||
let store = Arc::new(crate::store::ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
disk_map: other_store.disk_map.clone(),
|
||||
@@ -6645,7 +6648,19 @@ mod decommission_lock_order_tests {
|
||||
|
||||
let restore_get_barrier = if matches!(mutation, ExternalObjectMutation::Restore) {
|
||||
let tier_name = format!("ORDERRESTORE{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&store.pools[2].instance_ctx().tier_config_mgr(), &tier_name).await;
|
||||
let source_tiers = store.tier_config_mgr();
|
||||
let restore_tiers = other_store.tier_config_mgr();
|
||||
let backend = register_mock_tier(&source_tiers, &tier_name).await;
|
||||
// Both peers must resolve the same persisted backend identity, including its prefix.
|
||||
let tier_config = source_tiers.read().await.tiers[&tier_name].clone();
|
||||
restore_tiers.write().await.tiers.insert(tier_name.clone(), tier_config);
|
||||
crate::services::tier::tier::TierConfigMgr::install_test_driver_in(
|
||||
&restore_tiers,
|
||||
&tier_name,
|
||||
Box::new(backend.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("install the same mock tier on the restoring peer");
|
||||
store.pools[2]
|
||||
.transition_object(
|
||||
&bucket,
|
||||
@@ -6919,7 +6934,10 @@ mod decommission_lock_order_tests {
|
||||
}
|
||||
});
|
||||
if let Some(get_barrier) = restore_get_barrier.as_ref() {
|
||||
get_barrier.wait_until_paused().await;
|
||||
tokio::select! {
|
||||
() = get_barrier.wait_until_paused() => {}
|
||||
result = &mut ordinary_mutation => panic!("restore finished before the tier GET barrier: {result:?}"),
|
||||
}
|
||||
let read_opts = ObjectOptions {
|
||||
skip_decommissioned: true,
|
||||
..Default::default()
|
||||
|
||||
@@ -324,6 +324,15 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
/// Keep simulated peers' metadata ownership separate while sharing disks and lock clients.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
for set in &mut self.disk_set {
|
||||
Arc::make_mut(set).set_instance_ctx_for_test(Arc::clone(&ctx));
|
||||
}
|
||||
self.ctx = ctx;
|
||||
}
|
||||
|
||||
async fn monitor_and_connect_endpoints_task(sets: Weak<Sets>, mut rx: Receiver<()>) {
|
||||
let startup_delay = tokio::time::sleep(Duration::from_secs(5));
|
||||
tokio::pin!(startup_delay);
|
||||
@@ -790,6 +799,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
version_id: dst_opts.version_id.clone(),
|
||||
mod_time: dst_opts.mod_time,
|
||||
http_preconditions: dst_opts.http_preconditions.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -642,6 +642,21 @@ pub(crate) fn data_movement_stage_source(err: &Error) -> Option<&Error> {
|
||||
.downcast_ref::<Error>()
|
||||
}
|
||||
|
||||
/// Recover a concrete typed error wrapped by [`data_movement_stage_error`].
|
||||
pub(crate) fn data_movement_stage_source_as<T>(err: &Error) -> Option<&T>
|
||||
where
|
||||
T: std::error::Error + 'static,
|
||||
{
|
||||
let Error::Io(io_err) = err else {
|
||||
return None;
|
||||
};
|
||||
io_err
|
||||
.get_ref()?
|
||||
.downcast_ref::<DataMovementStageError>()?
|
||||
.source
|
||||
.downcast_ref::<T>()
|
||||
}
|
||||
|
||||
fn schedule_data_movement_multipart_abort_cleanup(
|
||||
store: Arc<ECStore>,
|
||||
target_pool_idx: usize,
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::data_movement::data_movement_context_error;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -27,6 +28,28 @@ use tokio::io::AsyncReadExt;
|
||||
pub const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024;
|
||||
pub(crate) const SCANNER_PAUSE_BACKLOG_PATH: &str = "buckets/.scanner-pause-backlog.json";
|
||||
|
||||
/// Typed failure from the scanner-owned native retirement planner.
|
||||
///
|
||||
/// Decommission must preserve the source when the native authority cannot be
|
||||
/// proven. The handoff variant is deliberately narrower than other authority
|
||||
/// failures so the worker can retry without converting a transient membership
|
||||
/// transition into a terminal pool failure.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ScannerPauseBacklogRetirementError {
|
||||
#[error("scanner pause backlog retirement has a retryable handoff conflict: {reason}")]
|
||||
HandoffConflict { reason: String },
|
||||
#[error("scanner pause backlog retirement has an unsafe authority conflict: {reason}")]
|
||||
AuthorityConflict { reason: String },
|
||||
#[error("scanner pause backlog retirement found an invalid native record: {reason}")]
|
||||
InvalidRecord { reason: String },
|
||||
}
|
||||
|
||||
impl ScannerPauseBacklogRetirementError {
|
||||
pub fn is_retryable_handoff_conflict(&self) -> bool {
|
||||
matches!(self, Self::HandoffConflict { .. })
|
||||
}
|
||||
}
|
||||
|
||||
/// A bounded, storage-fenced native replica. Only a confirmed missing object
|
||||
/// has no payload; read failures never enter the Scanner verifier.
|
||||
pub struct ScannerPauseBacklogRetirementReplica {
|
||||
@@ -44,7 +67,10 @@ pub struct ScannerPauseBacklogRetirementPlan {
|
||||
}
|
||||
|
||||
pub type ScannerPauseBacklogRetirementPlanner =
|
||||
fn(usize, &[ScannerPauseBacklogRetirementReplica]) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, String>;
|
||||
fn(
|
||||
usize,
|
||||
&[ScannerPauseBacklogRetirementReplica],
|
||||
) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, ScannerPauseBacklogRetirementError>;
|
||||
|
||||
static RETIREMENT_PLANNER: OnceLock<ScannerPauseBacklogRetirementPlanner> = OnceLock::new();
|
||||
|
||||
@@ -161,7 +187,8 @@ pub(crate) fn plan_scanner_pause_backlog_retirement(
|
||||
data: read.replica.data.clone(),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
planner(source_pool_index, &snapshots).map_err(Error::other)
|
||||
planner(source_pool_index, &snapshots)
|
||||
.map_err(|err| data_movement_context_error(format!("scanner pause backlog retirement planner failed: {err}"), err))
|
||||
}
|
||||
|
||||
/// The native writer and retirement handoff use the same conditional, full-tail
|
||||
|
||||
@@ -30,6 +30,8 @@ use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_madmin::{info_commands::DiskMetrics, metrics::TimedAction};
|
||||
#[cfg(not(test))]
|
||||
use std::sync::OnceLock;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::PathBuf,
|
||||
@@ -628,6 +630,8 @@ pub struct DiskHealthTracker {
|
||||
/// Authoritative atomically published runtime/status pair.
|
||||
state_snapshot: AtomicU64,
|
||||
transition_lock: std::sync::Mutex<()>,
|
||||
#[cfg(test)]
|
||||
test_forced_offline: AtomicBool,
|
||||
}
|
||||
|
||||
fn pack_health_state(runtime_state: RuntimeDriveHealthState, status: u32) -> u64 {
|
||||
@@ -975,6 +979,8 @@ impl DiskHealthTracker {
|
||||
last_capacity_probe_unix_secs: AtomicI64::new(0),
|
||||
state_snapshot: AtomicU64::new(pack_health_state(RuntimeDriveHealthState::Online, DISK_HEALTH_OK)),
|
||||
transition_lock: std::sync::Mutex::new(()),
|
||||
#[cfg(test)]
|
||||
test_forced_offline: AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1045,6 +1051,13 @@ impl DiskHealthTracker {
|
||||
self.publish_state(state, status);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
self.test_forced_offline.store(true, Ordering::Release);
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
self.publish_state(RuntimeDriveHealthState::Offline, DISK_HEALTH_FAULTY);
|
||||
}
|
||||
|
||||
pub fn swap_ok_to_faulty(&self) -> bool {
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let (_, status) = unpack_health_state(self.state_snapshot.load(Ordering::Acquire));
|
||||
@@ -1123,6 +1136,8 @@ impl DiskHealthTracker {
|
||||
/// Remote disks are marked faulty on timeout/network errors; the init loop retries with the
|
||||
/// same [`DiskStore`] handles, which would otherwise fail immediately at `is_faulty()`.
|
||||
pub fn reset_for_store_init_retry(&self, endpoint: &Endpoint) {
|
||||
#[cfg(test)]
|
||||
self.test_forced_offline.store(false, Ordering::Release);
|
||||
self.reset_for_store_init_retry_at(endpoint, current_unix_time());
|
||||
}
|
||||
|
||||
@@ -1142,6 +1157,10 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
|
||||
#[cfg(test)]
|
||||
if self.test_forced_offline.load(Ordering::Acquire) {
|
||||
return false;
|
||||
}
|
||||
let _guard = self.transition_lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let current = self.runtime_state();
|
||||
let next = match current {
|
||||
@@ -1440,6 +1459,11 @@ impl LocalDiskWrapper {
|
||||
self.health.force_runtime_state_for_test(state);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
self.health.force_offline_for_test();
|
||||
}
|
||||
|
||||
/// Same as [`DiskHealthTracker::reset_for_store_init_retry`]: undo a transient faulty mark before another format load attempt.
|
||||
pub fn reset_health_for_store_init_retry(&self) {
|
||||
self.health.reset_for_store_init_retry(&self.disk.endpoint());
|
||||
|
||||
@@ -94,6 +94,8 @@ const STALE_TMP_OBJECT_EXPIRY: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static DIRECTORY_LISTING_ENTRY_PROBE_COUNT: Arc<AtomicUsize>;
|
||||
/// `(sampled, complete)` directory reads issued by the delete-residue probe.
|
||||
static DELETE_RESIDUE_PROBE_READS: Arc<(AtomicUsize, AtomicUsize)>;
|
||||
}
|
||||
const RUSTFS_META_TMP_OLD_BUCKET: &str = ".rustfs.sys/tmp-old";
|
||||
const INLINE_METADATA_ROLLBACK_DIR_XOR: u128 = 0x7275737466735f696e6c696e655f7262;
|
||||
@@ -105,6 +107,21 @@ pub(crate) const RESERVED_DELETE_DATA_DIR_MARKER_PREFIX: &str = "reserve-delete-
|
||||
/// under-filled batch settles the common case without materializing large
|
||||
/// child sets; a full batch cannot prove no listable child hides behind it.
|
||||
const DELETE_RESIDUE_PROBE_LIMIT: i32 = 8;
|
||||
/// Most directory reads one delete-residue probe spends walking down through
|
||||
/// the ancestors of a deleted key before it gives up and lets the prefix
|
||||
/// surface. A genuine prefix settles within its depth (the first object's
|
||||
/// `xl.meta` ends the walk), so the budget only caps the cost of hiding a
|
||||
/// large residue tree, which then hides one level at a time instead.
|
||||
const DELETE_RESIDUE_PROBE_READ_BUDGET: usize = 32;
|
||||
|
||||
/// One directory read owed by the delete-residue probe.
|
||||
enum ResidueProbeStep {
|
||||
/// Read a bounded batch of `dir` and probe what it holds.
|
||||
Sample(String),
|
||||
/// Read all of `dir` once every child in `sampled` proved unlistable, and
|
||||
/// probe the rest.
|
||||
Remainder { dir: String, sampled: Vec<String> },
|
||||
}
|
||||
|
||||
/// A `part.N` file with a positive part number, the shape erasure data takes
|
||||
/// inside a version data dir.
|
||||
@@ -8047,72 +8064,116 @@ impl LocalDisk {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Whether the metadata-less directory `dir_name` holds nothing but the
|
||||
/// data dirs of deleted versions: it is itself a non-nil UUID directory of
|
||||
/// `part.N` files and delete-transaction markers, or every child is one.
|
||||
/// That is what an interrupted or deferred version delete leaves behind
|
||||
/// once the `xl.meta` is gone, and it must not surface as a prefix. Real
|
||||
/// object children are directories carrying their own `xl.meta`, so the
|
||||
/// first non-UUID child, stray file, or subdirectory inside a UUID child
|
||||
/// proves the directory is a genuine prefix. Reads are bounded: a
|
||||
/// Whether the metadata-less directory `dir_name` holds nothing that a
|
||||
/// listing could show: every leaf under it is the data dir of a deleted
|
||||
/// version (a non-nil UUID directory of `part.N` files and
|
||||
/// delete-transaction markers) or an empty directory. That is what an
|
||||
/// interrupted or deferred version delete leaves behind once the `xl.meta`
|
||||
/// is gone, and neither the object directory nor the date-style ancestors
|
||||
/// above it may surface as prefixes (#6898). Real objects are directories
|
||||
/// carrying their own `xl.meta`, so the first file met outside a UUID data
|
||||
/// dir, or any stray entry inside one, proves a genuine prefix and ends the
|
||||
/// walk at the depth of the first object.
|
||||
///
|
||||
/// Reads stay small on genuine prefixes: each directory is sampled with one
|
||||
/// bounded batch, the sampled children are probed depth-first, and the
|
||||
/// directory is only read in full once every sampled child proved to hold
|
||||
/// nothing listable. A total read budget makes a residue tree larger than
|
||||
/// the budget surface and be hidden one level down instead, so the cost on
|
||||
/// a genuine prefix never exceeds its depth in small directory reads. A
|
||||
/// directory that vanishes mid-probe holds nothing listable.
|
||||
async fn directory_is_delete_residue(&self, bucket: &str, dir_name: &str, stall: Option<Duration>) -> Result<bool> {
|
||||
let dir_name = dir_name.trim_end_matches(SLASH_SEPARATOR);
|
||||
let Some(entries) = self.read_dir_for_residue_probe(bucket, dir_name, stall).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
if entries.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let is_data_dir = dir_name
|
||||
.rsplit(SLASH_SEPARATOR)
|
||||
.next()
|
||||
.is_some_and(|name| Uuid::parse_str(name).is_ok_and(|uuid| !uuid.is_nil()));
|
||||
if is_data_dir && entries.iter().all(|entry| is_metadata_less_data_dir_entry(entry)) {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
for entry in entries {
|
||||
let Some(child) = entry.strip_suffix(SLASH_SEPARATOR) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !Uuid::parse_str(child).is_ok_and(|uuid| !uuid.is_nil()) {
|
||||
let mut reads = 0usize;
|
||||
let mut pending = vec![ResidueProbeStep::Sample(dir_name.to_owned())];
|
||||
while let Some(step) = pending.pop() {
|
||||
if reads >= DELETE_RESIDUE_PROBE_READ_BUDGET {
|
||||
return Ok(false);
|
||||
}
|
||||
reads += 1;
|
||||
|
||||
let child_path = path_join_buf(&[dir_name, child]);
|
||||
let Some(child_entries) = self.read_dir_for_residue_probe(bucket, &child_path, stall).await? else {
|
||||
let (dir, entries, complete) = match step {
|
||||
ResidueProbeStep::Sample(dir) => {
|
||||
let Some((entries, complete)) = self.read_dir_for_residue_probe(bucket, &dir, stall, false).await? else {
|
||||
continue;
|
||||
};
|
||||
(dir, entries, complete)
|
||||
}
|
||||
ResidueProbeStep::Remainder { dir, sampled } => {
|
||||
let Some((entries, _)) = self.read_dir_for_residue_probe(bucket, &dir, stall, true).await? else {
|
||||
continue;
|
||||
};
|
||||
let entries = entries
|
||||
.into_iter()
|
||||
.filter(|entry| !sampled.contains(entry))
|
||||
.collect::<Vec<_>>();
|
||||
(dir, entries, true)
|
||||
}
|
||||
};
|
||||
|
||||
let is_data_dir = dir
|
||||
.rsplit(SLASH_SEPARATOR)
|
||||
.next()
|
||||
.is_some_and(|name| Uuid::parse_str(name).is_ok_and(|uuid| !uuid.is_nil()));
|
||||
if is_data_dir {
|
||||
if !entries.iter().all(|entry| is_metadata_less_data_dir_entry(entry)) {
|
||||
// A subdirectory, an `xl.meta`, or an unknown file inside a
|
||||
// UUID directory: not plain delete residue.
|
||||
return Ok(false);
|
||||
}
|
||||
if !complete {
|
||||
// Only the sampled part files were seen; the rest must be
|
||||
// read before the data dir counts as plain residue.
|
||||
pending.push(ResidueProbeStep::Remainder { dir, sampled: entries });
|
||||
}
|
||||
continue;
|
||||
};
|
||||
if !child_entries.iter().all(|entry| is_metadata_less_data_dir_entry(entry)) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut children = Vec::with_capacity(entries.len());
|
||||
for entry in &entries {
|
||||
let Some(child) = entry.strip_suffix(SLASH_SEPARATOR) else {
|
||||
// A file outside a plain data dir: `xl.meta` or something
|
||||
// this probe does not understand. Either way, not residue.
|
||||
return Ok(false);
|
||||
};
|
||||
children.push(path_join_buf(&[&dir, child]));
|
||||
}
|
||||
if !complete {
|
||||
// Revisit the unsampled siblings only after every sampled
|
||||
// child, probed first, turned out to hold nothing listable.
|
||||
pending.push(ResidueProbeStep::Remainder { dir, sampled: entries });
|
||||
}
|
||||
pending.extend(children.into_iter().map(ResidueProbeStep::Sample));
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Read `dir` with a bounded batch first and a complete read only when the
|
||||
/// batch was full. `None` when the directory does not exist any more.
|
||||
async fn read_dir_for_residue_probe(&self, bucket: &str, dir: &str, stall: Option<Duration>) -> Result<Option<Vec<String>>> {
|
||||
for count in [DELETE_RESIDUE_PROBE_LIMIT, -1] {
|
||||
let entries = match with_walk_stall_timeout(stall, self.list_dir("", bucket, dir, count)).await {
|
||||
Ok(entries) => entries,
|
||||
Err(err) => {
|
||||
if err == DiskError::VolumeNotFound || err == Error::FileNotFound {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
if count < 0 || entries.len() < count as usize {
|
||||
return Ok(Some(entries));
|
||||
/// Read `dir` for the residue probe: one bounded batch, or the complete
|
||||
/// directory when `complete` is set. The flag in the result says whether
|
||||
/// the batch held the whole directory. `None` when the directory does not
|
||||
/// exist any more.
|
||||
async fn read_dir_for_residue_probe(
|
||||
&self,
|
||||
bucket: &str,
|
||||
dir: &str,
|
||||
stall: Option<Duration>,
|
||||
complete: bool,
|
||||
) -> Result<Option<(Vec<String>, bool)>> {
|
||||
let count = if complete { -1 } else { DELETE_RESIDUE_PROBE_LIMIT };
|
||||
#[cfg(test)]
|
||||
let _ = DELETE_RESIDUE_PROBE_READS.try_with(|reads| {
|
||||
let counter = if complete { &reads.1 } else { &reads.0 };
|
||||
counter.fetch_add(1, Ordering::Relaxed)
|
||||
});
|
||||
match with_walk_stall_timeout(stall, self.list_dir("", bucket, dir, count)).await {
|
||||
Ok(entries) => {
|
||||
let complete = complete || entries.len() < DELETE_RESIDUE_PROBE_LIMIT as usize;
|
||||
Ok(Some((entries, complete)))
|
||||
}
|
||||
Err(err) if err == DiskError::VolumeNotFound || err == Error::FileNotFound => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Whether anything under `dir_name` would appear in a listing. With
|
||||
@@ -18719,9 +18780,9 @@ mod test {
|
||||
let (fast_path_names, fast_path_probes) = scan_prefixes(&disk, bucket, true).await;
|
||||
|
||||
assert_eq!(conservative_names, expected_names);
|
||||
let mut expected_fast_path_names = expected_names.clone();
|
||||
expected_fast_path_names.push("stale/".to_owned());
|
||||
assert_eq!(fast_path_names, expected_fast_path_names);
|
||||
// The fast path hides the empty `stale/` chain too, at the cost of a
|
||||
// few bounded directory reads rather than the metadata probes.
|
||||
assert_eq!(fast_path_names, expected_names);
|
||||
let expected_probes = PREFIX_COUNT * 3 + 3;
|
||||
assert_eq!(conservative_probes, expected_probes);
|
||||
assert_eq!(fast_path_probes, 0);
|
||||
@@ -18832,9 +18893,9 @@ mod test {
|
||||
}
|
||||
|
||||
// Directories whose only content is a deleted version's data dir are
|
||||
// not prefixes; their ancestors stay ordinary directories until an
|
||||
// empty listing reclaims them.
|
||||
// not prefixes, and neither are their ancestors.
|
||||
assert_eq!(scan_names(&disk, bucket, "residue/2026/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "residue/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "committed/").await, Vec::<String>::new());
|
||||
|
||||
// UUID-named directories holding real objects, an object keyed by a
|
||||
@@ -18842,18 +18903,258 @@ mod test {
|
||||
assert_eq!(scan_names(&disk, bucket, "uploads/").await, vec![format!("uploads/{upload}/")]);
|
||||
assert_eq!(scan_names(&disk, bucket, "named/").await, vec![format!("named/{named}")]);
|
||||
assert_eq!(scan_names(&disk, bucket, "mixed/").await, vec!["mixed/child".to_owned()]);
|
||||
assert_eq!(
|
||||
scan_names(&disk, bucket, "").await,
|
||||
vec!["mixed/".to_owned(), "named/".to_owned(), "uploads/".to_owned()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_nonrecursive_fast_path_hides_delete_residue_ancestors() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let bucket = "test-bucket";
|
||||
let bucket_dir = dir.path().join(bucket);
|
||||
|
||||
async fn write_object(object_dir: &Path, object_name: &str) {
|
||||
fs::create_dir_all(object_dir)
|
||||
.await
|
||||
.expect("object directory should be created");
|
||||
let mut metadata = FileMeta::default();
|
||||
let mut file_info = FileInfo::new(object_name, 1, 1);
|
||||
file_info.mod_time = Some(OffsetDateTime::now_utc());
|
||||
metadata.add_version(file_info).expect("metadata should be valid");
|
||||
fs::write(
|
||||
object_dir.join(STORAGE_FORMAT_FILE),
|
||||
metadata.marshal_msg().expect("metadata should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
}
|
||||
|
||||
async fn write_residue(object_dir: &Path, committed: bool) {
|
||||
let residue = object_dir.join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&residue).await.expect("residue should be created");
|
||||
fs::write(residue.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
if committed {
|
||||
fs::write(residue.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4())), [])
|
||||
.await
|
||||
.expect("delete marker should be written");
|
||||
}
|
||||
}
|
||||
|
||||
// The reported shape: a date-partitioned key deleted on an older build,
|
||||
// whose data dir survived without a delete-transaction marker. Every
|
||||
// ancestor up to `metrics/` holds nothing else.
|
||||
write_residue(&bucket_dir.join("metrics/kubelet/2026/08/28/23/74992556388248657933757.parquet"), false).await;
|
||||
// Several committed residues under one hour plus an empty sibling hour.
|
||||
write_residue(&bucket_dir.join("metrics/cpu/2026/08/28/22/a.parquet"), true).await;
|
||||
write_residue(&bucket_dir.join("metrics/cpu/2026/08/28/22/b.parquet"), true).await;
|
||||
fs::create_dir_all(bucket_dir.join("metrics/cpu/2026/08/28/21"))
|
||||
.await
|
||||
.expect("empty hour directory should be created");
|
||||
|
||||
// A live object deep under an otherwise identical tree keeps every
|
||||
// ancestor visible, even beside residue.
|
||||
write_residue(&bucket_dir.join("logs/default/2026/08/28/23/old.parquet"), false).await;
|
||||
write_object(
|
||||
&bucket_dir.join("logs/default/2026/08/28/23/live.parquet"),
|
||||
"logs/default/2026/08/28/23/live.parquet",
|
||||
)
|
||||
.await;
|
||||
|
||||
// A prefix with more residue directories than the probe budget stays
|
||||
// visible rather than costing an unbounded walk.
|
||||
for hour in 0..(DELETE_RESIDUE_PROBE_READ_BUDGET + 1) {
|
||||
write_residue(&bucket_dir.join(format!("bulk/2026/08/28/{hour:02}/a.parquet")), false).await;
|
||||
}
|
||||
// Exactly at the budget: `edge/` with N object dirs costs one sampled
|
||||
// read of `edge`, two reads per object dir (its own and its data dir),
|
||||
// and one remainder read of `edge` once its 8-entry sample was full.
|
||||
for object in 0..15 {
|
||||
write_residue(&bucket_dir.join(format!("edge-hide/{object:02}.parquet")), false).await;
|
||||
}
|
||||
for object in 0..16 {
|
||||
write_residue(&bucket_dir.join(format!("edge-show/{object:02}.parquet")), false).await;
|
||||
}
|
||||
|
||||
// A file that is not `xl.meta` outside a UUID data dir is not residue
|
||||
// either: the probe does not guess, the prefix surfaces.
|
||||
fs::create_dir_all(bucket_dir.join("stray/2026/08"))
|
||||
.await
|
||||
.expect("stray directory should be created");
|
||||
fs::write(bucket_dir.join("stray/2026/08/notes.txt"), b"?")
|
||||
.await
|
||||
.expect("stray file should be written");
|
||||
|
||||
// A UUID directory holding a subdirectory is not a data dir shape the
|
||||
// probe understands, so it surfaces even when the subdirectory is empty.
|
||||
let odd = Uuid::new_v4().to_string();
|
||||
fs::create_dir_all(bucket_dir.join("odd").join(&odd).join("sub"))
|
||||
.await
|
||||
.expect("odd directory should be created");
|
||||
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be UTF-8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should initialize");
|
||||
|
||||
async fn scan_names(disk: &LocalDisk, bucket: &str, current: &str) -> Vec<String> {
|
||||
let (reader, mut writer) = tokio::io::duplex(64 * 1024);
|
||||
let mut output = MetacacheWriter::new(&mut writer);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: current.to_string(),
|
||||
skip_hidden_prefix_check: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut objects_returned = 0;
|
||||
disk.scan_dir(
|
||||
current.to_string(),
|
||||
"".to_string(),
|
||||
&opts,
|
||||
&mut output,
|
||||
&mut objects_returned,
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("scan_dir should succeed");
|
||||
output.close().await.expect("metacache writer should close");
|
||||
drop(output);
|
||||
drop(writer);
|
||||
|
||||
let mut names = MetacacheReader::new(reader)
|
||||
.read_all()
|
||||
.await
|
||||
.expect("scan output should decode")
|
||||
.into_iter()
|
||||
.map(|entry| entry.name)
|
||||
.collect::<Vec<_>>();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
// Every level of a tree whose only leaves are deleted data dirs is
|
||||
// hidden, not just the object directory itself.
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/kubelet/2026/08/28/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/kubelet/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/cpu/2026/08/28/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "metrics/").await, Vec::<String>::new());
|
||||
assert_eq!(
|
||||
scan_names(&disk, bucket, "logs/default/2026/08/28/").await,
|
||||
vec!["logs/default/2026/08/28/23/".to_owned()]
|
||||
);
|
||||
assert_eq!(scan_names(&disk, bucket, "logs/").await, vec!["logs/default/".to_owned()]);
|
||||
assert_eq!(scan_names(&disk, bucket, "bulk/2026/08/").await, vec!["bulk/2026/08/28/".to_owned()]);
|
||||
// Listed directly, every object dir is probed on its own and hidden;
|
||||
// the budget only bites where the whole tree hangs off one entry, so
|
||||
// the root listing below shows `edge-show/` but not `edge-hide/`.
|
||||
assert_eq!(scan_names(&disk, bucket, "edge-hide/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "edge-show/").await, Vec::<String>::new());
|
||||
assert_eq!(scan_names(&disk, bucket, "stray/").await, vec!["stray/2026/".to_owned()]);
|
||||
assert_eq!(scan_names(&disk, bucket, "odd/").await, vec![format!("odd/{odd}/")]);
|
||||
assert_eq!(
|
||||
scan_names(&disk, bucket, "").await,
|
||||
vec![
|
||||
"committed/".to_owned(),
|
||||
"mixed/".to_owned(),
|
||||
"named/".to_owned(),
|
||||
"residue/".to_owned(),
|
||||
"uploads/".to_owned(),
|
||||
"bulk/".to_owned(),
|
||||
"edge-show/".to_owned(),
|
||||
"logs/".to_owned(),
|
||||
"odd/".to_owned(),
|
||||
"stray/".to_owned(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_nonrecursive_fast_path_residue_probe_reads_stay_within_prefix_depth() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let dir = tempdir().expect("tempdir should be created");
|
||||
let bucket = "test-bucket";
|
||||
let bucket_dir = dir.path().join(bucket);
|
||||
|
||||
async fn write_object(object_dir: &Path, object_name: &str) {
|
||||
fs::create_dir_all(object_dir)
|
||||
.await
|
||||
.expect("object directory should be created");
|
||||
let mut metadata = FileMeta::default();
|
||||
let mut file_info = FileInfo::new(object_name, 1, 1);
|
||||
file_info.mod_time = Some(OffsetDateTime::now_utc());
|
||||
metadata.add_version(file_info).expect("metadata should be valid");
|
||||
fs::write(
|
||||
object_dir.join(STORAGE_FORMAT_FILE),
|
||||
metadata.marshal_msg().expect("metadata should encode"),
|
||||
)
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
}
|
||||
|
||||
// A genuine date-partitioned prefix wider than the probe batch at two
|
||||
// levels: 20 days, each hour holding 100 objects.
|
||||
const DAYS: usize = 20;
|
||||
const OBJECTS: usize = 100;
|
||||
for day in 0..DAYS {
|
||||
for object in 0..OBJECTS {
|
||||
let key = format!("data/stream/2026/08/{day:02}/23/{object:03}.parquet");
|
||||
write_object(&bucket_dir.join(&key), &key).await;
|
||||
}
|
||||
}
|
||||
|
||||
let endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be UTF-8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should initialize");
|
||||
|
||||
let reads = Arc::new((AtomicUsize::new(0), AtomicUsize::new(0)));
|
||||
let (reader, mut writer) = tokio::io::duplex(64 * 1024);
|
||||
let mut output = MetacacheWriter::new(&mut writer);
|
||||
let opts = WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
base_dir: "data/".to_string(),
|
||||
skip_hidden_prefix_check: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut objects_returned = 0;
|
||||
DELETE_RESIDUE_PROBE_READS
|
||||
.scope(
|
||||
Arc::clone(&reads),
|
||||
disk.scan_dir(
|
||||
"data/".to_string(),
|
||||
"".to_string(),
|
||||
&opts,
|
||||
&mut output,
|
||||
&mut objects_returned,
|
||||
false,
|
||||
None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("scan_dir should succeed");
|
||||
output.close().await.expect("metacache writer should close");
|
||||
drop(output);
|
||||
drop(writer);
|
||||
|
||||
let names = MetacacheReader::new(reader)
|
||||
.read_all()
|
||||
.await
|
||||
.expect("scan output should decode")
|
||||
.into_iter()
|
||||
.map(|entry| entry.name)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(names, vec!["data/stream/".to_owned()]);
|
||||
|
||||
// stream, 2026, 08, one sampled day, its hour, one sampled object:
|
||||
// the walk ends at the first `xl.meta`, one bounded read per level,
|
||||
// without ever reading a wide directory in full.
|
||||
let (sampled, complete) = (reads.0.load(Ordering::Relaxed), reads.1.load(Ordering::Relaxed));
|
||||
assert_eq!(sampled, 6, "one sampled read per level down to the first object");
|
||||
assert_eq!(complete, 0, "a genuine prefix must never cost a complete directory read");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_dir_nonrecursive_skips_dirs_with_only_hidden_delete_markers() {
|
||||
use rustfs_filemeta::MetacacheReader;
|
||||
|
||||
@@ -979,6 +979,14 @@ impl Disk {
|
||||
Disk::Remote(remote_disk) => remote_disk.force_runtime_state_for_test(state),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_offline_for_test(&self) {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.force_offline_for_test(),
|
||||
Disk::Remote(remote_disk) => remote_disk.force_offline_for_test(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
||||
@@ -41,7 +41,7 @@ impl Debug for NamespaceLockFence {
|
||||
}
|
||||
|
||||
impl NamespaceLockFence {
|
||||
fn new() -> Self {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
signals: Arc::default(),
|
||||
#[cfg(test)]
|
||||
@@ -153,7 +153,7 @@ pub struct ObjectLockConfigSnapshot {
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_fence: NamespaceLockFence,
|
||||
_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
metadata_transaction_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
}
|
||||
|
||||
impl ObjectLockConfigSnapshot {
|
||||
@@ -170,6 +170,7 @@ impl ObjectLockConfigSnapshot {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn for_store_bucket(
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
@@ -210,7 +211,7 @@ impl ObjectLockConfigSnapshot {
|
||||
state,
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: Some(lifecycle_guard),
|
||||
metadata_transaction_guard: Some(metadata_transaction_guard),
|
||||
metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,7 +222,7 @@ impl ObjectLockConfigSnapshot {
|
||||
config_revision: OffsetDateTime,
|
||||
state: crate::bucket::metadata_sys::ObjectLockConfigState,
|
||||
lifecycle_fence: NamespaceLockFence,
|
||||
metadata_transaction_guard: rustfs_lock::NamespaceLockGuard,
|
||||
metadata_transaction_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Self {
|
||||
Self {
|
||||
store_id: Some(store_id),
|
||||
@@ -265,6 +266,23 @@ impl ObjectLockConfigSnapshot {
|
||||
.is_some_and(|guard| !guard.is_lock_lost())
|
||||
}
|
||||
|
||||
/// Share the held transaction lock without queuing another reader behind
|
||||
/// a metadata writer that is itself waiting for this snapshot to drop.
|
||||
pub(crate) fn metadata_transaction_guard_for(
|
||||
&self,
|
||||
store_id: Uuid,
|
||||
bucket: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Option<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
let bucket_incarnation_id = self.bucket_incarnation_id?;
|
||||
if expected_incarnation_id.is_some_and(|expected| expected != bucket_incarnation_id)
|
||||
|| !self.is_valid_for_destructive_put(store_id, bucket, bucket_incarnation_id)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
self.metadata_transaction_guard.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn add_lock_fences(&self, opts: &mut ObjectOptions) {
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.get_or_insert_with(NamespaceLockFence::new)
|
||||
|
||||
@@ -9,11 +9,12 @@ use super::meta::{
|
||||
};
|
||||
use super::worker::{
|
||||
rebalance_max_attempts, rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result,
|
||||
resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, retry_rebalance_metadata_access,
|
||||
resolve_rebalance_deferred_last_error, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
|
||||
retry_rebalance_metadata_access,
|
||||
};
|
||||
use super::{
|
||||
DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
|
||||
RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
|
||||
RebalStatus, RebalanceDeferKind, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
|
||||
encode_rebalance_stop_propagation_record,
|
||||
};
|
||||
use crate::core::pools::{
|
||||
@@ -1103,6 +1104,7 @@ impl ECStore {
|
||||
bucket: String,
|
||||
last_error: String,
|
||||
expected_id: &str,
|
||||
kind: RebalanceDeferKind,
|
||||
) -> Result<()> {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "defer rebalance bucket")?;
|
||||
@@ -1116,7 +1118,8 @@ impl ECStore {
|
||||
};
|
||||
|
||||
defer_bucket_in_rebalance_queue(pool_stat, &bucket)?;
|
||||
pool_stat.info.last_error = Some(last_error);
|
||||
let pending_entry_defer = pool_stat.info.last_error.clone();
|
||||
pool_stat.info.last_error = resolve_rebalance_deferred_last_error(kind, pending_entry_defer.as_deref(), &last_error);
|
||||
meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1209,7 +1209,7 @@ mod tests {
|
||||
cancel: &warning_cancel,
|
||||
};
|
||||
let warning_result = store
|
||||
.finish_rebalance_entry_after_cleanup(&warning_cleanup_context, async { Err(Error::SlowDown.into()) })
|
||||
.finish_rebalance_entry_after_cleanup(&warning_cleanup_context, async { Err(Error::FileAccessDenied.into()) })
|
||||
.await
|
||||
.expect("cleanup warnings should not fail the completed migration");
|
||||
assert!(matches!(warning_result, RebalanceEntryCleanupResult::Completed { warning: Some(_) }));
|
||||
@@ -1250,6 +1250,35 @@ mod tests {
|
||||
let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stats.bytes, 0, "deferred cleanup must not commit completion stats");
|
||||
assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
|
||||
drop(meta);
|
||||
|
||||
let transient_guard = store
|
||||
.rebalance_run_guard(rebalance_id, "rebalance transient cleanup deferral test")
|
||||
.await
|
||||
.expect("rebalance transient cleanup deferral test guard should be acquired");
|
||||
let transient_cancel = CancellationToken::new();
|
||||
let transient_stats_updates = [&warning_version];
|
||||
let transient_cleanup_context = RebalanceEntryCleanupContext {
|
||||
run_guard: &transient_guard,
|
||||
pool_index: 0,
|
||||
bucket: "bucket",
|
||||
object: "object.bin",
|
||||
stats_updates: &transient_stats_updates,
|
||||
expected_id: rebalance_id,
|
||||
cancel: &transient_cancel,
|
||||
};
|
||||
let transient = store
|
||||
.finish_rebalance_entry_after_cleanup(&transient_cleanup_context, async { Err(Error::SlowDown.into()) })
|
||||
.await
|
||||
.expect("retryable cleanup failures should defer without failing the worker");
|
||||
assert!(matches!(transient, RebalanceEntryCleanupResult::Deferred { .. }));
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stats = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stats.bytes, 0, "retryable cleanup failures must not commit completion stats");
|
||||
assert_eq!(
|
||||
pool_stats.cleanup_warnings.count, 1,
|
||||
"retryable cleanup failures must not add a permanent warning"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -29,7 +29,7 @@ const REBAL_META_FMT: u16 = 1; // Replace with actual format value
|
||||
const REBAL_META_VER: u16 = 1; // Replace with actual version value
|
||||
pub(crate) const REBAL_META_NAME: &str = "rebalance.bin";
|
||||
const DEFAULT_REBALANCE_MAX_ATTEMPTS: usize = 3;
|
||||
pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 3;
|
||||
pub(crate) const REBALANCE_SOURCE_CLEANUP_MAX_DEFERS: usize = 8;
|
||||
const REBALANCE_MAX_ATTEMPTS_ENV: &str = "RUSTFS_REBALANCE_MAX_ATTEMPTS";
|
||||
const REBALANCE_STOP_PROPAGATION_ERROR_PREFIX: &str = "rebalance stop propagation incomplete: ";
|
||||
const REBALANCE_LISTING_RETRY_BASE_DELAY: Duration = Duration::from_millis(250);
|
||||
@@ -55,7 +55,7 @@ pub use types::{
|
||||
DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta,
|
||||
RebalanceStats, RebalanceStopPropagationRecord,
|
||||
};
|
||||
use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceEntryOutcome};
|
||||
use types::{RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceDeferKind, RebalanceEntryOutcome};
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn test_store_with_persisted_rebalance_meta(
|
||||
@@ -222,10 +222,14 @@ async fn test_pool_stores_with_contexts(
|
||||
std::sync::Arc::clone(&ctx)
|
||||
};
|
||||
let make_store = |store_ctx: std::sync::Arc<crate::runtime::instance::InstanceContext>| {
|
||||
let mut store_pools = pools.clone();
|
||||
for pool in &mut store_pools {
|
||||
std::sync::Arc::make_mut(pool).set_instance_ctx_for_test(std::sync::Arc::clone(&store_ctx));
|
||||
}
|
||||
std::sync::Arc::new(crate::store::ECStore {
|
||||
id: uuid::Uuid::new_v4(),
|
||||
disk_map: std::collections::HashMap::new(),
|
||||
pools: pools.clone(),
|
||||
pools: store_pools,
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, std::sync::Arc::clone(&store_ctx)),
|
||||
pool_meta: tokio::sync::RwLock::new(pool_meta.clone()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()),
|
||||
|
||||
@@ -34,27 +34,29 @@ use super::migration::{
|
||||
};
|
||||
use super::runtime::{
|
||||
RebalanceLocalActivationOutcome, commit_local_rebalance_worker_activation,
|
||||
commit_local_rebalance_worker_activation_candidate, should_fail_repeated_rebalance_bucket_defer,
|
||||
source_cleanup_defer_attempt, stage_local_rebalance_worker_activation,
|
||||
commit_local_rebalance_worker_activation_candidate, reached_rebalance_source_cleanup_defer_limit,
|
||||
should_fail_repeated_rebalance_bucket_defer, source_cleanup_defer_attempt, stage_local_rebalance_worker_activation,
|
||||
};
|
||||
use super::worker::{
|
||||
RebalanceEntryCleanupResult, ensure_rebalance_listing_disks_available, is_transient_rebalance_error,
|
||||
parse_rebalance_max_attempts, rebalance_listing_retry_delay, rebalance_migration_retry_delay,
|
||||
resolve_load_rebalance_stats_update_result, resolve_rebalance_bucket_error, resolve_rebalance_bucket_result,
|
||||
resolve_rebalance_entry_cleanup_delete_result, resolve_rebalance_file_info_versions_result,
|
||||
resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, resolve_rebalance_migrate_result_error,
|
||||
resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result, resolve_rebalance_stats_update_result,
|
||||
resolve_rebalance_terminal_error, resolve_rebalance_worker_result, run_rebalance_listing_with_retry,
|
||||
send_rebalance_done_signal, should_cleanup_rebalance_source_entry, should_count_rebalance_version_complete,
|
||||
should_defer_rebalance_entry_failure, should_retry_rebalance_listing, should_skip_rebalance_delete_marker,
|
||||
wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
|
||||
resolve_rebalance_deferred_last_error, resolve_rebalance_entry_cleanup_delete_result,
|
||||
resolve_rebalance_file_info_versions_result, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
|
||||
resolve_rebalance_migrate_result_error, resolve_rebalance_optional_bucket_config_result, resolve_rebalance_save_task_result,
|
||||
resolve_rebalance_stats_update_result, resolve_rebalance_terminal_error, resolve_rebalance_worker_result,
|
||||
run_rebalance_listing_with_retry, send_rebalance_done_signal, should_cleanup_rebalance_source_entry,
|
||||
should_count_rebalance_version_complete, should_defer_rebalance_entry_failure, should_retry_rebalance_listing,
|
||||
should_skip_rebalance_delete_marker, wait_rebalance_entry_tasks, wait_rebalance_listing_retry, with_rebalance_entry_context,
|
||||
};
|
||||
use super::{
|
||||
DiskStat, GetObjectReader, ObjectInfo, ObjectOptions, RebalSaveOpt, RebalStatus, RebalanceBucketConfigs,
|
||||
RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta, RebalanceStats,
|
||||
RebalanceStopPropagationRecord,
|
||||
RebalanceBucketOutcome, RebalanceCleanupWarnings, RebalanceDeferKind, RebalanceEntryOutcome, RebalanceInfo, RebalanceMeta,
|
||||
RebalanceStats, RebalanceStopPropagationRecord,
|
||||
};
|
||||
use super::{
|
||||
REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_MAX_DEFERS,
|
||||
};
|
||||
use super::{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX};
|
||||
use crate::bucket::replication::{ReplicationState, ReplicationStatusType, replication_state_to_filemeta};
|
||||
use crate::data_movement;
|
||||
use crate::data_movement::SourceCleanupError;
|
||||
@@ -1775,7 +1777,7 @@ fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_not_found() {
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failures() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::SlowDown.into()), "bucket-a", "obj.txt");
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(Error::FileAccessDenied.into()), "bucket-a", "obj.txt");
|
||||
assert!(matches!(
|
||||
result,
|
||||
RebalanceEntryCleanupResult::Completed { warning: Some(ref message) }
|
||||
@@ -1783,6 +1785,122 @@ fn test_resolve_rebalance_entry_cleanup_delete_result_returns_warning_for_failur
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_transient_failures() {
|
||||
let cases = [
|
||||
(Error::SlowDown, "slow down"),
|
||||
(
|
||||
Error::Lock(rustfs_lock::LockError::timeout("bucket-a/obj.txt@latest", Duration::from_secs(5))),
|
||||
"object lock timeout",
|
||||
),
|
||||
(
|
||||
Error::Lock(rustfs_lock::LockError::network(
|
||||
"peer unavailable",
|
||||
std::io::Error::from(std::io::ErrorKind::ConnectionReset),
|
||||
)),
|
||||
"object lock network failure",
|
||||
),
|
||||
(Error::ErasureWriteQuorum, "write quorum"),
|
||||
(Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)), "io timeout"),
|
||||
(
|
||||
Error::other("Lock error: Lock acquisition timeout for resource 'bucket-a/obj.txt@latest' after 5s"),
|
||||
"rendered lock timeout text",
|
||||
),
|
||||
];
|
||||
|
||||
for (err, label) in cases {
|
||||
match resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt") {
|
||||
RebalanceEntryCleanupResult::Deferred { last_error } => {
|
||||
assert!(
|
||||
last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX),
|
||||
"{label}: {last_error}"
|
||||
);
|
||||
assert!(last_error.contains("bucket-a/obj.txt"), "{label}: {last_error}");
|
||||
}
|
||||
RebalanceEntryCleanupResult::Completed { warning } => {
|
||||
panic!("{label} must defer source cleanup instead of completing the entry with warning {warning:?}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_stage_wrapped_lock_timeout() {
|
||||
let err = data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance",
|
||||
"delete_object",
|
||||
"bucket-a",
|
||||
"obj.txt",
|
||||
Error::Lock(rustfs_lock::LockError::timeout("bucket-a/obj.txt@latest", Duration::from_secs(5))),
|
||||
);
|
||||
|
||||
assert!(matches!(
|
||||
resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt"),
|
||||
RebalanceEntryCleanupResult::Deferred { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_ignores_stage_wrapped_not_found() {
|
||||
let err = data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance",
|
||||
"delete_object",
|
||||
"bucket-a",
|
||||
"obj.txt",
|
||||
Error::ObjectNotFound("bucket-a".to_string(), "obj.txt".to_string()),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
resolve_rebalance_entry_cleanup_delete_result(Err(err.into()), "bucket-a", "obj.txt"),
|
||||
RebalanceEntryCleanupResult::Completed { warning: None }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_deferred_last_error_hides_retryable_cleanup_conflicts() {
|
||||
let entry_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} timeout");
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(RebalanceDeferKind::Entry, None, entry_error.as_str()),
|
||||
Some(entry_error.clone()),
|
||||
"transient migration deferrals must stay visible to the completion guards"
|
||||
);
|
||||
|
||||
let cleanup_error = format!("{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} lock acquisition timeout");
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(RebalanceDeferKind::SourceCleanup, None, cleanup_error.as_str()),
|
||||
None,
|
||||
"a retryable source cleanup conflict is progress, not a pool failure"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(
|
||||
RebalanceDeferKind::SourceCleanup,
|
||||
Some(entry_error.as_str()),
|
||||
cleanup_error.as_str()
|
||||
),
|
||||
Some(entry_error.clone()),
|
||||
"a cleanup deferral for one bucket must not erase an unresolved migration deferral of the same pool"
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(
|
||||
RebalanceDeferKind::SourceCleanup,
|
||||
Some(cleanup_error.as_str()),
|
||||
cleanup_error.as_str()
|
||||
),
|
||||
None,
|
||||
"a stale retryable cleanup message must not survive as a pool failure"
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_rebalance_deferred_last_error(
|
||||
RebalanceDeferKind::SourceCleanup,
|
||||
Some(entry_error.as_str()),
|
||||
entry_error.as_str()
|
||||
),
|
||||
Some(entry_error),
|
||||
"repeated cleanup deferrals must keep the pending migration deferral visible"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_rebalance_entry_cleanup_delete_result_defers_source_change() {
|
||||
let result = resolve_rebalance_entry_cleanup_delete_result(Err(SourceCleanupError::SourceChanged), "bucket-a", "obj.txt");
|
||||
@@ -1817,6 +1935,89 @@ fn test_source_cleanup_defer_does_not_fail_repeated_bucket_retry() {
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 1);
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 2);
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut source_attempts, "bucket-c"), 3);
|
||||
|
||||
let mut bounded_attempts = std::collections::HashMap::new();
|
||||
for expected in 1..REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
|
||||
assert_eq!(source_cleanup_defer_attempt(&mut bounded_attempts, "bucket-d"), expected);
|
||||
assert!(
|
||||
!reached_rebalance_source_cleanup_defer_limit(expected),
|
||||
"a retryable cleanup conflict must stay retryable at deferral {expected}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
source_cleanup_defer_attempt(&mut bounded_attempts, "bucket-d"),
|
||||
REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
);
|
||||
assert!(
|
||||
reached_rebalance_source_cleanup_defer_limit(REBALANCE_SOURCE_CLEANUP_MAX_DEFERS),
|
||||
"an unreclaimable source replica must fail the bucket after the bounded deferral budget"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_defer_rebalance_bucket_keeps_pending_entry_defer_without_surfacing_cleanup_conflicts() {
|
||||
let id = "rebalance-defer-last-error";
|
||||
let meta = RebalanceMeta {
|
||||
id: id.to_string(),
|
||||
pool_stats: vec![RebalanceStats {
|
||||
participating: true,
|
||||
buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let store = test_store_with_rebalance_meta(meta);
|
||||
let cleanup_error = format!("{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} lock acquisition timeout");
|
||||
|
||||
store
|
||||
.defer_rebalance_bucket(0, "bucket-a".to_string(), cleanup_error.clone(), id, RebalanceDeferKind::SourceCleanup)
|
||||
.await
|
||||
.expect("a retryable cleanup conflict must be deferrable");
|
||||
{
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(
|
||||
pool_stat.info.last_error, None,
|
||||
"a retryable cleanup conflict is progress and must not surface as a pool failure"
|
||||
);
|
||||
assert_eq!(pool_stat.buckets, vec!["bucket-b".to_string(), "bucket-a".to_string()]);
|
||||
}
|
||||
|
||||
let entry_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} slow down");
|
||||
{
|
||||
let mut meta = store.rebalance_meta.write().await;
|
||||
meta.as_mut().expect("rebalance metadata should exist").pool_stats[0]
|
||||
.info
|
||||
.last_error = Some(entry_error.clone());
|
||||
}
|
||||
store
|
||||
.defer_rebalance_bucket(0, "bucket-b".to_string(), cleanup_error, id, RebalanceDeferKind::SourceCleanup)
|
||||
.await
|
||||
.expect("a cleanup deferral must be accepted while another bucket still defers an entry");
|
||||
{
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(
|
||||
pool_stat.info.last_error,
|
||||
Some(entry_error),
|
||||
"a cleanup deferral must not erase the pending migration deferral that blocks goal completion"
|
||||
);
|
||||
assert!(has_deferred_rebalance_error(pool_stat));
|
||||
assert_eq!(pool_stat.buckets, vec!["bucket-a".to_string(), "bucket-b".to_string()]);
|
||||
}
|
||||
|
||||
let migration_error = format!("{REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX} i/o timeout");
|
||||
store
|
||||
.defer_rebalance_bucket(0, "bucket-a".to_string(), migration_error.clone(), id, RebalanceDeferKind::Entry)
|
||||
.await
|
||||
.expect("migration deferrals must keep their last error");
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let pool_stat = &meta.as_ref().expect("rebalance metadata should exist").pool_stats[0];
|
||||
assert_eq!(pool_stat.info.last_error, Some(migration_error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -12,8 +12,8 @@ use super::worker::{
|
||||
};
|
||||
use super::{
|
||||
EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalSaveOpt, RebalStatus,
|
||||
RebalanceBucketOutcome,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_MAX_DEFERS,
|
||||
RebalSaveOpt, RebalStatus, RebalanceBucketOutcome, RebalanceDeferKind,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
@@ -40,6 +40,12 @@ pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<Strin
|
||||
*attempts
|
||||
}
|
||||
|
||||
/// Retryable source cleanup conflicts are bounded per run, so a source replica that stays
|
||||
/// unreclaimable fails the bucket explicitly instead of deferring forever.
|
||||
pub(super) fn reached_rebalance_source_cleanup_defer_limit(attempt: usize) -> bool {
|
||||
attempt >= REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum RebalanceLocalActivationOutcome {
|
||||
Started,
|
||||
@@ -623,6 +629,11 @@ impl ECStore {
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let defer_kind = if source_cleanup_deferred {
|
||||
RebalanceDeferKind::SourceCleanup
|
||||
} else {
|
||||
RebalanceDeferKind::Entry
|
||||
};
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -634,7 +645,7 @@ impl ECStore {
|
||||
"Deferred rebalance bucket after transient object failures"
|
||||
);
|
||||
if let Err(err) = self
|
||||
.defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone(), rebalance_id.as_ref())
|
||||
.defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone(), rebalance_id.as_ref(), defer_kind)
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
@@ -654,10 +665,10 @@ impl ECStore {
|
||||
break;
|
||||
}
|
||||
if source_cleanup_deferred {
|
||||
if source_cleanup_attempt >= super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
|
||||
if reached_rebalance_source_cleanup_defer_limit(source_cleanup_attempt) {
|
||||
let err = Error::other(format!(
|
||||
"rebalance bucket {bucket} source cleanup remained unstable after {} deferrals: {last_error}",
|
||||
super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
|
||||
));
|
||||
warn!(
|
||||
event = EVENT_REBALANCE_BUCKET,
|
||||
|
||||
@@ -51,6 +51,18 @@ pub(super) enum RebalanceEntryOutcome {
|
||||
Deferred { last_error: String },
|
||||
}
|
||||
|
||||
/// Why a rebalance bucket was put back at the end of the queue.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum RebalanceDeferKind {
|
||||
/// A transient object-migration failure. Persisting it as `lastError` keeps the pool
|
||||
/// from being completed at the free-space goal while the entry is still retried.
|
||||
Entry,
|
||||
/// A retryable source-cleanup conflict. The bucket stays queued and is retried, so a
|
||||
/// transient lock conflict must not be recorded as a permanent warning that would block
|
||||
/// pool completion for the rest of the run.
|
||||
SourceCleanup,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
pub enum RebalStatus {
|
||||
#[default]
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use super::migration::MigrationVersionResult;
|
||||
use super::{
|
||||
DEFAULT_REBALANCE_MAX_ATTEMPTS, EVENT_REBALANCE_LISTING, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
|
||||
REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV, REBALANCE_MIGRATION_LOCK_RETRY_CAP,
|
||||
REBALANCE_MIGRATION_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalanceBucketConfigs,
|
||||
RebalanceBucketOutcome, RebalanceEntryOutcome, Result,
|
||||
REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX, REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_MAX_ATTEMPTS_ENV,
|
||||
REBALANCE_MIGRATION_LOCK_RETRY_CAP, REBALANCE_MIGRATION_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX,
|
||||
RebalanceBucketConfigs, RebalanceBucketOutcome, RebalanceDeferKind, RebalanceEntryOutcome, Result,
|
||||
};
|
||||
use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
|
||||
use crate::core::pools::ListCallback;
|
||||
@@ -177,7 +177,7 @@ pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
|
||||
) -> RebalanceEntryCleanupResult {
|
||||
match result {
|
||||
Ok(_) => RebalanceEntryCleanupResult::Completed { warning: None },
|
||||
Err(SourceCleanupError::Storage(err)) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
Err(SourceCleanupError::Storage(err)) if is_source_cleanup_not_found(&err) => {
|
||||
RebalanceEntryCleanupResult::Completed { warning: None }
|
||||
}
|
||||
Err(SourceCleanupError::SourceChanged) => RebalanceEntryCleanupResult::Deferred {
|
||||
@@ -185,12 +185,25 @@ pub(super) fn resolve_rebalance_entry_cleanup_delete_result(
|
||||
"{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} source changed during cleanup preflight for {bucket}/{object_name}"
|
||||
),
|
||||
},
|
||||
// A transient cleanup failure is not evidence that the source replica is gone, so the
|
||||
// entry stays incomplete and the bucket is retried instead of recording a permanent
|
||||
// cleanup warning that would block pool completion.
|
||||
Err(SourceCleanupError::Storage(err)) if is_transient_rebalance_error(&err) => RebalanceEntryCleanupResult::Deferred {
|
||||
last_error: format!(
|
||||
"{REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX} transient source cleanup failure for {bucket}/{object_name} will be retried: {err}"
|
||||
),
|
||||
},
|
||||
Err(SourceCleanupError::Storage(err)) => RebalanceEntryCleanupResult::Completed {
|
||||
warning: Some(format!("rebalance cleanup delete failed for {bucket}/{object_name}: {err}")),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn is_source_cleanup_not_found(err: &Error) -> bool {
|
||||
let err = rebalance_error_source(err);
|
||||
is_err_object_not_found(err) || is_err_version_not_found(err)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_migrate_result_error(
|
||||
err: Option<Error>,
|
||||
pool_idx: usize,
|
||||
@@ -210,6 +223,23 @@ pub(super) fn should_defer_rebalance_entry_failure(err: &Error) -> bool {
|
||||
is_transient_rebalance_error(err)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_deferred_last_error(
|
||||
kind: RebalanceDeferKind,
|
||||
pending_entry_defer: Option<&str>,
|
||||
last_error: &str,
|
||||
) -> Option<String> {
|
||||
match kind {
|
||||
RebalanceDeferKind::Entry => Some(last_error.to_string()),
|
||||
// A retryable cleanup conflict is progress, not a pool failure, so it must not surface as
|
||||
// `lastError`. It also must not erase an unresolved migration deferral: that marker is the
|
||||
// only signal keeping the pool from completing at the free-space goal while an entry is
|
||||
// still retried, and the two deferrals can be reported by different buckets of one pool.
|
||||
RebalanceDeferKind::SourceCleanup => pending_entry_defer
|
||||
.filter(|pending| pending.starts_with(REBALANCE_DEFERRED_ENTRY_ERROR_PREFIX))
|
||||
.map(str::to_string),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resolve_load_rebalance_stats_update_result(result: Result<()>) -> Result<()> {
|
||||
result.map_err(|err| Error::other(format!("rebalance metadata stats refresh failed after load: {err}")))
|
||||
}
|
||||
@@ -299,7 +329,10 @@ fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
|
||||
|
||||
fn is_rebalance_transient_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
message.contains("lock acquisition timed out")
|
||||
// `LockError::Timeout` renders "Lock acquisition timeout for resource ...", while the
|
||||
// namespace-lock layer renders "lock acquisition timed out on ..."; both are retryable.
|
||||
message.contains("lock acquisition timeout")
|
||||
|| message.contains("lock acquisition timed out")
|
||||
|| message.contains("remote lock rpc timed out")
|
||||
|| message.contains("keepalivetimedout")
|
||||
|| message.contains("i/o timeout")
|
||||
@@ -380,7 +413,8 @@ fn is_rebalance_lock_or_rpc_timeout(err: &Error) -> bool {
|
||||
|
||||
fn is_rebalance_lock_or_rpc_timeout_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
message.contains("lock acquisition timed out")
|
||||
message.contains("lock acquisition timeout")
|
||||
|| message.contains("lock acquisition timed out")
|
||||
|| message.contains("remote lock rpc timed out")
|
||||
|| message.contains("keepalivetimedout")
|
||||
}
|
||||
@@ -807,4 +841,44 @@ mod error_source_tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn rendered_lock_timeout_text_selects_the_lock_backoff() {
|
||||
// The lock backend renders a timeout as "Lock acquisition timeout for resource ...",
|
||||
// so the message matcher must recognize that text when the error arrives re-rendered
|
||||
// instead of as a typed `Error::Lock`.
|
||||
let rendered = rustfs_lock::LockError::timeout("bucket/object@latest", Duration::from_secs(5)).to_string();
|
||||
assert!(
|
||||
rendered.contains("Lock acquisition timeout for resource"),
|
||||
"unexpected lock timeout text: {rendered}"
|
||||
);
|
||||
|
||||
// The lock policy jitters the delay inside its own cap, so a far-out attempt identifies
|
||||
// the selected policy: the linear fallback would return `base * (attempt + 1)`.
|
||||
let far_attempt = 100;
|
||||
assert!(REBALANCE_MIGRATION_RETRY_BASE_DELAY * 101 > REBALANCE_MIGRATION_LOCK_RETRY_CAP);
|
||||
|
||||
let mut error = Error::other(format!("Lock error: {rendered}"));
|
||||
for depth in 0..=3 {
|
||||
assert!(
|
||||
is_transient_rebalance_error(&error),
|
||||
"rendered lock timeout lost retryability at depth {depth}: {error:?}"
|
||||
);
|
||||
assert!(
|
||||
is_rebalance_lock_or_rpc_timeout(&error),
|
||||
"rendered lock timeout lost the lock backoff at depth {depth}: {error:?}"
|
||||
);
|
||||
let delay = rebalance_migration_retry_delay(far_attempt, &error);
|
||||
assert!(
|
||||
delay <= REBALANCE_MIGRATION_LOCK_RETRY_CAP && delay >= Duration::from_millis(1),
|
||||
"rendered lock timeout must stay inside the lock backoff cap at depth {depth}: {delay:?}"
|
||||
);
|
||||
error = crate::data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"baseline/00042.bin",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3494,19 +3494,44 @@ fn dangling_delete_grace() -> time::Duration {
|
||||
/// Result of scanning one disk's copy of a directory prefix while deciding
|
||||
/// whether an orphan (metadata-less) directory tree can be safely purged.
|
||||
enum OrphanDirScan {
|
||||
/// The subtree holds object metadata or uncommitted data, so it must not be
|
||||
/// purged.
|
||||
HasData,
|
||||
/// The prefix contains only empty directories and/or UUID data directories
|
||||
/// carrying a committed delete marker.
|
||||
Purgeable {
|
||||
empty_dirs: Vec<String>,
|
||||
/// The prefix exists on this disk. `blocked` holds every directory that is
|
||||
/// itself, or an ancestor of, object metadata or uncommitted data (closed
|
||||
/// under taking parents, root included when anything under it is blocked);
|
||||
/// `dirs` is the pre-order list of every directory reached that is not
|
||||
/// blocked, and `committed_files` the erasure data and committed delete
|
||||
/// markers found in the unblocked UUID data dirs among them.
|
||||
Scanned {
|
||||
blocked: HashSet<String>,
|
||||
dirs: Vec<String>,
|
||||
committed_files: Vec<String>,
|
||||
},
|
||||
/// A directory read failed for a reason other than absence, so nothing
|
||||
/// under the prefix can be classified on this disk.
|
||||
Unreadable,
|
||||
/// The prefix does not exist on this disk.
|
||||
Missing,
|
||||
}
|
||||
|
||||
/// How long an orphan prefix whose purge scan met unpurgeable data is left
|
||||
/// alone before an empty listing scans it again.
|
||||
const ORPHAN_PURGE_BACKOFF: Duration = Duration::from_secs(60);
|
||||
/// Upper bound on remembered backoff entries; the oldest is dropped first.
|
||||
const ORPHAN_PURGE_BACKOFF_MAX_ENTRIES: usize = 4096;
|
||||
|
||||
/// Mark `dir` and every ancestor up to and including `root` as blocked.
|
||||
fn block_orphan_dir_chain(blocked: &mut HashSet<String>, root: &str, dir: &str) {
|
||||
let mut current = dir;
|
||||
loop {
|
||||
if !blocked.insert(current.to_owned()) || current == root {
|
||||
return;
|
||||
}
|
||||
let Some((parent, _)) = current.rsplit_once(SLASH_SEPARATOR) else {
|
||||
return;
|
||||
};
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
fn is_safe_orphan_dir_entry(entry: &str) -> bool {
|
||||
let component = entry.strip_suffix(SLASH_SEPARATOR).unwrap_or(entry);
|
||||
!component.is_empty()
|
||||
@@ -6311,9 +6336,12 @@ impl SetDisks {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Scan a single disk's copy of `prefix` and decide whether it is an orphan
|
||||
/// directory subtree. Only empty directories and UUID data directories with
|
||||
/// valid committed delete markers are purgeable; every child is still scanned.
|
||||
/// Scan a single disk's copy of `prefix` and classify every directory
|
||||
/// under it. Only empty directories and UUID data directories with valid
|
||||
/// committed delete markers are purgeable; anything else blocks its whole
|
||||
/// ancestor chain while sibling subtrees stay purgeable, so committed
|
||||
/// residue is still reclaimed when it shares an ancestor with residue from
|
||||
/// an older build that never wrote markers (#6898).
|
||||
async fn scan_orphan_dir(disk: &DiskStore, bucket: &str, prefix: &str) -> OrphanDirScan {
|
||||
let root = prefix.trim_end_matches(SLASH_SEPARATOR).to_string();
|
||||
let mut stack = vec![root.clone()];
|
||||
@@ -6321,6 +6349,7 @@ impl SetDisks {
|
||||
// so reversing it yields a safe children-first removal order.
|
||||
let mut dirs: Vec<String> = Vec::new();
|
||||
let mut committed_files: Vec<String> = Vec::new();
|
||||
let mut blocked: HashSet<String> = HashSet::new();
|
||||
let mut existed = false;
|
||||
|
||||
while let Some(dir) = stack.pop() {
|
||||
@@ -6335,16 +6364,18 @@ impl SetDisks {
|
||||
}
|
||||
// Classification must fail closed: committed residue is safe to
|
||||
// remove only after every reachable child was inspected.
|
||||
Err(_) => return OrphanDirScan::HasData,
|
||||
Err(_) => return OrphanDirScan::Unreadable,
|
||||
};
|
||||
|
||||
existed = true;
|
||||
let mut child_dirs = Vec::new();
|
||||
let mut files = Vec::new();
|
||||
|
||||
let mut has_data = false;
|
||||
for entry in entries {
|
||||
if !is_safe_orphan_dir_entry(&entry) {
|
||||
return OrphanDirScan::HasData;
|
||||
has_data = true;
|
||||
break;
|
||||
}
|
||||
match entry.strip_suffix(SLASH_SEPARATOR) {
|
||||
Some(child) => child_dirs.push(format!("{dir}{SLASH_SEPARATOR}{child}")),
|
||||
@@ -6352,28 +6383,29 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if !files.is_empty() {
|
||||
if !has_data && !files.is_empty() {
|
||||
let data_dir_name = dir.rsplit(SLASH_SEPARATOR).next().unwrap_or_default();
|
||||
let is_uuid_data_dir = Uuid::parse_str(data_dir_name).is_ok_and(|uuid| !uuid.is_nil());
|
||||
let has_committed_delete = files.iter().any(|entry| is_committed_delete_marker(entry));
|
||||
has_data = !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE);
|
||||
}
|
||||
|
||||
if !is_uuid_data_dir || !has_committed_delete || files.iter().any(|entry| entry == STORAGE_FORMAT_FILE) {
|
||||
return OrphanDirScan::HasData;
|
||||
}
|
||||
|
||||
committed_files.extend(files.into_iter().map(|entry| path_join_buf(&[&dir, &entry])));
|
||||
dirs.push(dir);
|
||||
stack.extend(child_dirs);
|
||||
if has_data {
|
||||
// Nothing below a blocked directory is ever removed, so its
|
||||
// children need no classification.
|
||||
block_orphan_dir_chain(&mut blocked, &root, &dir);
|
||||
continue;
|
||||
}
|
||||
|
||||
committed_files.extend(files.into_iter().map(|entry| format!("{dir}{SLASH_SEPARATOR}{entry}")));
|
||||
dirs.push(dir);
|
||||
stack.extend(child_dirs);
|
||||
}
|
||||
|
||||
if existed {
|
||||
OrphanDirScan::Purgeable {
|
||||
empty_dirs: dirs,
|
||||
OrphanDirScan::Scanned {
|
||||
blocked,
|
||||
dirs,
|
||||
committed_files,
|
||||
}
|
||||
} else {
|
||||
@@ -6468,43 +6500,101 @@ impl SetDisks {
|
||||
/// of this set (the caller should surface the original NotFound), and `Err` on
|
||||
/// a hard disk failure.
|
||||
pub(crate) async fn purge_orphan_dir_object(&self, bucket: &str, object: &str) -> disk::error::Result<bool> {
|
||||
let backoff_key = format!("{bucket}{SLASH_SEPARATOR}{object}");
|
||||
if self.orphan_purge_in_backoff(&backoff_key) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
// Phase 1: classify every online disk. Refuse to purge if ANY disk holds
|
||||
// object data under the prefix, so a degraded/healable object is never
|
||||
// destroyed.
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<String>, Vec<String>)> = Vec::new();
|
||||
let mut existed = false;
|
||||
// Phase 1: classify every online disk. A directory that holds object
|
||||
// data or uncommitted residue on ANY disk blocks itself and its
|
||||
// ancestors on every disk, so a degraded/healable object is never
|
||||
// destroyed; purgeable subtrees beside it are still reclaimed.
|
||||
let mut per_disk: Vec<(usize, Vec<String>, Vec<String>)> = Vec::new();
|
||||
let mut blocked: HashSet<String> = HashSet::new();
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
let Some(disk) = disk else { continue };
|
||||
match Self::scan_orphan_dir(disk, bucket, object).await {
|
||||
OrphanDirScan::HasData => return Ok(false),
|
||||
OrphanDirScan::Purgeable {
|
||||
empty_dirs,
|
||||
OrphanDirScan::Unreadable => return Ok(false),
|
||||
OrphanDirScan::Scanned {
|
||||
blocked: disk_blocked,
|
||||
dirs,
|
||||
committed_files,
|
||||
} => {
|
||||
existed = true;
|
||||
per_disk_dirs.push((i, empty_dirs, committed_files));
|
||||
blocked.extend(disk_blocked);
|
||||
per_disk.push((i, dirs, committed_files));
|
||||
}
|
||||
OrphanDirScan::Missing => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !existed {
|
||||
return Ok(false);
|
||||
if !blocked.is_empty() {
|
||||
// Whatever is purgeable goes now; what blocks the rest will still
|
||||
// block it on the next empty listing, so do not rescan for a while.
|
||||
self.record_orphan_purge_backoff(backoff_key);
|
||||
}
|
||||
|
||||
// Phase 2: remove only the files classified as committed residue, then
|
||||
// remove directories children-first. Every directory delete is
|
||||
// non-recursive, so a directory that concurrently gained an object fails
|
||||
// with DirectoryNotEmpty and is skipped — a racing PutObject is never
|
||||
// Phase 2: remove only the files classified as committed residue in
|
||||
// unblocked data dirs, then remove unblocked directories
|
||||
// children-first. Every directory delete is non-recursive, so a
|
||||
// directory that concurrently gained an object fails with
|
||||
// DirectoryNotEmpty and is skipped — a racing PutObject is never
|
||||
// clobbered.
|
||||
for (i, empty_dirs, committed_files) in per_disk_dirs {
|
||||
let mut purged = false;
|
||||
for (i, dirs, committed_files) in per_disk {
|
||||
let Some(disk) = disks[i].as_ref() else { continue };
|
||||
Self::delete_purgeable_orphan_entries(disk, bucket, object, empty_dirs, committed_files).await;
|
||||
let dirs = dirs.into_iter().filter(|dir| !blocked.contains(dir)).collect::<Vec<_>>();
|
||||
let committed_files = committed_files
|
||||
.into_iter()
|
||||
.filter(|file| {
|
||||
file.rsplit_once(SLASH_SEPARATOR)
|
||||
.is_some_and(|(data_dir, _)| !blocked.contains(data_dir))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if dirs.is_empty() && committed_files.is_empty() {
|
||||
continue;
|
||||
}
|
||||
purged = true;
|
||||
Self::delete_purgeable_orphan_entries(disk, bucket, object, dirs, committed_files).await;
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
Ok(purged)
|
||||
}
|
||||
|
||||
fn orphan_purge_in_backoff(&self, key: &str) -> bool {
|
||||
let backoff = self
|
||||
.orphan_purge_backoff
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
backoff
|
||||
.get(key)
|
||||
.is_some_and(|scanned_at| scanned_at.elapsed() < ORPHAN_PURGE_BACKOFF)
|
||||
}
|
||||
|
||||
fn record_orphan_purge_backoff(&self, key: String) {
|
||||
let mut backoff = self
|
||||
.orphan_purge_backoff
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let now = Instant::now();
|
||||
backoff.retain(|_, scanned_at| now.duration_since(*scanned_at) < ORPHAN_PURGE_BACKOFF);
|
||||
if backoff.len() >= ORPHAN_PURGE_BACKOFF_MAX_ENTRIES
|
||||
&& let Some(oldest) = backoff
|
||||
.iter()
|
||||
.min_by_key(|(_, scanned_at)| **scanned_at)
|
||||
.map(|(key, _)| key.clone())
|
||||
{
|
||||
backoff.remove(&oldest);
|
||||
}
|
||||
backoff.insert(key, now);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn clear_orphan_purge_backoff(&self) {
|
||||
self.orphan_purge_backoff
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clear();
|
||||
}
|
||||
|
||||
/// Reclaim orphaned physical data directories under `bucket/object` that no
|
||||
@@ -7621,13 +7711,15 @@ mod tests {
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
|
||||
let OrphanDirScan::Purgeable {
|
||||
empty_dirs,
|
||||
let OrphanDirScan::Scanned {
|
||||
blocked,
|
||||
dirs: empty_dirs,
|
||||
committed_files,
|
||||
} = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
|
||||
else {
|
||||
panic!("committed residue should be classified as purgeable");
|
||||
};
|
||||
assert!(blocked.is_empty(), "committed residue alone blocks nothing");
|
||||
|
||||
let nested_object = residue.join("nested");
|
||||
tokio::fs::create_dir_all(&nested_object)
|
||||
@@ -7669,13 +7761,15 @@ mod tests {
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
|
||||
let OrphanDirScan::Purgeable {
|
||||
empty_dirs,
|
||||
let OrphanDirScan::Scanned {
|
||||
blocked,
|
||||
dirs: empty_dirs,
|
||||
committed_files,
|
||||
} = SetDisks::scan_orphan_dir(&disk, "bucket", "pfx/").await
|
||||
else {
|
||||
panic!("committed residue should be classified as purgeable");
|
||||
};
|
||||
assert!(blocked.is_empty(), "committed residue alone blocks nothing");
|
||||
tokio::fs::set_permissions(&residue, std::fs::Permissions::from_mode(0o555))
|
||||
.await
|
||||
.expect("residue directory should become read-only");
|
||||
|
||||
@@ -3900,6 +3900,12 @@ pub struct SetDisks {
|
||||
/// writes skip the global registry mutex (backlog#1315). `Arc` so clones of
|
||||
/// a set share one generation marker.
|
||||
capacity_dirty_generation: Arc<AtomicU64>,
|
||||
/// Orphan prefixes whose last purge scan met data that can never be
|
||||
/// purged by listing (residue without a committed marker, an in-flight
|
||||
/// write), keyed by `bucket/prefix` with the time of that scan. Empty
|
||||
/// listings of such a prefix are frequent and the scan reads the whole
|
||||
/// subtree, so it is not repeated within `ORPHAN_PURGE_BACKOFF` (#6898).
|
||||
orphan_purge_backoff: Arc<std::sync::Mutex<HashMap<String, std::time::Instant>>>,
|
||||
#[cfg(test)]
|
||||
storage_class_config_override: Arc<std::sync::RwLock<Option<Arc<storageclass::Config>>>>,
|
||||
#[cfg(test)]
|
||||
@@ -4690,6 +4696,7 @@ impl SetDisks {
|
||||
ctx,
|
||||
capacity_scope_cache: Arc::new(std::sync::RwLock::new(CapacityScopeCache::default())),
|
||||
capacity_dirty_generation: Arc::new(AtomicU64::new(u64::MAX)),
|
||||
orphan_purge_backoff: Arc::new(std::sync::Mutex::new(HashMap::new())),
|
||||
#[cfg(test)]
|
||||
storage_class_config_override: Arc::new(std::sync::RwLock::new(None)),
|
||||
#[cfg(test)]
|
||||
@@ -4703,6 +4710,12 @@ impl SetDisks {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
self.local_lock_manager = ctx.lock_manager();
|
||||
self.ctx = ctx;
|
||||
}
|
||||
|
||||
/// Read the persisted bucket identity through this set's metadata owner.
|
||||
/// Missing or non-authoritative legacy identities remain errors.
|
||||
pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
|
||||
@@ -9331,6 +9344,175 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn write_committed_residue(object_dir: &std::path::Path) -> std::path::PathBuf {
|
||||
let residue = object_dir.join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&residue)
|
||||
.await
|
||||
.expect("committed data directory should be created");
|
||||
fs::write(residue.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
fs::write(
|
||||
residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, Uuid::new_v4())),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
residue
|
||||
}
|
||||
|
||||
// #6898: residue from a build that never wrote delete markers blocks only
|
||||
// its own ancestor chain; a committed sibling subtree is still reclaimed
|
||||
// and the call reports the partial purge.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_reclaims_committed_subtree_beside_blocked_subtree() {
|
||||
let (dir0, disk0) = make_single_local_disk().await;
|
||||
let (dir1, disk1) = make_single_local_disk().await;
|
||||
|
||||
let committed = write_committed_residue(&dir0.path().join("bucket/pfx/a/obj")).await;
|
||||
let unmarked = dir1.path().join("bucket/pfx/b/obj").join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&unmarked)
|
||||
.await
|
||||
.expect("unmarked residue should be created");
|
||||
fs::write(unmarked.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
fs::create_dir_all(dir0.path().join("bucket/pfx/b/obj"))
|
||||
.await
|
||||
.expect("disk0 copy of the blocked chain should be created");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk0), Some(disk1)]).await;
|
||||
let purged = set
|
||||
.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed");
|
||||
|
||||
assert!(purged, "the committed subtree must be reported as purged");
|
||||
assert!(!committed.exists(), "committed residue beside a blocked subtree must be reclaimed");
|
||||
assert!(!dir0.path().join("bucket/pfx/a").exists(), "the reclaimed subtree's directories must go");
|
||||
assert!(unmarked.join("part.1").exists(), "unmarked residue must survive");
|
||||
assert!(
|
||||
dir0.path().join("bucket/pfx/b/obj").exists(),
|
||||
"a directory blocked on another disk must be left alone on this one"
|
||||
);
|
||||
assert!(dir0.path().join("bucket/pfx").exists() && dir1.path().join("bucket/pfx").exists());
|
||||
}
|
||||
|
||||
// A data dir that is committed residue on one disk but still holds an
|
||||
// object below it on another disk is blocked on every disk.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_blocks_committed_files_by_other_disks_data() {
|
||||
let (dir0, disk0) = make_single_local_disk().await;
|
||||
let (dir1, disk1) = make_single_local_disk().await;
|
||||
|
||||
let data_dir = Uuid::new_v4();
|
||||
let committed = dir0.path().join("bucket/pfx/x").join(data_dir.to_string());
|
||||
fs::create_dir_all(&committed)
|
||||
.await
|
||||
.expect("committed data directory should be created");
|
||||
fs::write(committed.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
fs::write(
|
||||
committed.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, Uuid::new_v4())),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
let nested = dir1.path().join("bucket/pfx/x").join(data_dir.to_string()).join("nested");
|
||||
fs::create_dir_all(&nested)
|
||||
.await
|
||||
.expect("nested object dir should be created");
|
||||
fs::write(nested.join(STORAGE_FORMAT_FILE), b"meta")
|
||||
.await
|
||||
.expect("nested object metadata should be written");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk0), Some(disk1)]).await;
|
||||
let purged = set
|
||||
.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed");
|
||||
|
||||
assert!(!purged);
|
||||
assert!(
|
||||
committed.join("part.1").exists(),
|
||||
"committed files under a dir blocked elsewhere must remain"
|
||||
);
|
||||
assert!(nested.join(STORAGE_FORMAT_FILE).exists());
|
||||
}
|
||||
|
||||
// An unreadable directory anywhere under the prefix aborts the purge on
|
||||
// every disk before anything is deleted.
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_aborts_when_a_directory_is_unreadable() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let committed = write_committed_residue(&dir.path().join("bucket/pfx/b/obj")).await;
|
||||
let sealed = dir.path().join("bucket/pfx/a");
|
||||
fs::create_dir_all(&sealed).await.expect("sealed directory should be created");
|
||||
fs::set_permissions(&sealed, std::fs::Permissions::from_mode(0o000))
|
||||
.await
|
||||
.expect("sealed directory should become unreadable");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
let purged = set.purge_orphan_dir_object("bucket", "pfx/").await;
|
||||
fs::set_permissions(&sealed, std::fs::Permissions::from_mode(0o755))
|
||||
.await
|
||||
.expect("sealed directory should be restored");
|
||||
|
||||
assert!(matches!(purged, Ok(false)), "an unreadable directory must abort the purge");
|
||||
assert!(committed.join("part.1").exists(), "nothing may be deleted once classification failed");
|
||||
}
|
||||
|
||||
// After a scan met unpurgeable data the prefix is not rescanned for a
|
||||
// while, even when new committed residue appears under it.
|
||||
#[tokio::test]
|
||||
async fn purge_orphan_dir_object_backs_off_after_blocked_scan() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let unmarked = dir.path().join("bucket/pfx/b/obj").join(Uuid::new_v4().to_string());
|
||||
fs::create_dir_all(&unmarked)
|
||||
.await
|
||||
.expect("unmarked residue should be created");
|
||||
fs::write(unmarked.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
assert!(
|
||||
!set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed")
|
||||
);
|
||||
|
||||
let committed = write_committed_residue(&dir.path().join("bucket/pfx/a/obj")).await;
|
||||
assert!(
|
||||
!set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed"),
|
||||
"a prefix in backoff is not rescanned"
|
||||
);
|
||||
assert!(committed.exists());
|
||||
assert!(
|
||||
set.purge_orphan_dir_object("bucket", "pfx/a/")
|
||||
.await
|
||||
.expect("scan should succeed"),
|
||||
"backoff is per prefix, a narrower prefix still purges"
|
||||
);
|
||||
assert!(!committed.exists());
|
||||
|
||||
let committed = write_committed_residue(&dir.path().join("bucket/pfx/a/obj")).await;
|
||||
set.clear_orphan_purge_backoff();
|
||||
assert!(
|
||||
set.purge_orphan_dir_object("bucket", "pfx/")
|
||||
.await
|
||||
.expect("scan should succeed")
|
||||
);
|
||||
assert!(!committed.exists());
|
||||
assert!(unmarked.join("part.1").exists());
|
||||
}
|
||||
|
||||
// Build an `xl.meta` under `object_dir` whose versions reference `data_dirs`
|
||||
// (one Object version per data dir, each with its own version id).
|
||||
async fn write_object_meta_with_data_dirs(object_dir: &std::path::Path, bucket: &str, object: &str, data_dirs: &[Uuid]) {
|
||||
|
||||
@@ -2613,16 +2613,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
fi.parts = Vec::with_capacity(uploaded_parts.len());
|
||||
|
||||
let quota_context = reservation::begin(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts.quota_admission,
|
||||
opts.data_movement,
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
)
|
||||
.await?;
|
||||
let quota_context = reservation::begin(&self.ctx, bucket, object, opts, self.pool_index, self.set_index).await?;
|
||||
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
|
||||
let preserve_replication_ciphertext = opts.replication_request
|
||||
&& contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
|
||||
|
||||
@@ -3427,8 +3427,8 @@ impl SetDisks {
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
async fn validate_bucket_incarnation(&self, bucket: &str, expected: Uuid) -> Result<()> {
|
||||
let current = metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
async fn validate_bucket_incarnation(&self, bucket: &str, expected: Uuid, opts: &ObjectOptions) -> Result<()> {
|
||||
let current = metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, opts).await?;
|
||||
if current != expected {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -3636,13 +3636,22 @@ impl SetDisks {
|
||||
// Transformed streams (unknown stored size) are admitted by their
|
||||
// plaintext size and then take the streaming encode with inline
|
||||
// buffer writers, since the single-block fast path needs a known length.
|
||||
// Protected inline writes retain the single-block bound on proof metadata.
|
||||
// Every inline candidate keeps the single-block bound on its admission
|
||||
// size: the inline buffer writer grows without a cap, so an explicit
|
||||
// INLINE_BLOCK budget above `block_size` must not admit multi-block
|
||||
// payloads into memory and xl.meta. Protected inline writes also need
|
||||
// a known stored length for their proof metadata, so they never fall
|
||||
// back to the plaintext size.
|
||||
let inline_admission_size = if put_object_size >= 0 || protect_write {
|
||||
put_object_size
|
||||
} else {
|
||||
data.actual_size()
|
||||
};
|
||||
let is_inline_buffer = storage_class_config.should_inline(
|
||||
inline_admission_shard_size(&erasure, put_object_size, data.actual_size()),
|
||||
erasure.data_shards,
|
||||
opts.versioned,
|
||||
) && (!protect_write
|
||||
|| usize::try_from(put_object_size).is_ok_and(|size| size <= erasure.block_size));
|
||||
) && usize::try_from(inline_admission_size).is_ok_and(|size| size <= erasure.block_size);
|
||||
|
||||
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
|
||||
let shard_file_size = shard_file_size_raw;
|
||||
@@ -4234,16 +4243,7 @@ impl SetDisks {
|
||||
None
|
||||
};
|
||||
|
||||
let quota_context = reservation::begin(
|
||||
&self.ctx,
|
||||
bucket,
|
||||
object,
|
||||
opts.quota_admission,
|
||||
opts.data_movement,
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
)
|
||||
.await?;
|
||||
let quota_context = reservation::begin(&self.ctx, bucket, object, opts, self.pool_index, self.set_index).await?;
|
||||
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
|
||||
let mut replication_quota_size = None;
|
||||
|
||||
@@ -8869,7 +8869,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id {
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id, &opts)
|
||||
.await?;
|
||||
}
|
||||
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
|
||||
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
|
||||
@@ -9283,7 +9284,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
check_object_lock_retention_update(bucket, object, &obj_info, opts)?;
|
||||
|
||||
if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id {
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id, opts)
|
||||
.await?;
|
||||
}
|
||||
if _lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| opts
|
||||
@@ -9863,7 +9865,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
.await?
|
||||
.acquire_bucket_lifecycle_read_lock(bucket)
|
||||
.await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
|
||||
self.validate_bucket_incarnation(bucket, expected_incarnation_id, opts)
|
||||
.await?;
|
||||
Some(guard)
|
||||
} else {
|
||||
None
|
||||
@@ -11801,6 +11804,54 @@ mod inline_put_commit_path_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn unprotected_puts_keep_multi_block_inline_candidates_external() {
|
||||
for transformed in [true, false] {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let storage_class =
|
||||
temp_env::with_var(INLINE_BLOCK_ENV, Some("16MiB"), || lookup_config_for_pools(&KVS::new(), &[4]))
|
||||
.expect("large inline budget should resolve");
|
||||
set_disks.set_test_storage_class_config(storage_class);
|
||||
let bucket = "unprotected-inline-candidates";
|
||||
let object = "object.bin";
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
// Above the single erasure block: an explicit INLINE_BLOCK budget must
|
||||
// not pull a multi-block payload into the unbounded inline buffer.
|
||||
let plaintext = compressible_payload(1024 * 1024 + 17);
|
||||
let (mut reader, opts) = if transformed {
|
||||
transformed_put(plaintext.clone())
|
||||
} else {
|
||||
(PutObjReader::from_vec(plaintext.clone()), ObjectOptions::default())
|
||||
};
|
||||
temp_env::async_with_vars(
|
||||
[(
|
||||
crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
|
||||
Some("false"),
|
||||
)],
|
||||
set_disks.put_object(bucket, object, &mut reader, &opts),
|
||||
)
|
||||
.await
|
||||
.expect("multi-block PUT should commit with part files");
|
||||
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let file_info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk {disk_index} should persist metadata: {err}"));
|
||||
assert!(
|
||||
!file_info.inline_data(),
|
||||
"disk {disk_index}: a payload above block_size must stay external (transformed={transformed})"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
count_part_files(&temp_dirs) > 0,
|
||||
"a multi-block object must keep part files (transformed={transformed})"
|
||||
);
|
||||
assert_eq!(read_back(&set_disks, bucket, object).await, plaintext);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transformed_put_above_inline_budget_keeps_part_files() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
|
||||
@@ -2776,7 +2776,16 @@ mod tests {
|
||||
|
||||
let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
|
||||
.put_object(
|
||||
&bucket,
|
||||
active_object,
|
||||
&mut active_reader,
|
||||
&ObjectOptions {
|
||||
// Drain the rename tail before removing a physical shard.
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("active owner object should be written");
|
||||
let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
|
||||
@@ -2812,6 +2821,7 @@ mod tests {
|
||||
&mut duplicate_reader,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
|
||||
@@ -2905,8 +2905,17 @@ mod tests {
|
||||
if !version.is_empty() {
|
||||
insert_str(&mut metadata, SUFFIX_TRANSITIONED_VERSION_ID, version.to_string());
|
||||
}
|
||||
// The commit path acknowledges at write quorum and drains
|
||||
// the remaining rename fan-out in the background. This
|
||||
// fixture "crashes" the store right after the overwrite,
|
||||
// so it must wait for that tail: a disk left with the old
|
||||
// live transitioned source makes exact cleanup fail closed
|
||||
// (a minority live owner still references the remote
|
||||
// tuple) until heal repairs it, which this fixture never
|
||||
// runs (rustfs#7921).
|
||||
let options = ObjectOptions {
|
||||
version_suspended: suspended,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
store
|
||||
@@ -2978,6 +2987,7 @@ mod tests {
|
||||
assert_eq!(free.len(), 1, "{state:?}, suspended={suspended}, copy={self_copy}");
|
||||
assert_eq!(free[0].transitioned_objname, remote);
|
||||
assert_eq!(free[0].transition_version_state, state);
|
||||
assert_every_disk_holds_only_the_cleanup_owner(&set, &bucket, object, &remote).await;
|
||||
assert!(backend.contains(&remote).await, "commit must not delete remote bytes before cleanup");
|
||||
let removed_before = backend.remove_count().await;
|
||||
|
||||
@@ -3149,6 +3159,120 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#7674: a bucket carrying a legacy snapshot-protocol quota
|
||||
/// (written before durable reservations existed, so `quota.json` has no
|
||||
/// `reservation_protocol`) must accept a cross-key CopyObject whose
|
||||
/// destination options carry the handler's quota admission. The storage
|
||||
/// layer fails closed with `PartMissingOrCorrupt` when a quota-enforced
|
||||
/// bucket sees a write without admission, so dropping the admission while
|
||||
/// rebuilding the destination options turns every copy into a
|
||||
/// deterministic "part missing or corrupt" failure even though the source
|
||||
/// object is perfectly readable.
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn copy_object_forwards_quota_admission_on_legacy_snapshot_quota_bucket() {
|
||||
let temp_dir = tempfile::tempdir().expect("create legacy quota copy store dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-quota-copy", &[1])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
|
||||
let bucket = format!("legacy-quota-copy-{}", Uuid::new_v4());
|
||||
let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
|
||||
let target_object = "docker/registry/v2/blobs/sha256/a5/digest/data";
|
||||
let payload = vec![0x5A; 8178];
|
||||
let quota_limit = 1u64 << 30;
|
||||
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for legacy quota copy");
|
||||
// Legacy quota shape: no `reservation_protocol`, so the storage layer
|
||||
// requires the handler-supplied snapshot admission on every write.
|
||||
let legacy_quota = format!(r#"{{"quota":{quota_limit},"quota_type":"Hard"}}"#);
|
||||
crate::bucket::metadata_sys::update_in(
|
||||
&ctx,
|
||||
&bucket,
|
||||
crate::bucket::metadata::BUCKET_QUOTA_CONFIG_FILE,
|
||||
legacy_quota.into_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("persist legacy snapshot quota");
|
||||
let (quota, _, _) = crate::bucket::metadata_sys::get_quota_config_and_incarnation_from_disk_in(&ctx, &bucket)
|
||||
.await
|
||||
.expect("legacy quota should load");
|
||||
let quota = quota.expect("legacy quota must be persisted");
|
||||
assert_eq!(quota.quota, Some(quota_limit));
|
||||
assert!(!quota.uses_durable_reservations(), "fixture must stay on the snapshot protocol");
|
||||
|
||||
let mut write_opts = ObjectOptions::default();
|
||||
assert!(write_opts.set_quota_admission(0, quota_limit));
|
||||
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, source_object, &write_opts)
|
||||
.await
|
||||
.expect("create source multipart upload");
|
||||
let mut part_reader = PutObjReader::from_vec(payload.clone());
|
||||
let part = store
|
||||
.put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &write_opts)
|
||||
.await
|
||||
.expect("stage multipart source part");
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
source_object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&write_opts,
|
||||
)
|
||||
.await
|
||||
.expect("complete the multipart source under the legacy quota");
|
||||
|
||||
let source_reader = store
|
||||
.get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed multipart source should be readable");
|
||||
let mut copy_info = source_reader.object_info.clone();
|
||||
let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
|
||||
assert_eq!(actual_size, payload.len() as i64);
|
||||
let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
|
||||
.expect("copy source hash reader should build");
|
||||
copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
|
||||
|
||||
let mut dst_opts = ObjectOptions::default();
|
||||
assert!(dst_opts.set_quota_admission(payload.len() as u64, quota_limit));
|
||||
store
|
||||
.copy_object(
|
||||
&bucket,
|
||||
source_object,
|
||||
&bucket,
|
||||
target_object,
|
||||
&mut copy_info,
|
||||
&ObjectOptions::default(),
|
||||
&dst_opts,
|
||||
)
|
||||
.await
|
||||
.expect("CopyObject must forward the handler quota admission to the destination write");
|
||||
|
||||
let mut target_reader = store
|
||||
.get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("copied target should be readable");
|
||||
let mut target_body = Vec::new();
|
||||
target_reader
|
||||
.stream
|
||||
.read_to_end(&mut target_body)
|
||||
.await
|
||||
.expect("target body should stream");
|
||||
assert_eq!(target_body, payload);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
@@ -3966,7 +4090,7 @@ mod tests {
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disks {
|
||||
disk.close().await.expect("fault injection should stop per-disk monitoring");
|
||||
disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
disk.force_offline_for_test();
|
||||
}
|
||||
|
||||
// Sets has an independent endpoint monitor that renews missing slots.
|
||||
@@ -4003,7 +4127,7 @@ mod tests {
|
||||
.collect::<Vec<_>>();
|
||||
for disk in &disks {
|
||||
disk.close().await.expect("fault injection should stop per-disk monitoring");
|
||||
disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
disk.force_offline_for_test();
|
||||
}
|
||||
set.connect_disks().await;
|
||||
for disk in &disks {
|
||||
@@ -9712,14 +9836,7 @@ mod tests {
|
||||
.await
|
||||
.expect("manual task receipt path should resolve");
|
||||
let target_task_set = store.pools[1].get_disks_by_key(&manual_task_receipt_path);
|
||||
let original_target_task_disks = {
|
||||
let mut disks = target_task_set.disks.write().await;
|
||||
let original = disks.clone();
|
||||
for disk in disks.iter_mut().take(2) {
|
||||
*disk = None;
|
||||
}
|
||||
original
|
||||
};
|
||||
let offline_target_task_disks = force_set_disk_range_offline_for_test(&target_task_set, 0..2).await;
|
||||
let receipt_quorum_error = store
|
||||
.verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
|
||||
0,
|
||||
@@ -9728,7 +9845,7 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect_err("target read quorum without receipt write quorum must retain the source");
|
||||
*target_task_set.disks.write().await = original_target_task_disks;
|
||||
drop(offline_target_task_disks);
|
||||
let receipt_quorum_error = receipt_quorum_error.to_string();
|
||||
assert!(receipt_quorum_error.contains("receipt"));
|
||||
assert!(receipt_quorum_error.contains(&manual_task_path));
|
||||
@@ -12648,10 +12765,45 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Every physical copy must carry the replacement plus its tier
|
||||
/// free-version owner, and none may still hold the pre-overwrite live
|
||||
/// transitioned source. A stale minority copy is exactly what a lost
|
||||
/// early-ACK rename tail leaves behind, and exact cleanup refuses to
|
||||
/// delete remote bytes while such a live reference exists.
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn assert_every_disk_holds_only_the_cleanup_owner(
|
||||
set: &crate::set_disk::SetDisks,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
remote: &str,
|
||||
) {
|
||||
use crate::disk::DiskAPI as _;
|
||||
|
||||
let disk_object = rustfs_utils::path::encode_dir_object(object);
|
||||
for (index, disk) in set.disk_inventory().await.into_iter().enumerate() {
|
||||
let disk = disk.unwrap_or_else(|| panic!("disk{index} should be online"));
|
||||
let raw = disk
|
||||
.read_xl(bucket, &disk_object, false)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("disk{index} xl.meta should be readable after the overwrite: {err:?}"));
|
||||
let versions = rustfs_filemeta::FileMeta::load(&raw.buf)
|
||||
.and_then(|meta| meta.get_all_file_info_versions(bucket, object, true))
|
||||
.unwrap_or_else(|err| panic!("disk{index} xl.meta should decode: {err:?}"));
|
||||
let all: Vec<_> = versions.versions.iter().chain(versions.free_versions.iter()).collect();
|
||||
let owners = all.iter().filter(|fi| fi.tier_free_version()).count();
|
||||
let live_sources = all
|
||||
.iter()
|
||||
.filter(|fi| !fi.tier_free_version() && fi.transitioned_objname == remote)
|
||||
.count();
|
||||
assert_eq!(owners, 1, "disk{index} must hold exactly one cleanup owner after the drained overwrite");
|
||||
assert_eq!(live_sources, 0, "disk{index} must not retain the pre-overwrite live transitioned source");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn wait_for_expiry_workers_idle(store: &crate::store::ECStore) {
|
||||
let expiry_state = store.ctx.expiry_state();
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
let idle = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let idle = {
|
||||
let state = expiry_state.read().await;
|
||||
@@ -12671,8 +12823,15 @@ mod tests {
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("lifecycle expiry workers should become idle");
|
||||
.await;
|
||||
if idle.is_err() {
|
||||
let state = expiry_state.read().await;
|
||||
panic!(
|
||||
"lifecycle expiry workers should become idle: pending={} active={}",
|
||||
state.pending_tasks(),
|
||||
state.active_tasks()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -9678,6 +9678,131 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_delimiter_listing_of_residue_ancestor_hides_and_purges_whole_tree() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "listing-purge-ancestor-bucket";
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created with authoritative metadata");
|
||||
let data_dir = uuid::Uuid::new_v4();
|
||||
let transaction = uuid::Uuid::new_v4();
|
||||
for dir in &dirs {
|
||||
let residue = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("metrics/kubelet/2026/08/28/23/74992556388248657933757.parquet")
|
||||
.join(data_dir.to_string());
|
||||
tokio::fs::create_dir_all(&residue)
|
||||
.await
|
||||
.expect("committed delete residue should be created");
|
||||
tokio::fs::write(residue.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
tokio::fs::write(
|
||||
residue.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
}
|
||||
|
||||
// Browsing an ancestor of the deleted key must not show the empty
|
||||
// date folders, and the empty result reclaims the whole residue tree
|
||||
// in one pass instead of one level per listing.
|
||||
let result = store
|
||||
.clone()
|
||||
.list_objects_generic(bucket, "metrics/", None, Some("/".to_owned()), 1000, false)
|
||||
.await
|
||||
.expect("delimiter listing should succeed");
|
||||
assert!(result.objects.is_empty());
|
||||
assert!(result.prefixes.is_empty(), "ancestors of delete residue must not surface as prefixes");
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
!dir.path().join(bucket).join("metrics").exists(),
|
||||
"the empty delimiter listing should reclaim the committed delete residue tree under it"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_delimiter_listing_of_mixed_residue_ancestor_reclaims_only_committed_subtree() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "listing-purge-mixed-bucket";
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created with authoritative metadata");
|
||||
let committed_dir = uuid::Uuid::new_v4();
|
||||
let unmarked_dir = uuid::Uuid::new_v4();
|
||||
let transaction = uuid::Uuid::new_v4();
|
||||
for dir in &dirs {
|
||||
let committed = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("metrics/cpu/2026/08/28/22/a.parquet")
|
||||
.join(committed_dir.to_string());
|
||||
tokio::fs::create_dir_all(&committed)
|
||||
.await
|
||||
.expect("committed delete residue should be created");
|
||||
tokio::fs::write(committed.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
tokio::fs::write(
|
||||
committed.join(format!("{}{}", crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX, transaction)),
|
||||
[],
|
||||
)
|
||||
.await
|
||||
.expect("committed delete marker should be written");
|
||||
|
||||
// Residue left by a build that never wrote delete markers, or a
|
||||
// PUT still streaming its parts: indistinguishable, so never purged.
|
||||
let unmarked = dir
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("metrics/kubelet/2026/08/28/23/74992556388248657933757.parquet")
|
||||
.join(unmarked_dir.to_string());
|
||||
tokio::fs::create_dir_all(&unmarked)
|
||||
.await
|
||||
.expect("unmarked residue should be created");
|
||||
tokio::fs::write(unmarked.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("stale part should be written");
|
||||
}
|
||||
|
||||
let result = store
|
||||
.clone()
|
||||
.list_objects_generic(bucket, "metrics/", None, Some("/".to_owned()), 1000, false)
|
||||
.await
|
||||
.expect("delimiter listing should succeed");
|
||||
assert!(result.objects.is_empty());
|
||||
assert!(result.prefixes.is_empty(), "neither residue subtree may surface as a prefix");
|
||||
for dir in &dirs {
|
||||
let metrics = dir.path().join(bucket).join("metrics");
|
||||
assert!(
|
||||
!metrics.join("cpu").exists(),
|
||||
"the committed subtree must be reclaimed even though a sibling subtree is blocked"
|
||||
);
|
||||
assert!(
|
||||
metrics
|
||||
.join("kubelet/2026/08/28/23/74992556388248657933757.parquet")
|
||||
.join(unmarked_dir.to_string())
|
||||
.join("part.1")
|
||||
.exists(),
|
||||
"residue without a committed marker must survive the purge"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_objects_index_provider_state_uses_lifecycle_active_generation() {
|
||||
let provider = ListObjectsIndexProviderState::walker_key_only();
|
||||
|
||||
@@ -395,12 +395,19 @@ async fn scan_metadata_less_residue_with_budget(
|
||||
Ok(scan)
|
||||
}
|
||||
|
||||
async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEventSrc) -> Result<ObjectInfo> {
|
||||
async fn enqueue_transition_after_write(
|
||||
store: &ECStore,
|
||||
result: Result<ObjectInfo>,
|
||||
src: LcEventSrc,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
match result {
|
||||
Ok(oi) => {
|
||||
if should_enqueue_transition_immediately(&oi) {
|
||||
enqueue_transition_immediate(&oi, src.clone()).await;
|
||||
enqueue_immediate_expiry(&oi, src).await;
|
||||
if let Ok(api) = metadata_sys::object_store_in(&store.ctx).await {
|
||||
enqueue_immediate_expiry(api, &oi, src, opts).await;
|
||||
}
|
||||
}
|
||||
Ok(oi)
|
||||
}
|
||||
@@ -1281,9 +1288,11 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
let result = match self.handle_put_object(bucket, object, data, opts).await {
|
||||
Ok((object_info, old_current_size)) => enqueue_transition_after_write(Ok(object_info), LcEventSrc::S3PutObject)
|
||||
.await
|
||||
.map(|object_info| (object_info, old_current_size)),
|
||||
Ok((object_info, old_current_size)) => {
|
||||
enqueue_transition_after_write(self, Ok(object_info), LcEventSrc::S3PutObject, opts)
|
||||
.await
|
||||
.map(|object_info| (object_info, old_current_size))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if result.is_ok() {
|
||||
@@ -1356,9 +1365,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
|
||||
dst_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let result = enqueue_transition_after_write(
|
||||
self,
|
||||
self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
|
||||
.await,
|
||||
LcEventSrc::S3CopyObject,
|
||||
dst_opts,
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
@@ -1625,10 +1636,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let result = enqueue_transition_after_write(
|
||||
self.as_ref(),
|
||||
self.clone()
|
||||
.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
|
||||
.await,
|
||||
LcEventSrc::S3CompleteMultipartUpload,
|
||||
opts,
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
|
||||
@@ -355,7 +355,7 @@ impl ECStore {
|
||||
if let Some(guard) = guard.as_ref() {
|
||||
opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
}
|
||||
let current = crate::bucket::metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
let current = crate::bucket::metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
|
||||
if opts.expected_bucket_incarnation_id != Some(current) {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -1047,6 +1047,7 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
) -> Result<ObjectInfo> {
|
||||
let request_opts = opts;
|
||||
let (target_pool_idx, mutation_fence) = target;
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
if !opts.data_movement {
|
||||
@@ -1132,7 +1133,8 @@ impl ECStore {
|
||||
)
|
||||
.await;
|
||||
drop(publication_guard);
|
||||
let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
|
||||
let result =
|
||||
enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, request_opts).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
|
||||
}
|
||||
|
||||
@@ -29,8 +29,8 @@ use crate::bucket::lifecycle::{
|
||||
},
|
||||
};
|
||||
use crate::bucket::metadata_sys::{
|
||||
acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_in, get_cached_bucket_incarnation_id_in,
|
||||
get_object_lock_config_and_incarnation_from_disk_in,
|
||||
acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_for_options_in,
|
||||
get_cached_bucket_incarnation_id_in, get_object_lock_config_and_incarnation_from_disk_in,
|
||||
};
|
||||
use crate::bucket::object_lock::objectlock_sys::{
|
||||
check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
|
||||
@@ -2837,7 +2837,7 @@ impl ECStore {
|
||||
config_revision,
|
||||
state,
|
||||
lifecycle_fence.clone(),
|
||||
metadata_guard,
|
||||
Arc::new(metadata_guard),
|
||||
)))
|
||||
}
|
||||
|
||||
@@ -4296,6 +4296,7 @@ impl ECStore {
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("data movement PUT requires data_movement options"));
|
||||
}
|
||||
let request_opts = opts;
|
||||
let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
|
||||
let idx = self
|
||||
@@ -4338,7 +4339,7 @@ impl ECStore {
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await;
|
||||
let result = enqueue_transition_after_write(self, result, LcEventSrc::S3PutObject, request_opts).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_list_objects_mutation(self, bucket).await;
|
||||
}
|
||||
@@ -4445,7 +4446,7 @@ impl ECStore {
|
||||
};
|
||||
let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
|
||||
dst_opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, dst_bucket).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, dst_bucket, &dst_opts).await?;
|
||||
if dst_opts
|
||||
.expected_bucket_incarnation_id
|
||||
.is_some_and(|expected| expected != current_incarnation_id)
|
||||
@@ -4564,6 +4565,7 @@ impl ECStore {
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
if !self.single_pool() {
|
||||
@@ -4606,6 +4608,7 @@ impl ECStore {
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
if !self.single_pool() {
|
||||
@@ -4658,6 +4661,7 @@ impl ECStore {
|
||||
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
|
||||
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
quota_admission: dst_opts.quota_admission,
|
||||
..Default::default()
|
||||
};
|
||||
if !self.single_pool() {
|
||||
@@ -4823,22 +4827,26 @@ impl ECStore {
|
||||
get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
}
|
||||
let _object_lock_metadata_guard = if !is_meta_bucketname(bucket) {
|
||||
Some(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?)
|
||||
Some(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(guard) = _object_lock_metadata_guard.as_ref() {
|
||||
opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
|
||||
let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
|
||||
let (state, incarnation_id, config_revision) =
|
||||
get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
|
||||
self.id,
|
||||
bucket,
|
||||
incarnation_id,
|
||||
config_revision,
|
||||
state,
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.clone()
|
||||
.unwrap_or_else(NamespaceLockFence::new),
|
||||
Arc::clone(metadata_guard),
|
||||
)));
|
||||
Some(incarnation_id)
|
||||
} else {
|
||||
@@ -5257,26 +5265,32 @@ impl ECStore {
|
||||
let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
|
||||
None
|
||||
} else {
|
||||
Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
|
||||
})
|
||||
Some(Arc::new(
|
||||
match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
|
||||
},
|
||||
))
|
||||
};
|
||||
if let Some(guard) = _object_lock_metadata_guard.as_ref() {
|
||||
opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
|
||||
let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
|
||||
let (state, incarnation_id, config_revision) =
|
||||
match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
|
||||
};
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
|
||||
self.id,
|
||||
bucket,
|
||||
incarnation_id,
|
||||
config_revision,
|
||||
state,
|
||||
opts.bucket_lifecycle_lock_fence
|
||||
.clone()
|
||||
.unwrap_or_else(NamespaceLockFence::new),
|
||||
Arc::clone(metadata_guard),
|
||||
)));
|
||||
Some(incarnation_id)
|
||||
} else {
|
||||
@@ -5617,7 +5631,7 @@ impl ECStore {
|
||||
opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
}
|
||||
if !is_meta_bucketname(bucket) {
|
||||
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
|
||||
if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) {
|
||||
return Err(StorageError::BucketNotFound(bucket.to_string()));
|
||||
}
|
||||
@@ -5690,7 +5704,7 @@ impl ECStore {
|
||||
None
|
||||
} else {
|
||||
let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
|
||||
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
|
||||
if opts
|
||||
.expected_bucket_incarnation_id
|
||||
.is_some_and(|expected| expected != current_incarnation_id)
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![cfg(feature = "test-util")]
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use storage_api::metadata_lock::{
|
||||
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
|
||||
isolated_store_over_temp_disks,
|
||||
};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[tokio::test]
|
||||
async fn replica_put_completes_while_metadata_writer_waits_for_its_snapshot() {
|
||||
replica_write_with_waiting_metadata_writer(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replica_multipart_completes_while_metadata_writer_waits_for_its_snapshot() {
|
||||
replica_write_with_waiting_metadata_writer(true).await;
|
||||
}
|
||||
|
||||
async fn replica_write_with_waiting_metadata_writer(multipart: bool) {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "replica-metadata-read-reuse";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(
|
||||
bucket,
|
||||
"lifecycle.xml",
|
||||
br#"<LifecycleConfiguration><Rule><ID>expire-old-versions</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("configure post-write lifecycle evaluation");
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
replication_request: true,
|
||||
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
|
||||
object_lock_config_snapshot: Some(
|
||||
store
|
||||
.object_lock_config_snapshot(bucket)
|
||||
.await
|
||||
.expect("capture Object Lock snapshot"),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
let upload = if multipart {
|
||||
let upload = store
|
||||
.new_multipart_upload(bucket, "object", &opts)
|
||||
.await
|
||||
.expect("stage upload");
|
||||
let mut data = PutObjReader::from_vec(b"replicated version".to_vec());
|
||||
let part = store
|
||||
.put_object_part(bucket, "object", &upload.upload_id, 1, &mut data, &opts)
|
||||
.await
|
||||
.expect("stage part");
|
||||
Some((
|
||||
upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("create metadata writer");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
// Leave the writer registered while the replica continues. A second read
|
||||
// acquisition would queue behind this writer, which needs our first read.
|
||||
assert!(futures::poll!(&mut writer).is_pending());
|
||||
timeout(Duration::from_secs(2), async {
|
||||
if let Some((upload_id, parts)) = upload {
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "object", &upload_id, parts, &opts)
|
||||
.await
|
||||
} else {
|
||||
let mut data = PutObjReader::from_vec(b"replicated version".to_vec());
|
||||
store.put_object(bucket, "object", &mut data, &opts).await
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("replica must reuse its metadata read lock instead of waiting behind the writer")
|
||||
.expect("replica PUT should commit");
|
||||
assert!(futures::poll!(&mut writer).is_pending(), "the snapshot must still fence metadata updates");
|
||||
let read_opts = ObjectOptions {
|
||||
version_id: opts.version_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
drop(opts);
|
||||
timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("metadata writer must proceed after the snapshot drops")
|
||||
.expect("acquire metadata writer");
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, "object", None, Default::default(), &read_opts)
|
||||
.await
|
||||
.expect("read committed replica version");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read replica body");
|
||||
assert_eq!(bytes, b"replicated version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reused_metadata_lock_still_enforces_quota() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-quota";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("set quota");
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
|
||||
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(opts.set_quota_admission(0, 1));
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("create writer");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
assert!(futures::poll!(&mut writer).is_pending());
|
||||
let mut data = PutObjReader::from_vec(b"too large".to_vec());
|
||||
let error = timeout(Duration::from_secs(2), store.put_object(bucket, "object", &mut data, &opts))
|
||||
.await
|
||||
.expect("quota admission must not reacquire the lock")
|
||||
.expect_err("quota must reject growth");
|
||||
assert!(matches!(error, Error::QuotaExceeded { limit: 1, .. }), "{error:?}");
|
||||
drop(opts);
|
||||
timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("release metadata reader after denial")
|
||||
.expect("acquire writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_put_releases_shared_metadata_guard() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-cancel";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
|
||||
..Default::default()
|
||||
};
|
||||
let barrier = PutObjectCommitBarrier::install(bucket, "object", PutObjectCommitPause::AfterQuotaReservation);
|
||||
let put_store = Arc::clone(&store);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut data = PutObjReader::from_vec(b"cancel before commit".to_vec());
|
||||
put_store.put_object(bucket, "object", &mut data, &opts).await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
let lock = store
|
||||
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
|
||||
.await
|
||||
.expect("create writer");
|
||||
let writer = lock.get_write_lock(Duration::from_secs(5));
|
||||
tokio::pin!(writer);
|
||||
assert!(futures::poll!(&mut writer).is_pending());
|
||||
put.abort();
|
||||
assert!(put.await.expect_err("PUT should be cancelled").is_cancelled());
|
||||
timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("all metadata reader owners must drop after cancellation")
|
||||
.expect("acquire writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_rejects_metadata_snapshots_from_another_scope() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let (_other_dirs, other_store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&other_store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-scope";
|
||||
for target in [&store, &other_store] {
|
||||
target
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
}
|
||||
store
|
||||
.make_bucket("other-bucket", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create other bucket");
|
||||
let other_store_incarnation = other_store
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.expect("load other store incarnation");
|
||||
let other_bucket_incarnation = store
|
||||
.bucket_incarnation_id("other-bucket")
|
||||
.await
|
||||
.expect("load other bucket incarnation");
|
||||
for (snapshot_store, snapshot_bucket, expected) in [
|
||||
(&other_store, bucket, other_store_incarnation),
|
||||
(&store, "other-bucket", other_bucket_incarnation),
|
||||
(&store, bucket, Uuid::new_v4()),
|
||||
] {
|
||||
let opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(expected),
|
||||
object_lock_config_snapshot: Some(
|
||||
snapshot_store
|
||||
.object_lock_config_snapshot(snapshot_bucket)
|
||||
.await
|
||||
.expect("capture snapshot"),
|
||||
),
|
||||
..Default::default()
|
||||
};
|
||||
let error = store
|
||||
.new_multipart_upload(bucket, "object", &opts)
|
||||
.await
|
||||
.expect_err("foreign snapshot must be rejected");
|
||||
assert!(error.to_string().contains("valid metadata transaction fence"), "{error:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_reuses_its_metadata_snapshot_for_generation_validation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "metadata-read-reuse-delete";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let mut data = PutObjReader::from_vec(b"remove prefix".to_vec());
|
||||
store
|
||||
.put_object(bucket, "prefix/object", &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("seed object");
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"prefix/",
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete prefix with an internally captured metadata snapshot");
|
||||
let error = store
|
||||
.get_object_info(bucket, "prefix/object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("prefix object must be deleted");
|
||||
assert!(matches!(error, Error::ObjectNotFound(..)), "{error:?}");
|
||||
}
|
||||
@@ -6,6 +6,17 @@ pub(crate) use rustfs_ecstore::api::erasure::Erasure;
|
||||
pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
pub(crate) use rustfs_ecstore::api::{error::Error, set_disk::SetDisks, storage::ECStore};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) mod metadata_lock {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
BucketOperations, CompletePart, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
};
|
||||
pub(crate) use super::{Error, ObjectOptions, PutObjReader};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
}
|
||||
|
||||
pub(crate) mod contract_compat {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
|
||||
@@ -72,6 +72,41 @@ const LEGACY_ROOT_HEAL_PATH: &str = ".";
|
||||
const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
|
||||
const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
const RESUME_GC_INTERVAL: Duration = Duration::from_secs(60 * 60);
|
||||
const BEST_EFFORT_QUEUE_RESERVE_PERCENT: usize = 10;
|
||||
const BEST_EFFORT_SOURCE_COUNT: usize = 3;
|
||||
|
||||
fn is_best_effort_source(source: HealRequestSource) -> bool {
|
||||
matches!(
|
||||
source,
|
||||
HealRequestSource::Scanner | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
|
||||
)
|
||||
}
|
||||
|
||||
fn best_effort_source_quota_index(source: HealRequestSource) -> Option<usize> {
|
||||
match source {
|
||||
HealRequestSource::Scanner => Some(0),
|
||||
HealRequestSource::ReadRepair => Some(1),
|
||||
HealRequestSource::AutoHeal => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn best_effort_queue_reserve(queue_capacity: usize) -> usize {
|
||||
match queue_capacity {
|
||||
0..=2 => 0,
|
||||
capacity => (capacity.saturating_mul(BEST_EFFORT_QUEUE_RESERVE_PERCENT) / 100).max(BEST_EFFORT_SOURCE_COUNT),
|
||||
}
|
||||
}
|
||||
|
||||
fn best_effort_source_reserves(queue_capacity: usize) -> [usize; BEST_EFFORT_SOURCE_COUNT] {
|
||||
let reserve = best_effort_queue_reserve(queue_capacity);
|
||||
if reserve == 0 {
|
||||
return [0; BEST_EFFORT_SOURCE_COUNT];
|
||||
}
|
||||
let base = reserve / BEST_EFFORT_SOURCE_COUNT;
|
||||
let remainder = reserve % BEST_EFFORT_SOURCE_COUNT;
|
||||
[base + usize::from(remainder > 0), base + usize::from(remainder > 1), base]
|
||||
}
|
||||
|
||||
// Admission/scheduler outcomes for per-object requests (Object/Metadata/
|
||||
// ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner
|
||||
@@ -965,11 +1000,7 @@ struct HealQueueContext<'a> {
|
||||
|
||||
impl HealManager {
|
||||
fn classify_full_admission(request: &HealRequest, config: &HealConfig) -> HealAdmissionResult {
|
||||
let best_effort_source = matches!(
|
||||
request.source,
|
||||
HealRequestSource::Scanner | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
|
||||
);
|
||||
if best_effort_source || (request.priority == HealPriority::Low && config.low_priority_drop_when_full) {
|
||||
if request.priority == HealPriority::Low && config.low_priority_drop_when_full {
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)
|
||||
} else {
|
||||
HealAdmissionResult::Full
|
||||
@@ -980,32 +1011,6 @@ impl HealManager {
|
||||
queue_len.saturating_mul(100).checked_div(queue_capacity).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn classify_pressure_admission(
|
||||
request: &HealRequest,
|
||||
queue_len: usize,
|
||||
queue_capacity: usize,
|
||||
) -> Option<HealAdmissionResult> {
|
||||
if request.force_start || queue_capacity == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let queue_usage_pct = Self::queue_usage_pct(queue_len, queue_capacity);
|
||||
if queue_usage_pct < 80 {
|
||||
return None;
|
||||
}
|
||||
|
||||
match request.source {
|
||||
HealRequestSource::ReadRepair => Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)),
|
||||
HealRequestSource::Scanner if request.priority == HealPriority::Low => {
|
||||
Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped))
|
||||
}
|
||||
HealRequestSource::AutoHeal if queue_usage_pct >= 95 => {
|
||||
Some(HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn duplicate_admission_for_request(request: &HealRequest, config: &HealConfig) -> HealAdmissionResult {
|
||||
if request.priority == HealPriority::Low && !config.low_priority_merge_enable {
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped)
|
||||
@@ -1156,6 +1161,8 @@ impl HealManager {
|
||||
publish_heal_queue_length(queue);
|
||||
let queue_capacity = config.queue_size;
|
||||
let per_object_request = request.heal_type.is_per_object();
|
||||
let best_effort_reserve = best_effort_queue_reserve(queue_capacity);
|
||||
let best_effort_source_reserves = best_effort_source_reserves(queue_capacity);
|
||||
|
||||
if queue_len >= queue_capacity && !request.force_start {
|
||||
if Self::can_displace_queued_work(&request)
|
||||
@@ -1243,27 +1250,55 @@ impl HealManager {
|
||||
return HealAdmissionDecision::new(admission);
|
||||
}
|
||||
|
||||
if let Some(admission) = Self::classify_pressure_admission(&request, queue_len, queue_capacity) {
|
||||
Self::record_admission_metric(request.source, admission, context);
|
||||
if let HealAdmissionResult::Dropped(reason) = admission {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
source = request.source.as_str(),
|
||||
context,
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
queue_usage_pct = Self::queue_usage_pct(queue_len, queue_capacity),
|
||||
reason = reason.as_str(),
|
||||
result = "dropped_pressure",
|
||||
"Heal queue request dropped under pressure"
|
||||
);
|
||||
}
|
||||
return HealAdmissionDecision::new(admission);
|
||||
if !is_best_effort_source(request.source)
|
||||
&& request.priority <= HealPriority::Normal
|
||||
&& !request.force_start
|
||||
&& best_effort_reserve > 0
|
||||
&& queue_len >= queue_capacity.saturating_sub(best_effort_reserve)
|
||||
{
|
||||
Self::record_admission_metric(request.source, HealAdmissionResult::Full, context);
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
source = request.source.as_str(),
|
||||
context,
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
reserved_for_best_effort = best_effort_reserve,
|
||||
result = "rejected_for_reserved_capacity",
|
||||
"Normal heal request deferred to preserve best-effort capacity"
|
||||
});
|
||||
return HealAdmissionDecision::new(HealAdmissionResult::Full);
|
||||
}
|
||||
|
||||
if is_best_effort_source(request.source)
|
||||
&& request.priority <= HealPriority::Normal
|
||||
&& !request.force_start
|
||||
&& best_effort_reserve > 0
|
||||
&& queue_len >= queue_capacity.saturating_sub(best_effort_reserve)
|
||||
&& let Some(source_index) = best_effort_source_quota_index(request.source)
|
||||
&& queue.best_effort_source_count(request.source) >= best_effort_source_reserves[source_index]
|
||||
{
|
||||
Self::record_admission_metric(request.source, HealAdmissionResult::Full, context);
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
priority = ?request.priority,
|
||||
source = request.source.as_str(),
|
||||
context,
|
||||
queue_len,
|
||||
queue_capacity,
|
||||
reserved_for_best_effort = best_effort_reserve,
|
||||
source_quota = best_effort_source_reserves[source_index],
|
||||
result = "rejected_for_source_quota",
|
||||
"Best-effort source deferred to preserve per-source capacity"
|
||||
});
|
||||
return HealAdmissionDecision::new(HealAdmissionResult::Full);
|
||||
}
|
||||
|
||||
if queue_capacity > 0 {
|
||||
|
||||
@@ -34,6 +34,33 @@ pub(super) struct PriorityHealQueue {
|
||||
pub(super) sequence: u64,
|
||||
/// Deduplication index for queued requests
|
||||
pub(super) dedup_keys: HashMap<String, DedupKeyEntry>,
|
||||
pub(super) best_effort_queued: usize,
|
||||
pub(super) best_effort_queued_by_source: [usize; 3],
|
||||
/// Number of MRF/Normal tasks dispatched since a best-effort task ran.
|
||||
/// This is kept with the queue so event-driven scheduler wakeups cannot
|
||||
/// reset the fairness budget between cycles.
|
||||
pub(super) mrf_normal_dispatches_since_best_effort: usize,
|
||||
pub(super) last_best_effort_source: Option<HealRequestSource>,
|
||||
}
|
||||
|
||||
const MRF_NORMAL_DISPATCH_QUANTUM: usize = 4;
|
||||
|
||||
fn best_effort_source_index(source: HealRequestSource) -> Option<usize> {
|
||||
match source {
|
||||
HealRequestSource::Scanner => Some(0),
|
||||
HealRequestSource::ReadRepair => Some(1),
|
||||
HealRequestSource::AutoHeal => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn best_effort_source_order(last: Option<HealRequestSource>) -> [usize; 3] {
|
||||
match last {
|
||||
Some(HealRequestSource::Scanner) => [1, 2, 0],
|
||||
Some(HealRequestSource::ReadRepair) => [2, 0, 1],
|
||||
Some(HealRequestSource::AutoHeal) => [0, 1, 2],
|
||||
_ => [0, 1, 2],
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper for heap items to implement proper ordering
|
||||
@@ -264,6 +291,10 @@ impl PriorityHealQueue {
|
||||
heap: BinaryHeap::new(),
|
||||
sequence: 0,
|
||||
dedup_keys: HashMap::new(),
|
||||
best_effort_queued: 0,
|
||||
best_effort_queued_by_source: [0; 3],
|
||||
mrf_normal_dispatches_since_best_effort: 0,
|
||||
last_best_effort_source: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -271,9 +302,11 @@ impl PriorityHealQueue {
|
||||
self.heap.len()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn pop_next(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
@@ -301,12 +334,19 @@ impl PriorityHealQueue {
|
||||
})
|
||||
.refcount += 1;
|
||||
self.sequence += 1;
|
||||
let source = request.source;
|
||||
self.heap.push(PriorityQueueItem {
|
||||
priority: request.priority,
|
||||
sequence: self.sequence,
|
||||
dedup_key: key,
|
||||
request,
|
||||
});
|
||||
if is_best_effort_source(source) {
|
||||
self.best_effort_queued += 1;
|
||||
if let Some(index) = best_effort_source_index(source) {
|
||||
self.best_effort_queued_by_source[index] += 1;
|
||||
}
|
||||
}
|
||||
QueuePushOutcome::Accepted
|
||||
}
|
||||
|
||||
@@ -356,6 +396,7 @@ impl PriorityHealQueue {
|
||||
|
||||
let displaced = displaced.map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
self.refresh_dedup_representative(&item.dedup_key);
|
||||
item.request
|
||||
});
|
||||
@@ -400,6 +441,7 @@ impl PriorityHealQueue {
|
||||
pub(super) fn pop(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
@@ -429,12 +471,115 @@ impl PriorityHealQueue {
|
||||
(
|
||||
selected.map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
item.request
|
||||
}),
|
||||
skipped,
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn pop_runnable_with_fairness<F, G>(&mut self, can_run: F, skip_label: G) -> (Option<HealRequest>, Vec<String>)
|
||||
where
|
||||
F: Fn(&HealRequest) -> bool,
|
||||
G: Fn(&HealRequest) -> Option<String>,
|
||||
{
|
||||
if self.mrf_normal_dispatches_since_best_effort < MRF_NORMAL_DISPATCH_QUANTUM {
|
||||
let (chosen, skipped) = self.pop_runnable_with_skips(can_run, skip_label);
|
||||
if let Some(request) = chosen.as_ref() {
|
||||
self.record_dispatch(request);
|
||||
}
|
||||
return (chosen, skipped);
|
||||
}
|
||||
|
||||
if self.best_effort_queued == 0 {
|
||||
self.mrf_normal_dispatches_since_best_effort = 0;
|
||||
let (chosen, skipped) = self.pop_runnable_with_skips(can_run, skip_label);
|
||||
if let Some(request) = chosen.as_ref() {
|
||||
self.record_dispatch(request);
|
||||
}
|
||||
return (chosen, skipped);
|
||||
}
|
||||
|
||||
let mut items = std::mem::take(&mut self.heap).into_vec();
|
||||
let mut skipped = Vec::new();
|
||||
let mut first_runnable: Option<usize> = None;
|
||||
let mut best_effort_candidates: [Option<usize>; 3] = [None; 3];
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
if !can_run(&item.request) {
|
||||
if let Some(label) = skip_label(&item.request) {
|
||||
skipped.push(label);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if first_runnable.is_none_or(|current| item.cmp(&items[current]).is_gt()) {
|
||||
first_runnable = Some(index);
|
||||
}
|
||||
let Some(source_index) = best_effort_source_index(item.request.source) else {
|
||||
continue;
|
||||
};
|
||||
if best_effort_candidates[source_index].is_none_or(|current| {
|
||||
item.priority > items[current].priority
|
||||
|| (item.priority == items[current].priority && item.sequence < items[current].sequence)
|
||||
}) {
|
||||
best_effort_candidates[source_index] = Some(index);
|
||||
}
|
||||
}
|
||||
|
||||
let chosen_index = first_runnable.filter(|index| {
|
||||
let item = &items[*index];
|
||||
item.priority == HealPriority::Normal
|
||||
&& matches!(item.request.source, HealRequestSource::Mrf | HealRequestSource::Internal)
|
||||
});
|
||||
let best_effort_priority = best_effort_candidates
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|index| items[index].priority)
|
||||
.max();
|
||||
let chosen_index = chosen_index
|
||||
.and_then(|_| {
|
||||
let priority = best_effort_priority?;
|
||||
best_effort_source_order(self.last_best_effort_source)
|
||||
.into_iter()
|
||||
.find_map(|source| best_effort_candidates[source].filter(|index| items[*index].priority == priority))
|
||||
})
|
||||
.or(first_runnable);
|
||||
let chosen = chosen_index.map(|index| items.swap_remove(index));
|
||||
self.heap = BinaryHeap::from(items);
|
||||
|
||||
let chosen = chosen.map(|item| {
|
||||
let request = item.request;
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(request.source);
|
||||
self.record_dispatch(&request);
|
||||
request
|
||||
});
|
||||
(chosen, skipped)
|
||||
}
|
||||
|
||||
fn record_dispatch(&mut self, request: &HealRequest) {
|
||||
if is_best_effort_source(request.source) {
|
||||
self.mrf_normal_dispatches_since_best_effort = 0;
|
||||
self.last_best_effort_source = Some(request.source);
|
||||
} else if matches!(request.source, HealRequestSource::Mrf | HealRequestSource::Internal)
|
||||
&& request.priority == HealPriority::Normal
|
||||
{
|
||||
self.mrf_normal_dispatches_since_best_effort = self.mrf_normal_dispatches_since_best_effort.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_best_effort_count(&mut self, source: HealRequestSource) {
|
||||
if is_best_effort_source(source) {
|
||||
self.best_effort_queued = self.best_effort_queued.saturating_sub(1);
|
||||
if let Some(index) = best_effort_source_index(source) {
|
||||
self.best_effort_queued_by_source[index] = self.best_effort_queued_by_source[index].saturating_sub(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn best_effort_source_count(&self, source: HealRequestSource) -> usize {
|
||||
best_effort_source_index(source).map_or(0, |index| self.best_effort_queued_by_source[index])
|
||||
}
|
||||
|
||||
fn restore_deferred_items(&mut self, deferred: Vec<PriorityQueueItem>) {
|
||||
if deferred.is_empty() {
|
||||
return;
|
||||
@@ -590,6 +735,7 @@ impl PriorityHealQueue {
|
||||
if removed.is_none() && item.request.id == request_id {
|
||||
let key = item.dedup_key.clone();
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
affected_key = Some(key);
|
||||
removed = Some(item.request);
|
||||
} else {
|
||||
@@ -615,6 +761,7 @@ impl PriorityHealQueue {
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if should_remove(&item.request) {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.remove_best_effort_count(item.request.source);
|
||||
affected_keys.push(item.dedup_key);
|
||||
removed.push(item.request);
|
||||
} else {
|
||||
|
||||
@@ -150,7 +150,7 @@ impl HealManager {
|
||||
for _ in 0..available_slots {
|
||||
let selected_request = if config.set_bulkhead_enable || mainline_pressure.is_some() {
|
||||
let max_concurrent_per_set = config.max_concurrent_per_set;
|
||||
let (selected_request, skipped_sets) = queue.pop_runnable_with_skips(
|
||||
let (selected_request, skipped_sets) = queue.pop_runnable_with_fairness(
|
||||
|request| {
|
||||
let set_allowed = !config.set_bulkhead_enable
|
||||
|| can_schedule_request(request, &running_per_set, max_concurrent_per_set);
|
||||
@@ -164,7 +164,7 @@ impl HealManager {
|
||||
}
|
||||
selected_request
|
||||
} else {
|
||||
queue.pop_next()
|
||||
queue.pop_runnable_with_fairness(|_| true, |_| None).0
|
||||
};
|
||||
|
||||
if let Some(mut request) = selected_request {
|
||||
|
||||
@@ -1488,6 +1488,164 @@ fn test_priority_queue_ordering() {
|
||||
assert_eq!(queue.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_gives_best_effort_bounded_service() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..5 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, skipped) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert!(skipped.is_empty());
|
||||
assert_eq!(
|
||||
request.expect("MRF request should run before the fairness quantum").source,
|
||||
HealRequestSource::Mrf
|
||||
);
|
||||
}
|
||||
|
||||
let (request, skipped) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert!(skipped.is_empty());
|
||||
assert_eq!(
|
||||
request
|
||||
.expect("best-effort work must get a bounded service opportunity")
|
||||
.source,
|
||||
HealRequestSource::Scanner
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_covers_internal_normal_background_work() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..5 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(
|
||||
&format!("internal-{index}"),
|
||||
HealPriority::Normal,
|
||||
HealRequestSource::Internal
|
||||
)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("internal background work should run").source, HealRequestSource::Internal);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("Scanner should get a service opportunity").source,
|
||||
HealRequestSource::Scanner
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_round_robins_best_effort_sources() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..8 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
for (bucket, source) in [
|
||||
("scanner", HealRequestSource::Scanner),
|
||||
("read-repair", HealRequestSource::ReadRepair),
|
||||
("auto-heal", HealRequestSource::AutoHeal),
|
||||
] {
|
||||
assert_eq!(queue.push(bucket_request(bucket, HealPriority::Low, source)), QueuePushOutcome::Accepted);
|
||||
}
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("first MRF quantum should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("Scanner should get the first best-effort turn").source,
|
||||
HealRequestSource::Scanner
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("second MRF quantum should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("ReadRepair should get the next best-effort turn").source,
|
||||
HealRequestSource::ReadRepair
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_keeps_best_effort_priority_order() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..4 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("MRF request should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("higher-priority best-effort work should run first").source,
|
||||
HealRequestSource::ReadRepair
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fairness_skips_blocked_best_effort_source() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
for index in 0..4 {
|
||||
assert_eq!(
|
||||
queue.push(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
queue.push(bucket_request("read-repair", HealPriority::Low, HealRequestSource::ReadRepair)),
|
||||
QueuePushOutcome::Accepted
|
||||
);
|
||||
|
||||
for _ in 0..4 {
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|_| true, |_| None);
|
||||
assert_eq!(request.expect("MRF request should run").source, HealRequestSource::Mrf);
|
||||
}
|
||||
let (request, _) = queue.pop_runnable_with_fairness(|request| request.source != HealRequestSource::Scanner, |_| None);
|
||||
assert_eq!(
|
||||
request.expect("runnable ReadRepair should bypass blocked Scanner").source,
|
||||
HealRequestSource::ReadRepair
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_priority_queue_fifo_same_priority() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
@@ -4500,7 +4658,7 @@ async fn test_displacing_registered_mrf_task_drops_notice_ownership() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_drops_read_repair_under_pressure() {
|
||||
async fn test_submit_heal_request_admits_read_repair_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
storage,
|
||||
@@ -4510,7 +4668,7 @@ async fn test_submit_heal_request_drops_read_repair_under_pressure() {
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..8 {
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(
|
||||
@@ -4529,12 +4687,12 @@ async fn test_submit_heal_request_drops_read_repair_under_pressure() {
|
||||
.await
|
||||
.expect("read repair admission should return a result");
|
||||
|
||||
assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped));
|
||||
assert_eq!(admission, HealAdmissionResult::Accepted);
|
||||
assert_eq!(manager.get_queue_length().await, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_drops_low_scanner_under_pressure() {
|
||||
async fn test_submit_heal_request_admits_low_scanner_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
storage,
|
||||
@@ -4544,7 +4702,7 @@ async fn test_submit_heal_request_drops_low_scanner_under_pressure() {
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..8 {
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(
|
||||
@@ -4563,10 +4721,220 @@ async fn test_submit_heal_request_drops_low_scanner_under_pressure() {
|
||||
.await
|
||||
.expect("scanner admission should return a result");
|
||||
|
||||
assert_eq!(admission, HealAdmissionResult::Dropped(HealAdmissionDropReason::PolicyDropped));
|
||||
assert_eq!(admission, HealAdmissionResult::Accepted);
|
||||
assert_eq!(manager.get_queue_length().await, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mrf_admission_preserves_best_effort_queue_reserve() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf,))
|
||||
.await
|
||||
.expect("MRF request should be admitted before the reserve threshold"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("mrf-deferred", HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF admission should return a typed result"),
|
||||
HealAdmissionResult::Full
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("internal-deferred", HealPriority::Normal, HealRequestSource::Internal))
|
||||
.await
|
||||
.expect("ordinary background admission should return a typed result"),
|
||||
HealAdmissionResult::Full
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-reserved", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner should consume the reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair-reserved", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal-reserved", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(manager.get_queue_length().await, 10);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_small_queue_keeps_per_source_best_effort_slots() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 9,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..6 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF request should be admitted before the per-source reserve"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("mrf-deferred", HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF admission should return a typed result"),
|
||||
HealAdmissionResult::Full
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-reserved", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair-reserved", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal-reserved", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal should consume its reserved slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_queue_size_four_keeps_mrf_and_each_best_effort_source() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 4,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("mrf", HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF should retain one queue slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner should use its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair should use its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal should use its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_high_priority_mrf_bypasses_best_effort_reserve() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("normal MRF request should be admitted before the reserve"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("urgent-mrf", HealPriority::Urgent, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("urgent MRF admission should return a typed result"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_high_priority_best_effort_bypasses_source_quota() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("normal MRF request should be admitted before the reserve"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-low", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("low Scanner request should be admitted into its source slot"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner-high", HealPriority::High, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("high Scanner retry should bypass its low-priority quota"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
@@ -4578,7 +4946,7 @@ async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..8 {
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(
|
||||
@@ -4598,7 +4966,56 @@ async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
|
||||
.expect("admin admission should return a result");
|
||||
|
||||
assert_eq!(admission, HealAdmissionResult::Accepted);
|
||||
assert_eq!(manager.get_queue_length().await, 9);
|
||||
assert_eq!(manager.get_queue_length().await, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_admin_normal_displaces_best_effort_when_queue_is_full() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
|
||||
for index in 0..7 {
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request(&format!("mrf-{index}"), HealPriority::Normal, HealRequestSource::Mrf))
|
||||
.await
|
||||
.expect("MRF request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("scanner", HealPriority::Low, HealRequestSource::Scanner))
|
||||
.await
|
||||
.expect("Scanner request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("read-repair", HealPriority::Low, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("ReadRepair request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("auto-heal", HealPriority::Low, HealRequestSource::AutoHeal))
|
||||
.await
|
||||
.expect("AutoHeal request should be queued"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(bucket_request("admin", HealPriority::Normal, HealRequestSource::Admin))
|
||||
.await
|
||||
.expect("admin admission should return a typed result"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -105,13 +105,17 @@ pub struct HealObjectReceipt {
|
||||
|
||||
impl HealObjectReceipt {
|
||||
pub(crate) fn verified_for(&self, expected: &HealObjectIdentity) -> bool {
|
||||
matches!(
|
||||
self.disposition,
|
||||
let disposition_verifies = match self.disposition {
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::MetadataHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
) && self.identity.kind == expected.kind
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent => true,
|
||||
// A metadata/presence proof never certifies payload bytes, so it
|
||||
// cannot discharge a request that exists to decode the payload.
|
||||
HealObjectDisposition::MetadataHealthy => expected.kind != HealObjectKind::Decode,
|
||||
_ => false,
|
||||
};
|
||||
disposition_verifies
|
||||
&& self.identity.kind == expected.kind
|
||||
&& self.identity.bucket == expected.bucket
|
||||
&& self.identity.object == expected.object
|
||||
&& self.identity.version_id == expected.version_id
|
||||
@@ -530,6 +534,32 @@ mod canonical_outcome_tests {
|
||||
assert!(serde_json::from_value::<HealTaskOutcome>(value).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_health_receipt_never_discharges_a_payload_decode_request() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let mut expected = HealObjectIdentity {
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
..item(HealObjectDisposition::Unknown).identity
|
||||
};
|
||||
let mut receipt = HealObjectReceipt {
|
||||
identity: expected.clone(),
|
||||
disposition: HealObjectDisposition::MetadataHealthy,
|
||||
};
|
||||
assert!(
|
||||
receipt.verified_for(&expected),
|
||||
"a presence proof still settles a metadata-level object request"
|
||||
);
|
||||
|
||||
expected.kind = HealObjectKind::Decode;
|
||||
receipt.identity.kind = HealObjectKind::Decode;
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
"a presence-only proof must not clear a payload decode responsibility"
|
||||
);
|
||||
receipt.disposition = HealObjectDisposition::VerifiedHealthy;
|
||||
assert!(receipt.verified_for(&expected));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_outcome_categories_have_one_terminal_count() {
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
|
||||
@@ -69,6 +69,15 @@ pub fn record_remote_lock_rpc_timeout(peer: &str, op: &'static str) {
|
||||
counter!("rustfs_remote_lock_rpc_timeouts_total", "peer" => peer.to_string(), "op" => op).increment(1);
|
||||
}
|
||||
|
||||
/// Record a lock acquisition rejected by the per-peer request breaker or
|
||||
/// in-flight admission limit.
|
||||
#[inline(always)]
|
||||
pub fn record_remote_lock_request_suppressed(peer: &str, op: &'static str, reason: &'static str) {
|
||||
use metrics::counter;
|
||||
counter!("rustfs_remote_lock_request_suppressed_total", "peer" => peer.to_string(), "op" => op, "reason" => reason)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Record the cached lock channel to `peer` being evicted after an RPC failure.
|
||||
#[inline(always)]
|
||||
pub fn record_remote_lock_channel_eviction(peer: &str, trigger: &'static str) {
|
||||
@@ -229,6 +238,7 @@ mod tests {
|
||||
record_early_release();
|
||||
record_contention_event();
|
||||
record_remote_lock_rpc_timeout("http://peer:9000", "lock");
|
||||
record_remote_lock_request_suppressed("http://peer:9000", "lock", "breaker_open");
|
||||
record_remote_lock_channel_eviction("http://peer:9000", "timeout");
|
||||
record_remote_lock_channel_eviction_suppressed("http://peer:9000", "peer_recently_served");
|
||||
record_remote_lock_rpc_detached("lock", "detached");
|
||||
@@ -251,6 +261,7 @@ mod tests {
|
||||
"rustfs_lock_early_releases",
|
||||
"rustfs_lock_contentions",
|
||||
"rustfs_remote_lock_rpc_timeouts_total",
|
||||
"rustfs_remote_lock_request_suppressed_total",
|
||||
"rustfs_remote_lock_channel_evictions_total",
|
||||
"rustfs_remote_lock_channel_evictions_suppressed_total",
|
||||
"rustfs_remote_lock_rpc_detached_total",
|
||||
|
||||
+146
-24
@@ -13,12 +13,11 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::time::Instant;
|
||||
use tokio::time::{Instant, timeout};
|
||||
|
||||
use crate::{
|
||||
FastLockGuard, GlobalLockManager, LockClient, LockId, LockInfo, LockManager, LockMetadata, LockPriority, LockRequest,
|
||||
@@ -131,35 +130,52 @@ impl LocalClient {
|
||||
self.manager.clone().unwrap_or_else(crate::get_global_lock_manager)
|
||||
}
|
||||
|
||||
/// Get the shard index for a given lock ID
|
||||
fn get_shard_index(&self, lock_id: &LockId) -> usize {
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
lock_id.hash(&mut hasher);
|
||||
(hasher.finish() as usize) & self.shard_mask
|
||||
/// Guard entries for one resource share a shard so expiration checks do not
|
||||
/// need to take every shard's write lock on the acquisition hot path.
|
||||
fn get_resource_shard_index(&self, resource: &crate::ObjectKey) -> usize {
|
||||
resource.shard_index(self.shard_mask)
|
||||
}
|
||||
|
||||
/// Get the shard for a given lock ID
|
||||
fn get_shard(&self, lock_id: &LockId) -> &Arc<RwLock<HashMap<LockId, LocalGuardEntry>>> {
|
||||
let index = self.get_shard_index(lock_id);
|
||||
let index = self.get_resource_shard_index(&lock_id.resource);
|
||||
&self.guard_storage[index]
|
||||
}
|
||||
|
||||
async fn reclaim_expired_guards_for_resource(&self, resource: &crate::ObjectKey) -> usize {
|
||||
let expired_entries = Self::extract_expired_guards(&self.guard_storage, Some(resource)).await;
|
||||
let shard = &self.guard_storage[self.get_resource_shard_index(resource)];
|
||||
let has_expired = {
|
||||
let guards = shard.read().await;
|
||||
guards
|
||||
.iter()
|
||||
.any(|(lock_id, entry)| &lock_id.resource == resource && entry.is_expired())
|
||||
};
|
||||
if !has_expired {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let expired_entries = {
|
||||
let mut guards = shard.write().await;
|
||||
guards
|
||||
.extract_if(|lock_id, entry| &lock_id.resource == resource && entry.is_expired())
|
||||
.map(|(_, entry)| entry)
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
Self::release_reclaimed_guards(expired_entries, Some(resource))
|
||||
}
|
||||
|
||||
async fn extract_expired_guards(storage: &GuardStorage, resource: Option<&crate::ObjectKey>) -> Vec<LocalGuardEntry> {
|
||||
async fn extract_expired_guards(storage: &GuardStorage) -> Vec<LocalGuardEntry> {
|
||||
let mut expired_entries = Vec::new();
|
||||
for shard in storage.iter() {
|
||||
let has_expired = {
|
||||
let guards = shard.read().await;
|
||||
guards.values().any(LocalGuardEntry::is_expired)
|
||||
};
|
||||
if !has_expired {
|
||||
continue;
|
||||
}
|
||||
let mut guards = shard.write().await;
|
||||
expired_entries.extend(
|
||||
guards
|
||||
.extract_if(|lock_id, entry| {
|
||||
resource.is_none_or(|resource| &lock_id.resource == resource) && entry.is_expired()
|
||||
})
|
||||
.map(|(_, entry)| entry),
|
||||
);
|
||||
expired_entries.extend(guards.extract_if(|_, entry| entry.is_expired()).map(|(_, entry)| entry));
|
||||
}
|
||||
expired_entries
|
||||
}
|
||||
@@ -197,7 +213,7 @@ impl LocalClient {
|
||||
let Some(storage) = storage.upgrade() else {
|
||||
break;
|
||||
};
|
||||
let expired_entries = Self::extract_expired_guards(&storage, None).await;
|
||||
let expired_entries = Self::extract_expired_guards(&storage).await;
|
||||
Self::release_reclaimed_guards(expired_entries, None);
|
||||
}
|
||||
});
|
||||
@@ -214,11 +230,24 @@ impl Default for LocalClient {
|
||||
impl LockClient for LocalClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
self.ensure_reaper();
|
||||
let lock_manager = self.get_lock_manager();
|
||||
let reclaimed_before_acquire = self.reclaim_expired_guards_for_resource(&request.resource).await;
|
||||
if request.lock_id.resource != request.resource {
|
||||
return Ok(LockResponse::failure(
|
||||
"Lock id resource does not match the requested resource",
|
||||
Duration::ZERO,
|
||||
));
|
||||
}
|
||||
let acquire_deadline = Instant::now()
|
||||
.checked_add(request.acquire_timeout)
|
||||
.unwrap_or_else(Instant::now);
|
||||
let lock_manager = self.get_lock_manager();
|
||||
let initial_remaining = acquire_deadline.saturating_duration_since(Instant::now());
|
||||
let reclaimed_before_acquire = if initial_remaining.is_zero() {
|
||||
0
|
||||
} else {
|
||||
timeout(initial_remaining, self.reclaim_expired_guards_for_resource(&request.resource))
|
||||
.await
|
||||
.unwrap_or(0)
|
||||
};
|
||||
|
||||
let build_lock_request = |acquire_timeout| match request.lock_type {
|
||||
LockType::Exclusive => crate::ObjectLockRequest::new_write(request.resource.clone(), request.owner.clone())
|
||||
@@ -262,7 +291,11 @@ impl LockClient for LocalClient {
|
||||
return Ok(LockResponse::success(lock_info, Duration::ZERO));
|
||||
}
|
||||
Err(crate::fast_lock::LockResult::Timeout) => {
|
||||
if !retried_after_reclaim && self.reclaim_expired_guards_for_resource(&request.resource).await > 0 {
|
||||
if !retried_after_reclaim {
|
||||
let remaining = acquire_deadline.saturating_duration_since(Instant::now());
|
||||
if !remaining.is_zero() {
|
||||
let _ = timeout(remaining, self.reclaim_expired_guards_for_resource(&request.resource)).await;
|
||||
}
|
||||
retried_after_reclaim = true;
|
||||
continue;
|
||||
}
|
||||
@@ -272,7 +305,11 @@ impl LockClient for LocalClient {
|
||||
current_owner,
|
||||
current_mode,
|
||||
}) => {
|
||||
if !retried_after_reclaim && self.reclaim_expired_guards_for_resource(&request.resource).await > 0 {
|
||||
if !retried_after_reclaim {
|
||||
let remaining = acquire_deadline.saturating_duration_since(Instant::now());
|
||||
if !remaining.is_zero() {
|
||||
let _ = timeout(remaining, self.reclaim_expired_guards_for_resource(&request.resource)).await;
|
||||
}
|
||||
retried_after_reclaim = true;
|
||||
continue;
|
||||
}
|
||||
@@ -290,8 +327,11 @@ impl LockClient for LocalClient {
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let mut guards = shard.write().await;
|
||||
if let Some(guard) = guards.remove(lock_id) {
|
||||
let guard = {
|
||||
let mut guards = shard.write().await;
|
||||
guards.remove(lock_id)
|
||||
};
|
||||
if let Some(guard) = guard {
|
||||
// Guard automatically releases the lock when dropped
|
||||
drop(guard.guard);
|
||||
Ok(true)
|
||||
@@ -418,6 +458,88 @@ mod tests {
|
||||
panic!("lock guard was not reaped before test deadline");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_does_not_wait_on_unrelated_guard_shards() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_secs(60));
|
||||
client.reaper_started.store(true, Ordering::Release);
|
||||
let resource = crate::ObjectKey::new("bucket", "resource-shard-hot-path");
|
||||
let target_shard = client.get_resource_shard_index(&resource);
|
||||
|
||||
let mut unrelated_writes = Vec::new();
|
||||
for (index, shard) in client.guard_storage.iter().enumerate() {
|
||||
if index != target_shard {
|
||||
unrelated_writes.push(shard.write().await);
|
||||
}
|
||||
}
|
||||
|
||||
let lock_request = request(resource, "owner", Duration::from_secs(30));
|
||||
let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&lock_request))
|
||||
.await
|
||||
.expect("an acquire must not sweep unrelated guard shards")
|
||||
.expect("local acquire should return a response");
|
||||
assert!(response.success);
|
||||
|
||||
drop(unrelated_writes);
|
||||
assert!(client.release(&lock_request.lock_id).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reclaim_expired_guard_only_writes_the_resource_shard() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_secs(60));
|
||||
client.reaper_started.store(true, Ordering::Release);
|
||||
let resource = crate::ObjectKey::new("bucket", "resource-shard-expired");
|
||||
let target_shard = client.get_resource_shard_index(&resource);
|
||||
let expired = request(resource.clone(), "expired-owner", Duration::ZERO);
|
||||
assert!(client.acquire_lock(&expired).await.unwrap().success);
|
||||
|
||||
let mut unrelated_writes = Vec::new();
|
||||
for (index, shard) in client.guard_storage.iter().enumerate() {
|
||||
if index != target_shard {
|
||||
unrelated_writes.push(shard.write().await);
|
||||
}
|
||||
}
|
||||
|
||||
let replacement = request(resource, "replacement-owner", Duration::from_secs(30));
|
||||
let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&replacement))
|
||||
.await
|
||||
.expect("expired guard reclaim must only write the resource shard")
|
||||
.expect("local acquire should return a response");
|
||||
assert!(response.success);
|
||||
|
||||
drop(unrelated_writes);
|
||||
assert!(client.release(&replacement.lock_id).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_deadline_includes_resource_shard_reclaim_wait() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_secs(60));
|
||||
client.reaper_started.store(true, Ordering::Release);
|
||||
let resource = crate::ObjectKey::new("bucket", "resource-shard-deadline");
|
||||
let target_shard = client.get_resource_shard_index(&resource);
|
||||
let _target_write = client.guard_storage[target_shard].write().await;
|
||||
let lock_request = request(resource, "owner", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
|
||||
|
||||
let response = tokio::time::timeout(Duration::from_millis(100), client.acquire_lock(&lock_request))
|
||||
.await
|
||||
.expect("acquire should return after its reclaim budget expires")
|
||||
.expect("local acquire should return a response");
|
||||
assert!(!response.success);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_rejects_a_lock_id_for_another_resource() {
|
||||
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
|
||||
let mut lock_request = request(crate::ObjectKey::new("bucket", "requested"), "owner", Duration::from_secs(30));
|
||||
lock_request.lock_id.resource = crate::ObjectKey::new("bucket", "other");
|
||||
|
||||
let response = client.acquire_lock(&lock_request).await.unwrap();
|
||||
assert!(!response.success);
|
||||
assert!(response.error.unwrap().contains("does not match"));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn expired_guard_is_reaped_without_resource_reacquire() {
|
||||
let manager = Arc::new(GlobalLockManager::new());
|
||||
|
||||
@@ -53,7 +53,9 @@ impl NotifyBucketConfigManager {
|
||||
self.rule_engine.has_subscriber(bucket, event).await
|
||||
}
|
||||
|
||||
pub async fn load_bucket_notification_config(
|
||||
/// Validates that the runtime can accept `cfg` for `bucket` without
|
||||
/// mutating any state.
|
||||
pub async fn validate_bucket_notification_config(
|
||||
&self,
|
||||
bucket: &str,
|
||||
cfg: &BucketNotificationConfig,
|
||||
@@ -98,6 +100,11 @@ impl NotifyBucketConfigManager {
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Publishes an already validated `cfg` to the runtime rule state.
|
||||
pub async fn apply_bucket_notification_config(&self, bucket: &str, cfg: &BucketNotificationConfig) {
|
||||
self.subscriber_view.apply_bucket_config(bucket, cfg);
|
||||
self.rule_engine.set_bucket_rules(bucket, cfg.get_rules_map().clone()).await;
|
||||
info!(
|
||||
@@ -109,6 +116,15 @@ impl NotifyBucketConfigManager {
|
||||
rule_count = cfg.get_rules_map().inner().len(),
|
||||
"notify bucket config state"
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn load_bucket_notification_config(
|
||||
&self,
|
||||
bucket: &str,
|
||||
cfg: &BucketNotificationConfig,
|
||||
) -> Result<(), NotificationError> {
|
||||
self.validate_bucket_notification_config(bucket, cfg).await?;
|
||||
self.apply_bucket_notification_config(bucket, cfg).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -122,7 +138,7 @@ impl NotifyBucketConfigManager {
|
||||
mod tests {
|
||||
use super::NotifyBucketConfigManager;
|
||||
use crate::{
|
||||
BucketNotificationConfig, integration::NotificationMetrics,
|
||||
BucketNotificationConfig, NotificationError, integration::NotificationMetrics,
|
||||
notification_system_subscriber::NotificationSystemSubscriberView, notifier::EventNotifier, rule_engine::NotifyRuleEngine,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
@@ -156,4 +172,37 @@ mod tests {
|
||||
manager.remove_bucket_notification_config("bucket").await;
|
||||
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_bucket_notification_config_rejects_missing_targets_without_applying_rules() {
|
||||
let manager = build_manager();
|
||||
let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
|
||||
let mut cfg = BucketNotificationConfig::new("us-east-1");
|
||||
cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id);
|
||||
|
||||
let err = manager
|
||||
.validate_bucket_notification_config("bucket", &cfg)
|
||||
.await
|
||||
.expect_err("validation must fail when the runtime has no notify targets");
|
||||
assert!(matches!(err, NotificationError::Configuration(_)));
|
||||
assert!(!manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
assert!(!manager.subscriber_view.has_subscriber("bucket", &EventName::ObjectCreatedPut));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_validation_keeps_previous_bucket_rules() {
|
||||
let manager = build_manager();
|
||||
let target_id = TargetID::new("primary".to_string(), "webhook".to_string());
|
||||
let mut cfg = BucketNotificationConfig::new("us-east-1");
|
||||
cfg.add_rule(&[EventName::ObjectCreatedPut], "*".to_string(), target_id);
|
||||
manager.apply_bucket_notification_config("bucket", &cfg).await;
|
||||
assert!(manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
|
||||
let err = manager
|
||||
.validate_bucket_notification_config("bucket", &cfg)
|
||||
.await
|
||||
.expect_err("validation must fail when the runtime has no notify targets");
|
||||
assert!(matches!(err, NotificationError::Configuration(_)));
|
||||
assert!(manager.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
}
|
||||
}
|
||||
|
||||
+37
-10
@@ -239,6 +239,42 @@ pub mod notifier_global {
|
||||
.await
|
||||
}
|
||||
|
||||
fn bucket_notification_config_for_rules(
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> BucketNotificationConfig {
|
||||
let mut bucket_config = BucketNotificationConfig::new(region);
|
||||
|
||||
for (event_names, prefix, suffix, target_ids) in event_rules {
|
||||
// Use `new_pattern` to construct a matching pattern
|
||||
let pattern = crate::rules::pattern::new_pattern(Some(prefix.as_str()), Some(suffix.as_str()));
|
||||
|
||||
for target_id in target_ids {
|
||||
bucket_config.add_rule(event_names, pattern.clone(), target_id.clone());
|
||||
}
|
||||
}
|
||||
|
||||
bucket_config
|
||||
}
|
||||
|
||||
/// Checks that the runtime can accept `event_rules` for `region` without
|
||||
/// publishing them. Callers use this to fail before persisting a bucket
|
||||
/// notification configuration the notify subsystem would reject.
|
||||
pub async fn validate_event_specific_rules(
|
||||
bucket_name: &str,
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> Result<(), NotificationError> {
|
||||
let bucket_config = bucket_notification_config_for_rules(region, event_rules);
|
||||
|
||||
// Get global NotificationSystem instance
|
||||
let notification_sys = notification_system().ok_or(NotificationError::Lifecycle(LifecycleError::NotInitialized))?;
|
||||
|
||||
notification_sys
|
||||
.validate_bucket_notification_config(bucket_name, &bucket_config)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Dynamically add notification rules according to different event types.
|
||||
///
|
||||
/// # Parameter
|
||||
@@ -256,16 +292,7 @@ pub mod notifier_global {
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> Result<(), NotificationError> {
|
||||
let mut bucket_config = BucketNotificationConfig::new(region);
|
||||
|
||||
for (event_names, prefix, suffix, target_ids) in event_rules {
|
||||
// Use `new_pattern` to construct a matching pattern
|
||||
let pattern = crate::rules::pattern::new_pattern(Some(prefix.as_str()), Some(suffix.as_str()));
|
||||
|
||||
for target_id in target_ids {
|
||||
bucket_config.add_rule(event_names, pattern.clone(), target_id.clone());
|
||||
}
|
||||
}
|
||||
let bucket_config = bucket_notification_config_for_rules(region, event_rules);
|
||||
|
||||
// Get global NotificationSystem instance
|
||||
let notification_sys = notification_system().ok_or(NotificationError::Lifecycle(LifecycleError::NotInitialized))?;
|
||||
|
||||
@@ -372,6 +372,19 @@ impl NotificationSystem {
|
||||
self.services.config_manager.lifecycle().is_converged()
|
||||
}
|
||||
|
||||
/// Validates a bucket notification configuration against the runtime targets
|
||||
/// without publishing it, so callers can reject a request before persisting.
|
||||
pub async fn validate_bucket_notification_config(
|
||||
&self,
|
||||
bucket: &str,
|
||||
cfg: &BucketNotificationConfig,
|
||||
) -> Result<(), NotificationError> {
|
||||
self.services
|
||||
.bucket_config_manager
|
||||
.validate_bucket_notification_config(bucket, cfg)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Loads the bucket notification configuration
|
||||
pub async fn load_bucket_notification_config(
|
||||
&self,
|
||||
|
||||
@@ -65,7 +65,7 @@ rustfs-config = { workspace = true, features = ["constants"] }
|
||||
rustfs-io-metrics = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["os"] }
|
||||
futures = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "time", "fs", "rt-multi-thread"] }
|
||||
tokio = { workspace = true, features = ["sync", "time", "fs", "rt-multi-thread","macros"] }
|
||||
tokio-util = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
@@ -74,11 +74,11 @@ hotpath-cpu = [
|
||||
# Tokio runtime-level telemetry. Requires a `--cfg tokio_unstable` build; the
|
||||
# build script fails the compile when that flag is missing. Off by default so
|
||||
# ordinary builds neither pay for nor depend on Tokio's unstable API.
|
||||
dial9 = ["dep:dial9-tokio-telemetry", "dial9-tokio-telemetry/process-resource"]
|
||||
dial9 = ["dep:dial9"]
|
||||
#
|
||||
# NOTE: there is deliberately no `dial9-taskdump` feature. dial9 only captures a
|
||||
# task dump for futures it wrapped itself, i.e. those spawned via
|
||||
# `dial9_tokio_telemetry::spawn`. RustFS spawns with `tokio::spawn` throughout,
|
||||
# `dial9::spawn`. RustFS spawns with `tokio::spawn` throughout,
|
||||
# so enabling `tokio/taskdump` would cost a Linux-only build constraint and
|
||||
# record nothing. Measured on an identical workload: 0 dumps via `tokio::spawn`,
|
||||
# 14709 via `dial9::spawn`. Re-adding this feature only makes sense together
|
||||
@@ -149,7 +149,7 @@ tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "traci
|
||||
tokio = { workspace = true, features = ["sync", "rt-multi-thread", "time", "macros"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
url.workspace = true
|
||||
dial9-tokio-telemetry = { workspace = true, optional = true }
|
||||
dial9 = { workspace = true, optional = true, features = ["tokio", "process-resource"] }
|
||||
thiserror = { workspace = true }
|
||||
zstd = { workspace = true, features = ["zstdmt"] }
|
||||
sysinfo = { workspace = true }
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@
|
||||
|
||||
//! Enforces that the `dial9` feature is paired with `--cfg tokio_unstable`.
|
||||
//!
|
||||
//! `dial9-tokio-telemetry` hooks Tokio's unstable runtime instrumentation. The
|
||||
//! `dial9` hooks Tokio's unstable runtime instrumentation. The
|
||||
//! flag lives in `RUSTFLAGS`, which a caller can silently clobber by exporting
|
||||
//! their own value — that used to turn telemetry off with no diagnostic. Fail
|
||||
//! the build instead.
|
||||
|
||||
@@ -14,10 +14,10 @@
|
||||
|
||||
//! Configuration for dial9 Tokio runtime telemetry.
|
||||
//!
|
||||
//! This module carries no dependency on the `dial9-tokio-telemetry` crate, so
|
||||
//! it compiles identically whether or not the `dial9` feature is enabled. That
|
||||
//! lets callers read the configured state (and export metrics about it) from a
|
||||
//! binary that was built without telemetry support.
|
||||
//! This module carries no dependency on the optional `dial9` crate, so it
|
||||
//! compiles identically whether or not the `dial9` feature is enabled. That lets
|
||||
//! callers read the configured state (and export metrics about it) from a binary
|
||||
//! that was built without telemetry support.
|
||||
|
||||
use super::state::dial9_runtime_state;
|
||||
use rustfs_config::{
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
|
||||
use crate::TelemetryError;
|
||||
|
||||
/// Placeholder for `dial9_tokio_telemetry::telemetry::TelemetryGuard`.
|
||||
/// Placeholder for `dial9::TelemetryGuard`.
|
||||
///
|
||||
/// Never constructed: [`build_traced_runtime`] always fails without the
|
||||
/// `dial9` feature.
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! `dial9-tokio-telemetry` integration, compiled when the `dial9` feature is on.
|
||||
//! `dial9` integration, compiled when the `dial9` feature is on.
|
||||
//!
|
||||
//! Captures Tokio runtime-level events (poll start/end, worker park/unpark,
|
||||
//! task spawn/terminate) into rotating binary trace segments. This is an
|
||||
@@ -23,9 +23,8 @@ use super::config::Dial9Config;
|
||||
use super::state::{dial9_runtime_state, measure_disk_usage_bytes};
|
||||
use super::{EVENT_DIAL9_STATE, LOG_COMPONENT_OBS, LOG_SUBSYSTEM_DIAL9};
|
||||
use crate::TelemetryError;
|
||||
use dial9_tokio_telemetry::telemetry::{
|
||||
Dial9Handle, Dial9HandleTokioExt, DiskBuffer, ProcessResourceUsageConfig, RecorderPerfExt, TokioAttachOptions, recorder,
|
||||
};
|
||||
use dial9::process::ProcessResourceUsageConfig;
|
||||
use dial9::{Dial9Handle, Dial9HandleTokioExt, DiskBuffer, RecorderPerfExt, TokioAttachOptions, recorder};
|
||||
use std::time::Duration;
|
||||
use tracing::{info, warn};
|
||||
|
||||
@@ -134,7 +133,7 @@ pub fn build_traced_runtime(
|
||||
.map(|runtime| (runtime, guard, shutdown));
|
||||
|
||||
// No task dumps here. dial9 captures a task dump only for futures it
|
||||
// wrapped itself, i.e. those spawned via `dial9_tokio_telemetry::spawn`;
|
||||
// wrapped itself, i.e. those spawned via `dial9::spawn`;
|
||||
// `tokio::spawn` gets no wrapper. RustFS spawns with `tokio::spawn`
|
||||
// throughout, so enabling task dumps records nothing. Measured on an
|
||||
// identical workload: 0 dumps via `tokio::spawn`, 14709 via `dial9::spawn`.
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tokio runtime-level telemetry via `dial9-tokio-telemetry`.
|
||||
//! Tokio runtime-level telemetry via `dial9`.
|
||||
//!
|
||||
//! This is an on-demand profiler for executor-level faults — long polls that
|
||||
//! stall a worker, park/unpark storms, tasks that never yield — which are
|
||||
|
||||
@@ -34,7 +34,7 @@ metrics = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
rustfs-heal-contracts = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio = { workspace = true, features = ["sync","rt"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
|
||||
@@ -910,11 +910,26 @@ impl DataUsageCache {
|
||||
if self.info.leader_epoch > leader_epoch {
|
||||
return DataUsageCachePrepareOutcome::RejectedNewerLeader;
|
||||
}
|
||||
// A checkpoint is a durable traversal frontier, not a publication
|
||||
// result. A newer leader may adopt it only when the frontier is
|
||||
// independently fenced by the bucket identity and has a validated
|
||||
// cursor/coverage receipt. Caches without that proof still take the
|
||||
// normal rebuild path; this keeps an old, incomplete writer from
|
||||
// authorizing a new leader to skip namespace coverage.
|
||||
let cross_epoch_checkpoint = self.info.leader_epoch < leader_epoch
|
||||
&& self.info.next_cycle == next_cycle
|
||||
&& self.info.scan_identity == Some(identity)
|
||||
&& (self.validated_scan_frontier().is_some()
|
||||
|| self.validated_raw_enumeration_cursor().is_some()
|
||||
|| self.validated_raw_enumeration_page_index().is_some());
|
||||
let handoff_frontier = cross_epoch_checkpoint
|
||||
.then(|| self.validated_scan_frontier().map(str::to_owned))
|
||||
.flatten();
|
||||
let reusable = identity.is_valid()
|
||||
&& name != DATA_USAGE_ROOT
|
||||
&& self.info.name == name
|
||||
&& self.info.source == Some(source)
|
||||
&& self.info.leader_epoch == leader_epoch
|
||||
&& (self.info.leader_epoch == leader_epoch || cross_epoch_checkpoint)
|
||||
&& self.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT
|
||||
&& self.info.scan_identity == Some(identity)
|
||||
&& self.info.tier_registry_generation == Some(identity.tier_registry_generation)
|
||||
@@ -974,6 +989,15 @@ impl DataUsageCache {
|
||||
self.info.tier_registry_generation = Some(identity.tier_registry_generation);
|
||||
self.info.scan_identity = Some(identity);
|
||||
self.info.snapshot_complete = false;
|
||||
if let Some(frontier) = handoff_frontier
|
||||
&& let Ok(digest) = self.coverage_prefix_digest(&frontier)
|
||||
&& let Some(receipt) = self.info.scan_coverage_receipt.as_mut()
|
||||
{
|
||||
// Migrate a legacy epoch-bound receipt while the old epoch is
|
||||
// still available for validation. New receipts deliberately do
|
||||
// not bind to the process-local leader epoch.
|
||||
receipt.digest = digest;
|
||||
}
|
||||
if let Some(progress) = &mut self.info.scan_progress {
|
||||
progress.requested_plan = scan_plan_digest;
|
||||
} else {
|
||||
@@ -998,19 +1022,53 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
fn coverage_prefix_digest(&self, through: &str) -> Result<[u8; 32], serde_json::Error> {
|
||||
self.coverage_prefix_digest_with_epoch(through, None)
|
||||
}
|
||||
|
||||
fn legacy_coverage_prefix_digest(&self, through: &str) -> Result<[u8; 32], serde_json::Error> {
|
||||
self.coverage_prefix_digest_with_epoch(through, Some(self.info.leader_epoch))
|
||||
}
|
||||
|
||||
fn coverage_prefix_digest_with_epoch(
|
||||
&self,
|
||||
through: &str,
|
||||
legacy_leader_epoch: Option<u64>,
|
||||
) -> Result<[u8; 32], serde_json::Error> {
|
||||
let mut writer = CheckpointDigestWriter(Sha256::new());
|
||||
serde_json::to_writer(
|
||||
&mut writer,
|
||||
&(
|
||||
&self.info.name,
|
||||
self.info.scan_identity,
|
||||
self.info.source,
|
||||
self.info.leader_epoch,
|
||||
self.info.cache_key_format,
|
||||
self.info.scan_progress.map(|progress| progress.started_plan),
|
||||
through,
|
||||
),
|
||||
)?;
|
||||
if let Some(leader_epoch) = legacy_leader_epoch {
|
||||
// Keep the old tuple available for one-way migration of caches
|
||||
// written before leader handoff recovery was introduced.
|
||||
serde_json::to_writer(
|
||||
&mut writer,
|
||||
&(
|
||||
&self.info.name,
|
||||
self.info.scan_identity,
|
||||
self.info.source,
|
||||
leader_epoch,
|
||||
self.info.cache_key_format,
|
||||
self.info.scan_progress.map(|progress| progress.started_plan),
|
||||
through,
|
||||
),
|
||||
)?;
|
||||
} else {
|
||||
// The receipt must survive a scanner leader handoff. The bucket
|
||||
// identity and publication epoch below fence the namespace; the
|
||||
// process-local leader epoch is intentionally excluded because it
|
||||
// changes during a safe handoff. The domain version also keeps
|
||||
// old epoch-bound receipts from colliding with this digest.
|
||||
serde_json::to_writer(
|
||||
&mut writer,
|
||||
&(
|
||||
2u8,
|
||||
&self.info.name,
|
||||
self.info.scan_identity,
|
||||
self.info.source,
|
||||
self.info.cache_key_format,
|
||||
self.info.scan_progress.map(|progress| progress.started_plan),
|
||||
through,
|
||||
),
|
||||
)?;
|
||||
}
|
||||
let mut prefix = self
|
||||
.cache
|
||||
.iter()
|
||||
@@ -1049,7 +1107,8 @@ impl DataUsageCache {
|
||||
.strip_prefix(&self.info.name)
|
||||
.is_some_and(|suffix| suffix.starts_with('/'))
|
||||
&& self.find(&receipt.through).is_some()
|
||||
&& self.coverage_prefix_digest(&receipt.through).ok() == Some(receipt.digest))
|
||||
&& (self.coverage_prefix_digest(&receipt.through).ok() == Some(receipt.digest)
|
||||
|| self.legacy_coverage_prefix_digest(&receipt.through).ok() == Some(receipt.digest)))
|
||||
.then_some(receipt.through.as_str())
|
||||
}
|
||||
|
||||
|
||||
@@ -135,6 +135,15 @@ impl DataUsageCache {
|
||||
Ok(DataUsageCacheRevisions { main, backup })
|
||||
}
|
||||
|
||||
pub(crate) async fn read_revisions<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheRevisions> {
|
||||
let main_path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
let main = read_config_revision(store.clone(), &main_path).await?;
|
||||
let backup = Some(read_config_revision(store, &backup_path).await?);
|
||||
Ok(DataUsageCacheRevisions { main, backup })
|
||||
}
|
||||
|
||||
async fn load_cache<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheLoadResult> {
|
||||
let mut last_retryable = None;
|
||||
|
||||
|
||||
@@ -2033,6 +2033,59 @@ fn data_usage_cache_prepare_for_scan_fences_leader_epochs() {
|
||||
assert!(cache.cache.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_bucket_checkpoint_migrates_legacy_epoch_bound_receipt() {
|
||||
let identity = valid_scan_identity();
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 8,
|
||||
leader_epoch: 1,
|
||||
source: Some(source),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
scan_identity: Some(identity),
|
||||
tier_registry_generation: Some(identity.tier_registry_generation),
|
||||
scan_progress: Some(DataUsageScanProgress {
|
||||
started_plan: TEST_PLAN_DIGEST,
|
||||
requested_plan: TEST_PLAN_DIGEST,
|
||||
}),
|
||||
scan_resume_after: Some("bucket/a".to_string()),
|
||||
scan_checkpoint: Some(DataUsageScanCheckpoint::new(
|
||||
"bucket/a".to_string(),
|
||||
DataUsageScanCheckpointReason::Objects,
|
||||
)),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
cache.replace(
|
||||
"bucket/a",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let legacy_digest = cache
|
||||
.legacy_coverage_prefix_digest("bucket/a")
|
||||
.expect("legacy receipt digest");
|
||||
cache.info.scan_coverage_receipt = Some(DataUsageScanCoverageReceipt {
|
||||
through: "bucket/a".to_string(),
|
||||
digest: legacy_digest,
|
||||
});
|
||||
assert_eq!(cache.validated_scan_frontier(), Some("bucket/a"));
|
||||
|
||||
assert_eq!(
|
||||
cache.prepare_bucket_checkpoint("bucket", 8, 2, source, TEST_PLAN_DIGEST, identity),
|
||||
DataUsageCachePrepareOutcome::Reused
|
||||
);
|
||||
assert_eq!(cache.info.leader_epoch, 2);
|
||||
assert_eq!(cache.validated_scan_frontier(), Some("bucket/a"));
|
||||
assert_ne!(cache.info.scan_coverage_receipt.expect("migrated receipt").digest, legacy_digest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_usage_cache_mutations_update_in_place() {
|
||||
let mut cache = DataUsageCache {
|
||||
|
||||
@@ -170,7 +170,7 @@ fn cached_remote_scanner_fence_matches(
|
||||
.is_some_and(|cached| cached.matches(requested_cycle, requested_leader_epoch, now)))
|
||||
}
|
||||
|
||||
async fn validate_remote_scanner_request_fence_with_store(
|
||||
pub(crate) async fn validate_remote_scanner_request_fence_with_store(
|
||||
requested_cycle: u64,
|
||||
requested_leader_epoch: u64,
|
||||
store: Arc<impl ScannerObjectIO>,
|
||||
|
||||
@@ -16,9 +16,9 @@
|
||||
use crate::RUSTFS_META_BUCKET;
|
||||
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig};
|
||||
use crate::scanner_io::{
|
||||
DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerDiskScanOptions, ScannerDiskScanOutcome, ScannerIODisk,
|
||||
acquire_scanner_cache_locks, cache_root_entry_info, current_cache_root_or_prepare_with_generation,
|
||||
scanner_set_disk_inventory,
|
||||
DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerCheckpointPersistResult, ScannerDiskScanOptions,
|
||||
ScannerDiskScanOutcome, ScannerIODisk, acquire_scanner_cache_locks, cache_root_entry_info,
|
||||
current_cache_root_or_prepare_with_generation, persist_scanner_checkpoint, scanner_set_disk_inventory,
|
||||
};
|
||||
use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
|
||||
use crate::{
|
||||
@@ -43,7 +43,7 @@ use std::sync::{
|
||||
};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||
use tokio::sync::{Notify, OwnedSemaphorePermit};
|
||||
use tokio::sync::{Notify, OwnedSemaphorePermit, mpsc};
|
||||
use tokio::time::{Instant, MissedTickBehavior};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
@@ -709,7 +709,7 @@ async fn scan_and_persist_local_bucket(
|
||||
}
|
||||
})?;
|
||||
let mut cache = DataUsageCache::default();
|
||||
let revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
|
||||
let mut revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
|
||||
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
|
||||
})?;
|
||||
// Remote workers use the same cycle-frozen registry as `scan_data_folder`.
|
||||
@@ -779,6 +779,7 @@ async fn scan_and_persist_local_bucket(
|
||||
cache.info.skip_healing = skip_healing;
|
||||
|
||||
let set_disks = scanner_set_disk_inventory(set.as_ref()).await;
|
||||
let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::<DataUsageCache>(1);
|
||||
let scan_ctx = ctx.child_token();
|
||||
let scan = ScannerIODisk::nsscanner_disk(
|
||||
disk.clone(),
|
||||
@@ -790,6 +791,7 @@ async fn scan_and_persist_local_bucket(
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: Some(checkpoint_tx),
|
||||
},
|
||||
);
|
||||
tokio::pin!(scan);
|
||||
@@ -797,6 +799,7 @@ async fn scan_and_persist_local_bucket(
|
||||
tokio::pin!(fence_watch);
|
||||
let mut lock_watch = tokio::time::interval(NS_SCANNER_LOCK_POLL_INTERVAL);
|
||||
lock_watch.set_missed_tick_behavior(MissedTickBehavior::Delay);
|
||||
let mut checkpoint_channel_closed = false;
|
||||
let outcome = loop {
|
||||
tokio::select! {
|
||||
result = &mut scan => {
|
||||
@@ -823,6 +826,50 @@ async fn scan_and_persist_local_bucket(
|
||||
));
|
||||
}
|
||||
}
|
||||
checkpoint = checkpoint_rx.recv(), if !checkpoint_channel_closed => {
|
||||
let Some(checkpoint) = checkpoint else {
|
||||
checkpoint_channel_closed = true;
|
||||
continue;
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
scan_ctx.cancel();
|
||||
let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await;
|
||||
return Err(RemoteScannerServerError::worker(
|
||||
"remote namespace scanner cache lock was lost before checkpoint save",
|
||||
));
|
||||
}
|
||||
match persist_scanner_checkpoint(
|
||||
set.clone(),
|
||||
&cache_name,
|
||||
&checkpoint,
|
||||
&mut revisions,
|
||||
expected_publication_epoch,
|
||||
next_cycle,
|
||||
leader_epoch,
|
||||
)
|
||||
.await
|
||||
{
|
||||
ScannerCheckpointPersistResult::Saved => {
|
||||
if guard.is_lock_lost() {
|
||||
scan_ctx.cancel();
|
||||
let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await;
|
||||
return Err(RemoteScannerServerError::retry_bucket(
|
||||
"remote namespace scanner cache lock was lost after checkpoint save",
|
||||
));
|
||||
}
|
||||
}
|
||||
ScannerCheckpointPersistResult::FenceChanged => {
|
||||
scan_ctx.cancel();
|
||||
let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await;
|
||||
return Err(RemoteScannerServerError::retry_bucket(
|
||||
"remote namespace scanner cache fence changed during checkpoint save",
|
||||
));
|
||||
}
|
||||
ScannerCheckpointPersistResult::Failed(_error) => {
|
||||
checkpoint_channel_closed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
result = &mut fence_watch => {
|
||||
scan_ctx.cancel();
|
||||
let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await;
|
||||
|
||||
@@ -24,14 +24,15 @@ use crate::data_usage_define::DataUsageCacheRevision;
|
||||
use crate::storage_api::ScannerStorage;
|
||||
use crate::storage_api::owner::ObjectIO as _;
|
||||
use crate::storage_api::owner::{
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementReplica,
|
||||
register_scanner_pause_backlog_retirement_planner,
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan,
|
||||
ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,
|
||||
};
|
||||
use crate::{BUCKET_META_PREFIX, ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerObjectOptions, SetDisks};
|
||||
use futures::future::join_all;
|
||||
use http::HeaderMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::fmt;
|
||||
use std::sync::{Arc, LazyLock, RwLock};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -996,10 +997,77 @@ fn scanner_pause_backlog_replica_ids(replicas: &[ScannerPauseBacklogReplica]) ->
|
||||
ids
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum ScannerPauseBacklogCommitSelectionError {
|
||||
ConflictingMaximumMembershipCommitProofs,
|
||||
}
|
||||
|
||||
impl fmt::Display for ScannerPauseBacklogCommitSelectionError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("scanner pause backlog has conflicting maximum-membership commit proofs")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum ScannerPauseBacklogSelectionError {
|
||||
RetryableHandoffConflict,
|
||||
AuthorityConflict(String),
|
||||
InvalidRecord(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for ScannerPauseBacklogSelectionError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::RetryableHandoffConflict => {
|
||||
f.write_str("scanner pause backlog has conflicting maximum-membership commit proofs")
|
||||
}
|
||||
Self::AuthorityConflict(reason) | Self::InvalidRecord(reason) => f.write_str(reason),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ScannerPauseBacklogCommitSelectionError> for ScannerPauseBacklogSelectionError {
|
||||
fn from(_err: ScannerPauseBacklogCommitSelectionError) -> Self {
|
||||
Self::RetryableHandoffConflict
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_pause_backlog_retirement_selection_error(
|
||||
err: ScannerPauseBacklogSelectionError,
|
||||
) -> ScannerPauseBacklogRetirementError {
|
||||
match err {
|
||||
ScannerPauseBacklogSelectionError::RetryableHandoffConflict => {
|
||||
ScannerPauseBacklogRetirementError::HandoffConflict { reason: err.to_string() }
|
||||
}
|
||||
ScannerPauseBacklogSelectionError::AuthorityConflict(reason) => {
|
||||
ScannerPauseBacklogRetirementError::AuthorityConflict { reason }
|
||||
}
|
||||
ScannerPauseBacklogSelectionError::InvalidRecord(reason) => ScannerPauseBacklogRetirementError::InvalidRecord { reason },
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_pause_backlog_retirement_commit_error(
|
||||
err: ScannerPauseBacklogCommitSelectionError,
|
||||
) -> ScannerPauseBacklogRetirementError {
|
||||
scanner_pause_backlog_retirement_handoff_conflict(err.to_string())
|
||||
}
|
||||
|
||||
fn scanner_pause_backlog_retirement_handoff_conflict(reason: impl Into<String>) -> ScannerPauseBacklogRetirementError {
|
||||
ScannerPauseBacklogRetirementError::HandoffConflict { reason: reason.into() }
|
||||
}
|
||||
|
||||
fn scanner_pause_backlog_retirement_authority_conflict(reason: impl Into<String>) -> ScannerPauseBacklogRetirementError {
|
||||
ScannerPauseBacklogRetirementError::AuthorityConflict { reason: reason.into() }
|
||||
}
|
||||
|
||||
fn scanner_pause_backlog_retirement_invalid_record(reason: impl Into<String>) -> ScannerPauseBacklogRetirementError {
|
||||
ScannerPauseBacklogRetirementError::InvalidRecord { reason: reason.into() }
|
||||
}
|
||||
|
||||
fn select_scanner_pause_backlog_commit(
|
||||
replicas: &[ScannerPauseBacklogReplica],
|
||||
replica_ids: &[ScannerPauseBacklogReplicaId],
|
||||
) -> Result<Option<ScannerPauseBacklogCommitRecord>, String> {
|
||||
) -> Result<Option<ScannerPauseBacklogCommitRecord>, ScannerPauseBacklogCommitSelectionError> {
|
||||
let current_ids = replica_ids.iter().copied().collect::<BTreeSet<_>>();
|
||||
let replicas_by_id = replicas
|
||||
.iter()
|
||||
@@ -1031,15 +1099,19 @@ fn select_scanner_pause_backlog_commit(
|
||||
let Some(max_membership_len) = valid.iter().map(|committed| committed.replicas.len()).max() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut largest = valid
|
||||
let largest = valid
|
||||
.into_iter()
|
||||
.filter(|committed| committed.replicas.len() == max_membership_len);
|
||||
let Some(mut selected) = largest.next() else {
|
||||
let largest = largest.collect::<Vec<_>>();
|
||||
let Some(mut selected) = largest.first().cloned() else {
|
||||
return Ok(None);
|
||||
};
|
||||
for committed in largest {
|
||||
if largest.iter().any(|committed| committed.ledger != selected.ledger) {
|
||||
return Err(ScannerPauseBacklogCommitSelectionError::ConflictingMaximumMembershipCommitProofs);
|
||||
}
|
||||
for committed in largest.into_iter().skip(1) {
|
||||
if committed.ledger != selected.ledger {
|
||||
return Err("scanner pause backlog has conflicting maximum-membership commit proofs".to_string());
|
||||
unreachable!("conflicting maximum-membership commit proofs were checked above");
|
||||
}
|
||||
if committed.replicas < selected.replicas {
|
||||
selected = committed;
|
||||
@@ -1048,21 +1120,26 @@ fn select_scanner_pause_backlog_commit(
|
||||
Ok(Some(selected))
|
||||
}
|
||||
|
||||
fn select_scanner_pause_backlog_replicas(replicas: Vec<ScannerPauseBacklogReplica>) -> Result<LoadedScannerPauseBacklog, String> {
|
||||
fn select_scanner_pause_backlog_replicas(
|
||||
replicas: Vec<ScannerPauseBacklogReplica>,
|
||||
) -> Result<LoadedScannerPauseBacklog, ScannerPauseBacklogSelectionError> {
|
||||
if replicas.is_empty() {
|
||||
return Err("scanner pause backlog has no storage replicas".to_string());
|
||||
return Err(ScannerPauseBacklogSelectionError::AuthorityConflict(
|
||||
"scanner pause backlog has no storage replicas".to_string(),
|
||||
));
|
||||
}
|
||||
for replica in &replicas {
|
||||
if let ScannerPauseBacklogReplicaState::FutureSchema(version) = &replica.state {
|
||||
return Err(format!(
|
||||
return Err(ScannerPauseBacklogSelectionError::InvalidRecord(format!(
|
||||
"scanner pause backlog pool {} set {} uses future schema {version}",
|
||||
replica.id.pool_index, replica.id.set_index
|
||||
));
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
let replica_ids = scanner_pause_backlog_replica_ids(&replicas);
|
||||
let authoritative_commit = select_scanner_pause_backlog_commit(&replicas, &replica_ids)?;
|
||||
let authoritative_commit =
|
||||
select_scanner_pause_backlog_commit(&replicas, &replica_ids).map_err(ScannerPauseBacklogSelectionError::from)?;
|
||||
let stable_consensus = if replicas.iter().all(|replica| {
|
||||
matches!(
|
||||
&replica.state,
|
||||
@@ -1093,17 +1170,21 @@ fn select_scanner_pause_backlog_replicas(replicas: Vec<ScannerPauseBacklogReplic
|
||||
}
|
||||
_ => unreachable!("replica state was checked above"),
|
||||
};
|
||||
return Err(format!(
|
||||
let reason = format!(
|
||||
"scanner pause backlog pool {} set {} is unavailable: {reason}",
|
||||
replica.id.pool_index, replica.id.set_index
|
||||
));
|
||||
);
|
||||
return Err(match &replica.state {
|
||||
ScannerPauseBacklogReplicaState::Invalid(_) => ScannerPauseBacklogSelectionError::InvalidRecord(reason),
|
||||
_ => ScannerPauseBacklogSelectionError::AuthorityConflict(reason),
|
||||
});
|
||||
}
|
||||
match &stable_consensus {
|
||||
Ok(stable) => stable.clone(),
|
||||
Err(()) => {
|
||||
return Err(
|
||||
return Err(ScannerPauseBacklogSelectionError::AuthorityConflict(
|
||||
"scanner pause backlog has neither a surviving membership commit nor a stable rollback point".to_string(),
|
||||
);
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1192,7 +1273,7 @@ pub fn register_scanner_pause_backlog_retirement() {
|
||||
fn decode_scanner_pause_backlog_retirement_replicas(
|
||||
source_pool_index: usize,
|
||||
replicas: &[ScannerPauseBacklogRetirementReplica],
|
||||
) -> Result<Vec<ScannerPauseBacklogReplica>, String> {
|
||||
) -> Result<Vec<ScannerPauseBacklogReplica>, ScannerPauseBacklogRetirementError> {
|
||||
let mut ids = BTreeSet::new();
|
||||
let mut decoded = Vec::with_capacity(replicas.len());
|
||||
let mut has_source = false;
|
||||
@@ -1202,19 +1283,27 @@ fn decode_scanner_pause_backlog_retirement_replicas(
|
||||
set_index: replica.set_index,
|
||||
};
|
||||
if !ids.insert(id) {
|
||||
return Err("scanner pause backlog retirement has duplicate replica membership".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_invalid_record(
|
||||
"scanner pause backlog retirement has duplicate replica membership",
|
||||
));
|
||||
}
|
||||
let state = match replica.data.as_deref() {
|
||||
Some(data) if data.len() <= MAX_SCANNER_PAUSE_BACKLOG_BYTES as usize => {
|
||||
let state = decode_scanner_pause_backlog_ledger(data);
|
||||
if !matches!(state, ScannerPauseBacklogReplicaState::Valid(_)) {
|
||||
return Err("scanner pause backlog retirement found an invalid or unsupported native record".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_invalid_record(
|
||||
"scanner pause backlog retirement found an invalid or unsupported native record",
|
||||
));
|
||||
}
|
||||
has_source |= replica.pool_index == source_pool_index;
|
||||
state
|
||||
}
|
||||
None => ScannerPauseBacklogReplicaState::Missing,
|
||||
_ => return Err("scanner pause backlog retirement has an oversized replica".to_string()),
|
||||
_ => {
|
||||
return Err(scanner_pause_backlog_retirement_invalid_record(
|
||||
"scanner pause backlog retirement has an oversized replica",
|
||||
));
|
||||
}
|
||||
};
|
||||
decoded.push(ScannerPauseBacklogReplica {
|
||||
id,
|
||||
@@ -1223,7 +1312,9 @@ fn decode_scanner_pause_backlog_retirement_replicas(
|
||||
});
|
||||
}
|
||||
if !has_source {
|
||||
return Err("scanner pause backlog retirement has no native source record".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_invalid_record(
|
||||
"scanner pause backlog retirement has no native source record",
|
||||
));
|
||||
}
|
||||
Ok(decoded)
|
||||
}
|
||||
@@ -1231,12 +1322,14 @@ fn decode_scanner_pause_backlog_retirement_replicas(
|
||||
fn verify_scanner_pause_backlog_retirement(
|
||||
source_pool_index: usize,
|
||||
replicas: &[ScannerPauseBacklogRetirementReplica],
|
||||
) -> Result<(), String> {
|
||||
) -> Result<(), ScannerPauseBacklogRetirementError> {
|
||||
let mut decoded = decode_scanner_pause_backlog_retirement_replicas(source_pool_index, replicas)?;
|
||||
if decoded.iter().any(|replica| {
|
||||
replica.id.pool_index != source_pool_index && matches!(replica.state, ScannerPauseBacklogReplicaState::Missing)
|
||||
}) {
|
||||
return Err("scanner pause backlog retirement has a missing surviving replica".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_authority_conflict(
|
||||
"scanner pause backlog retirement has a missing surviving replica",
|
||||
));
|
||||
}
|
||||
// Earlier entries may already have removed a source sibling after a
|
||||
// successful handoff. It contributes no stored authority to final cleanup.
|
||||
@@ -1246,15 +1339,17 @@ fn verify_scanner_pause_backlog_retirement(
|
||||
.filter(|replica| replica.id.pool_index != source_pool_index)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let selected = select_scanner_pause_backlog_replicas(surviving)?;
|
||||
let selected = select_scanner_pause_backlog_replicas(surviving).map_err(scanner_pause_backlog_retirement_selection_error)?;
|
||||
if !selected.durable || !selected.stable_matches_ledger {
|
||||
return Err(
|
||||
"scanner pause backlog retirement requires stable authority on the complete surviving membership".to_string(),
|
||||
);
|
||||
return Err(scanner_pause_backlog_retirement_authority_conflict(
|
||||
"scanner pause backlog retirement requires stable authority on the complete surviving membership",
|
||||
));
|
||||
}
|
||||
let before = select_scanner_pause_backlog_replicas(decoded)?;
|
||||
let before = select_scanner_pause_backlog_replicas(decoded).map_err(scanner_pause_backlog_retirement_selection_error)?;
|
||||
if !before.durable || before.ledger != selected.ledger {
|
||||
return Err("scanner pause backlog retirement would change the native ledger authority".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_authority_conflict(
|
||||
"scanner pause backlog retirement would change the native ledger authority",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1262,7 +1357,7 @@ fn verify_scanner_pause_backlog_retirement(
|
||||
fn plan_scanner_pause_backlog_retirement(
|
||||
source_pool_index: usize,
|
||||
replicas: &[ScannerPauseBacklogRetirementReplica],
|
||||
) -> Result<Option<ScannerPauseBacklogRetirementPlan>, String> {
|
||||
) -> Result<Option<ScannerPauseBacklogRetirementPlan>, ScannerPauseBacklogRetirementError> {
|
||||
// Missing entries stay in the native selection. Omitting them could turn
|
||||
// an incomplete old cohort into a fabricated stable consensus.
|
||||
let decoded = decode_scanner_pause_backlog_retirement_replicas(source_pool_index, replicas)?;
|
||||
@@ -1272,12 +1367,16 @@ fn plan_scanner_pause_backlog_retirement(
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
if surviving.is_empty() {
|
||||
return Err("scanner pause backlog retirement has no surviving membership".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_authority_conflict(
|
||||
"scanner pause backlog retirement has no surviving membership",
|
||||
));
|
||||
}
|
||||
let all_ids = scanner_pause_backlog_replica_ids(&decoded);
|
||||
let surviving_ids = scanner_pause_backlog_replica_ids(&surviving);
|
||||
let full_commit = select_scanner_pause_backlog_commit(&decoded, &all_ids)?;
|
||||
let surviving_commit = select_scanner_pause_backlog_commit(&surviving, &surviving_ids)?;
|
||||
let full_commit =
|
||||
select_scanner_pause_backlog_commit(&decoded, &all_ids).map_err(scanner_pause_backlog_retirement_commit_error)?;
|
||||
let surviving_commit =
|
||||
select_scanner_pause_backlog_commit(&surviving, &surviving_ids).map_err(scanner_pause_backlog_retirement_commit_error)?;
|
||||
let selected = select_scanner_pause_backlog_replicas(surviving.clone());
|
||||
let full_selection = select_scanner_pause_backlog_replicas(decoded.clone());
|
||||
if full_commit.is_none()
|
||||
@@ -1293,15 +1392,20 @@ fn plan_scanner_pause_backlog_retirement(
|
||||
if record.committed.as_ref().is_some_and(|committed|
|
||||
committed.replicas.iter().any(|id| !all_ids.contains(id))))
|
||||
}) {
|
||||
return Err("scanner pause backlog bootstrap has an unread native commit member".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_invalid_record(
|
||||
"scanner pause backlog bootstrap has an unread native commit member",
|
||||
));
|
||||
}
|
||||
let ledger = claim_scanner_pause_backlog_writer(&all.ledger, unix_now())?;
|
||||
let ledger = claim_scanner_pause_backlog_writer(&all.ledger, unix_now())
|
||||
.map_err(scanner_pause_backlog_retirement_invalid_record)?;
|
||||
let committed = commit_scanner_pause_backlog_record(None, &ledger, &surviving_ids);
|
||||
let stable = stable_scanner_pause_backlog_record(&ledger, committed.committed.as_ref(), &surviving_ids);
|
||||
return Ok(Some(ScannerPauseBacklogRetirementPlan {
|
||||
seed_record: None,
|
||||
commit_record: encode_scanner_pause_backlog_record(&committed)?,
|
||||
stable_record: encode_scanner_pause_backlog_record(&stable)?,
|
||||
commit_record: encode_scanner_pause_backlog_record(&committed)
|
||||
.map_err(scanner_pause_backlog_retirement_invalid_record)?,
|
||||
stable_record: encode_scanner_pause_backlog_record(&stable)
|
||||
.map_err(scanner_pause_backlog_retirement_invalid_record)?,
|
||||
}));
|
||||
}
|
||||
let ledger = if let Some(committed) = &full_commit {
|
||||
@@ -1309,7 +1413,9 @@ fn plan_scanner_pause_backlog_retirement(
|
||||
.as_ref()
|
||||
.is_some_and(|current| current.ledger != committed.ledger)
|
||||
{
|
||||
return Err("scanner pause backlog retirement found a different surviving native commit".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_authority_conflict(
|
||||
"scanner pause backlog retirement found a different surviving native commit",
|
||||
));
|
||||
}
|
||||
if let Ok(current) = &selected
|
||||
&& current.durable
|
||||
@@ -1324,20 +1430,28 @@ fn plan_scanner_pause_backlog_retirement(
|
||||
if record.committed.as_ref() == Some(committed))
|
||||
});
|
||||
if !acknowledged_full_commit {
|
||||
return Err("scanner pause backlog retirement would replace surviving stable authority".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_authority_conflict(
|
||||
"scanner pause backlog retirement would replace surviving stable authority",
|
||||
));
|
||||
}
|
||||
}
|
||||
// Use the same native selector that validates the full commit, rather
|
||||
// than inferring authority from source epoch or physical object time.
|
||||
full_selection?.ledger
|
||||
full_selection
|
||||
.map_err(scanner_pause_backlog_retirement_selection_error)?
|
||||
.ledger
|
||||
} else {
|
||||
// An interrupted cohort switch can invalidate the old full commit
|
||||
// while all survivors still have its stable rollback point. Source
|
||||
// stable fields need not match, but any valid conflicting commit above
|
||||
// must have been rejected before this recovery path.
|
||||
let current = selected.as_ref().map_err(|err| err.clone())?;
|
||||
let current = selected
|
||||
.as_ref()
|
||||
.map_err(|err| scanner_pause_backlog_retirement_selection_error(err.clone()))?;
|
||||
if !current.durable {
|
||||
return Err("scanner pause backlog retirement has no proven native authority".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_authority_conflict(
|
||||
"scanner pause backlog retirement has no proven native authority",
|
||||
));
|
||||
}
|
||||
if decoded
|
||||
.iter()
|
||||
@@ -1348,7 +1462,9 @@ fn plan_scanner_pause_backlog_retirement(
|
||||
&& record.committed.as_ref().is_none_or(|committed| committed.ledger != current.ledger))
|
||||
})
|
||||
{
|
||||
return Err("scanner pause backlog retirement has unrelated source stable authority".to_string());
|
||||
return Err(scanner_pause_backlog_retirement_authority_conflict(
|
||||
"scanner pause backlog retirement has unrelated source stable authority",
|
||||
));
|
||||
}
|
||||
current.ledger.clone()
|
||||
};
|
||||
@@ -1370,19 +1486,23 @@ fn plan_scanner_pause_backlog_retirement(
|
||||
.as_ref()
|
||||
.filter(|committed| committed.ledger == ledger)
|
||||
.or_else(|| surviving_commit.as_ref().filter(|committed| committed.ledger == ledger))
|
||||
.ok_or_else(|| "scanner pause backlog retirement cannot seed without a native commit proof".to_string())?;
|
||||
Some(encode_scanner_pause_backlog_record(&stable_scanner_pause_backlog_record(
|
||||
&ledger,
|
||||
Some(authority),
|
||||
&surviving_ids,
|
||||
))?)
|
||||
.ok_or_else(|| {
|
||||
scanner_pause_backlog_retirement_authority_conflict(
|
||||
"scanner pause backlog retirement cannot seed without a native commit proof",
|
||||
)
|
||||
})?;
|
||||
Some(
|
||||
encode_scanner_pause_backlog_record(&stable_scanner_pause_backlog_record(&ledger, Some(authority), &surviving_ids))
|
||||
.map_err(scanner_pause_backlog_retirement_invalid_record)?,
|
||||
)
|
||||
};
|
||||
let committed = commit_scanner_pause_backlog_record(Some(&ledger), &ledger, &surviving_ids);
|
||||
let stable = stable_scanner_pause_backlog_record(&ledger, committed.committed.as_ref(), &surviving_ids);
|
||||
Ok(Some(ScannerPauseBacklogRetirementPlan {
|
||||
seed_record,
|
||||
commit_record: encode_scanner_pause_backlog_record(&committed)?,
|
||||
stable_record: encode_scanner_pause_backlog_record(&stable)?,
|
||||
commit_record: encode_scanner_pause_backlog_record(&committed)
|
||||
.map_err(scanner_pause_backlog_retirement_invalid_record)?,
|
||||
stable_record: encode_scanner_pause_backlog_record(&stable).map_err(scanner_pause_backlog_retirement_invalid_record)?,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -1432,7 +1552,7 @@ where
|
||||
return Err("scanner pause backlog has no surviving storage replicas".to_string());
|
||||
}
|
||||
let replicas = join_all(writable.into_iter().map(read_scanner_pause_backlog_replica)).await;
|
||||
select_scanner_pause_backlog_replicas(replicas)
|
||||
select_scanner_pause_backlog_replicas(replicas).map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
async fn write_scanner_pause_backlog_record<S>(
|
||||
@@ -3010,7 +3130,10 @@ mod tests {
|
||||
];
|
||||
let blocked = verify_scanner_pause_backlog_retirement(0, &replicas)
|
||||
.expect_err("a complete commit still needs its native stabilization barrier");
|
||||
assert!(blocked.contains("stable authority"), "{blocked}");
|
||||
assert!(
|
||||
matches!(blocked, ScannerPauseBacklogRetirementError::AuthorityConflict { .. }),
|
||||
"a commit without the native stabilization barrier is an authority conflict: {blocked}"
|
||||
);
|
||||
|
||||
let stable = replica_record_for_members(&new, &new, &targets);
|
||||
replicas[1] = retirement_replica(targets[0], &stable);
|
||||
@@ -3032,7 +3155,10 @@ mod tests {
|
||||
new.claim_writer(100).unwrap();
|
||||
prepare_scanner_pause_backlog_persist(&mut new, 100).unwrap();
|
||||
let stable = replica_record_for_members(&new, &new, &targets);
|
||||
for source_sets in [2, 3] {
|
||||
// Equal-size competing source proofs are a transient handoff conflict
|
||||
// the native writer can still resolve. A strictly larger source proof
|
||||
// is durable authority that must never be discarded.
|
||||
for (source_sets, authority_conflict) in [(2, false), (3, true)] {
|
||||
let sources = (0..source_sets).map(|set| replica_id(0, set)).collect::<Vec<_>>();
|
||||
let source_record = replica_record_for_members(&old, &old, &sources);
|
||||
let mut replicas = sources
|
||||
@@ -3040,8 +3166,13 @@ mod tests {
|
||||
.map(|id| retirement_replica(*id, &source_record))
|
||||
.collect::<Vec<_>>();
|
||||
replicas.extend(targets.iter().map(|id| retirement_replica(*id, &stable)));
|
||||
verify_scanner_pause_backlog_retirement(0, &replicas)
|
||||
let error = verify_scanner_pause_backlog_retirement(0, &replicas)
|
||||
.expect_err("all source sets must be consulted before discarding a competing or larger native proof");
|
||||
match (authority_conflict, &error) {
|
||||
(true, ScannerPauseBacklogRetirementError::AuthorityConflict { .. })
|
||||
| (false, ScannerPauseBacklogRetirementError::HandoffConflict { .. }) => {}
|
||||
_ => panic!("competing source proofs must keep their typed classification: {error}"),
|
||||
}
|
||||
assert!(plan_scanner_pause_backlog_retirement(0, &replicas).is_err());
|
||||
}
|
||||
}
|
||||
@@ -3068,7 +3199,10 @@ mod tests {
|
||||
let error = plan_scanner_pause_backlog_retirement(0, &replicas)
|
||||
.err()
|
||||
.expect("bootstrap must not turn an unread old cohort into a missing member");
|
||||
assert!(error.contains("unread"), "{error}");
|
||||
let ScannerPauseBacklogRetirementError::InvalidRecord { reason } = &error else {
|
||||
panic!("an unread old cohort member is an invalid record, not a missing member: {error}");
|
||||
};
|
||||
assert!(reason.contains("unread"), "{reason}");
|
||||
|
||||
let new = durable_ledger(100);
|
||||
let source_record = replica_record_for_members(&old, &old, &[source]);
|
||||
@@ -3079,7 +3213,10 @@ mod tests {
|
||||
let error = plan_scanner_pause_backlog_retirement(0, &replicas)
|
||||
.err()
|
||||
.expect("an independent source-only proof must not overwrite stable surviving authority");
|
||||
assert!(error.contains("stable authority"), "{error}");
|
||||
assert!(
|
||||
matches!(error, ScannerPauseBacklogRetirementError::AuthorityConflict { .. }),
|
||||
"an independent source-only proof must be an authority conflict: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
|
||||
@@ -91,6 +91,9 @@ const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000;
|
||||
const SCANNER_LIST_PATH_RAW_STALL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const SCANNER_ENTRY_PROGRESS_BATCH: u64 = 32;
|
||||
const SCANNER_ENTRY_PROGRESS_INTERVAL: Duration = Duration::from_secs(30);
|
||||
const SCANNER_CHECKPOINT_OBJECT_INTERVAL: u64 = 1024;
|
||||
const SCANNER_CHECKPOINT_MIN_INTERVAL: Duration = Duration::from_secs(5);
|
||||
const SCANNER_CHECKPOINT_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const SCANNER_RAW_ENUMERATION_PAGE_ENTRY_LIMIT: usize = 128;
|
||||
const SCANNER_RAW_ENUMERATION_PAGE_BUILD_BUDGET: usize = 1;
|
||||
// Erasure data directories contain direct part.N files; keep namespace probes bounded.
|
||||
@@ -723,7 +726,11 @@ pub struct FolderScanner {
|
||||
disks_quorum: usize,
|
||||
|
||||
updates: Option<mpsc::Sender<DataUsageEntry>>,
|
||||
checkpoint_tx: Option<mpsc::Sender<DataUsageCache>>,
|
||||
last_update: SystemTime,
|
||||
checkpoint_objects: u64,
|
||||
last_checkpoint_objects: u64,
|
||||
last_checkpoint_at: Instant,
|
||||
|
||||
update_current_path: UpdateCurrentPathFn,
|
||||
|
||||
@@ -1204,6 +1211,58 @@ impl FolderScanner {
|
||||
(progress.cursor(), progress.page_index())
|
||||
}
|
||||
|
||||
fn peek_raw_enumeration_resume_state(&self) -> (Option<DataUsageRawEnumerationCursor>, Option<RawEnumerationPageIndex>) {
|
||||
self.raw_enumeration_progress
|
||||
.iter()
|
||||
.max_by_key(|progress| progress.checkpointable_entry_count())
|
||||
.map_or((None, None), |progress| (progress.cursor(), progress.page_index()))
|
||||
}
|
||||
|
||||
fn maybe_send_checkpoint(&mut self) {
|
||||
let Some(checkpoint_tx) = self.checkpoint_tx.as_ref() else { return };
|
||||
let elapsed = self.last_checkpoint_at.elapsed();
|
||||
if self.new_cache.info.scan_progress.is_none()
|
||||
|| elapsed < SCANNER_CHECKPOINT_MIN_INTERVAL
|
||||
|| (self.checkpoint_objects.saturating_sub(self.last_checkpoint_objects) < SCANNER_CHECKPOINT_OBJECT_INTERVAL
|
||||
&& elapsed < SCANNER_CHECKPOINT_INTERVAL)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if self.new_cache.root().is_none() && self.raw_enumeration_progress.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut snapshot = self.new_cache.clone();
|
||||
snapshot.info.last_update = Some(SystemTime::now());
|
||||
snapshot.info.snapshot_complete = false;
|
||||
let (cursor, page_index) = self.peek_raw_enumeration_resume_state();
|
||||
if cursor.is_some() || page_index.is_some() {
|
||||
snapshot.info.scan_raw_enumeration_cursor = cursor;
|
||||
snapshot.info.scan_raw_enumeration_page_index = page_index;
|
||||
snapshot.info.scan_resume_after = None;
|
||||
snapshot.info.scan_checkpoint = None;
|
||||
snapshot.info.scan_coverage_receipt = None;
|
||||
} else if snapshot.seal_scan_frontier(self.coverage_frontier.as_deref()).is_err() {
|
||||
return;
|
||||
}
|
||||
if snapshot.root().is_none()
|
||||
&& snapshot.info.scan_raw_enumeration_cursor.is_none()
|
||||
&& snapshot.info.scan_raw_enumeration_page_index.is_none()
|
||||
{
|
||||
return;
|
||||
}
|
||||
if snapshot.info.scan_raw_enumeration_cursor.is_none()
|
||||
&& snapshot.info.scan_raw_enumeration_page_index.is_none()
|
||||
&& snapshot.validated_scan_frontier().is_none()
|
||||
{
|
||||
return;
|
||||
}
|
||||
if checkpoint_tx.try_send(snapshot).is_ok() {
|
||||
self.last_checkpoint_objects = self.checkpoint_objects;
|
||||
self.last_checkpoint_at = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
fn carry_forward_old_children(&mut self, parent_hash: &DataUsageHash, entry: &mut DataUsageEntry) {
|
||||
if entry.compacted {
|
||||
// Compacted entries store child totals directly; child links would be flattened twice.
|
||||
@@ -1592,6 +1651,7 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
self.record_raw_enumeration_entry(&folder.name, &file_name);
|
||||
self.maybe_send_checkpoint();
|
||||
let is_storage_format_entry = file_name == STORAGE_FORMAT_FILE;
|
||||
|
||||
let file_path = entry.path().to_string_lossy().to_string();
|
||||
@@ -1887,6 +1947,8 @@ impl FolderScanner {
|
||||
into.objects += 1;
|
||||
object_count += 1;
|
||||
self.budget.record_object_scanned();
|
||||
self.checkpoint_objects = self.checkpoint_objects.saturating_add(1);
|
||||
self.maybe_send_checkpoint();
|
||||
|
||||
timer.sleep().await;
|
||||
|
||||
@@ -2180,6 +2242,7 @@ impl FolderScanner {
|
||||
|
||||
into.add_child(&h);
|
||||
self.record_completed_child(&folder_item.name, dst.failed_objects == 0);
|
||||
self.maybe_send_checkpoint();
|
||||
// We scanned a folder, optionally send update.
|
||||
self.update_cache.delete_recursive(&h);
|
||||
self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent);
|
||||
@@ -2578,6 +2641,7 @@ impl FolderScanner {
|
||||
self.update_cache.delete_recursive(&h);
|
||||
self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent);
|
||||
self.send_update().await;
|
||||
self.maybe_send_checkpoint();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2668,7 +2732,7 @@ pub async fn scan_data_folder(
|
||||
scan_mode: HealScanMode,
|
||||
sleeper: DynamicSleeper,
|
||||
) -> Result<DataUsageCache, ScannerError> {
|
||||
scan_data_folder_scoped(ctx, budget, disks, local_disk, cache, updates, scan_mode, sleeper, None).await
|
||||
scan_data_folder_scoped(ctx, budget, disks, local_disk, cache, updates, scan_mode, sleeper, None, None).await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -2682,6 +2746,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
scan_mode: HealScanMode,
|
||||
sleeper: DynamicSleeper,
|
||||
prefix_scan_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
checkpoint_tx: Option<mpsc::Sender<DataUsageCache>>,
|
||||
) -> Result<DataUsageCache, ScannerError> {
|
||||
use crate::data_usage_define::DATA_USAGE_ROOT;
|
||||
|
||||
@@ -2740,7 +2805,11 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
disks,
|
||||
disks_quorum,
|
||||
updates,
|
||||
checkpoint_tx,
|
||||
last_update: SystemTime::UNIX_EPOCH,
|
||||
checkpoint_objects: 0,
|
||||
last_checkpoint_objects: 0,
|
||||
last_checkpoint_at: Instant::now(),
|
||||
update_current_path,
|
||||
budget: budget.clone(),
|
||||
skip_heal,
|
||||
|
||||
@@ -339,7 +339,11 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
disks: Vec::new(),
|
||||
disks_quorum: 0,
|
||||
updates: None,
|
||||
checkpoint_tx: None,
|
||||
last_update: SystemTime::UNIX_EPOCH,
|
||||
checkpoint_objects: 0,
|
||||
last_checkpoint_objects: 0,
|
||||
last_checkpoint_at: Instant::now(),
|
||||
update_current_path,
|
||||
budget: ScannerCycleBudget::new(&CancellationToken::new(), Default::default()),
|
||||
skip_heal: Arc::new(AtomicBool::new(false)),
|
||||
@@ -366,6 +370,55 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
(scanner, temp_dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_checkpoint_emits_at_object_threshold_without_wall_clock_wait() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel(1);
|
||||
scanner.checkpoint_tx = Some(checkpoint_tx);
|
||||
scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL;
|
||||
scanner.last_checkpoint_at = Instant::now()
|
||||
.checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL)
|
||||
.expect("test instant subtraction");
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.scan_progress = Some(crate::DataUsageScanProgress {
|
||||
started_plan: crate::DataUsageScanPlanDigest([1; 32]),
|
||||
requested_plan: crate::DataUsageScanPlanDigest([1; 32]),
|
||||
});
|
||||
scanner.new_cache.info.source = Some(crate::DataUsageCacheSource::new(0, 0));
|
||||
scanner.new_cache.info.scan_identity = Some(crate::DataUsageScanIdentity {
|
||||
version: 1,
|
||||
bucket_incarnation: Uuid::from_u128(1),
|
||||
set_layout: crate::DataUsageScanPlanDigest([2; 32]),
|
||||
publication_epoch: 1,
|
||||
tier_registry_generation: 0,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
});
|
||||
scanner.new_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.new_cache.replace("bucket/a", "bucket", DataUsageEntry::default());
|
||||
scanner.coverage_frontier = Some("bucket/a".to_string());
|
||||
scanner.new_cache.info.scan_checkpoint = Some(crate::DataUsageScanCheckpoint::new(
|
||||
"bucket/a".to_string(),
|
||||
crate::DataUsageScanCheckpointReason::Objects,
|
||||
));
|
||||
scanner.new_cache.info.scan_resume_after = Some("bucket/a".to_string());
|
||||
scanner
|
||||
.new_cache
|
||||
.seal_scan_frontier(Some("bucket/a"))
|
||||
.expect("fixture frontier");
|
||||
assert_eq!(scanner.new_cache.validated_scan_frontier(), Some("bucket/a"));
|
||||
|
||||
scanner.maybe_send_checkpoint();
|
||||
|
||||
let checkpoint = checkpoint_rx.try_recv().expect("object threshold emits a bounded checkpoint");
|
||||
assert_eq!(checkpoint.info.name, "bucket");
|
||||
assert!(!checkpoint.info.snapshot_complete);
|
||||
assert_eq!(scanner.last_checkpoint_objects, SCANNER_CHECKPOINT_OBJECT_INTERVAL);
|
||||
assert!(checkpoint_rx.try_recv().is_err(), "checkpoint queue remains bounded");
|
||||
tokio::fs::remove_dir_all(temp_dir)
|
||||
.await
|
||||
.expect("remove test scanner directory");
|
||||
}
|
||||
|
||||
struct TestGuard {
|
||||
temp_dir: Option<std::path::PathBuf>,
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ fn scan_options(scan_mode: HealScanMode) -> ScannerDiskScanOptions {
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -382,14 +383,31 @@ fn checkpoint_fixture_identity_changes_and_future_state_fail_closed() {
|
||||
assert!(next.info.scan_checkpoint.is_none());
|
||||
assert!(!next.info.snapshot_complete);
|
||||
}
|
||||
for (source, epoch) in [(crate::DataUsageCacheSource::new(1, 0), 7), (SOURCE, 8)] {
|
||||
let mut next = cache.clone();
|
||||
assert_eq!(
|
||||
next.prepare_bucket_checkpoint("bucket", 11, epoch, source, PLAN, identity),
|
||||
crate::DataUsageCachePrepareOutcome::Reset
|
||||
);
|
||||
assert!(next.cache.is_empty());
|
||||
}
|
||||
let mut source_mismatch = cache.clone();
|
||||
assert_eq!(
|
||||
source_mismatch.prepare_bucket_checkpoint("bucket", 11, 7, crate::DataUsageCacheSource::new(1, 0), PLAN, identity,),
|
||||
crate::DataUsageCachePrepareOutcome::Reset
|
||||
);
|
||||
assert!(source_mismatch.cache.is_empty());
|
||||
let mut handed_off = cache.clone();
|
||||
assert_eq!(
|
||||
handed_off.prepare_bucket_checkpoint("bucket", 11, 8, SOURCE, PLAN, identity),
|
||||
crate::DataUsageCachePrepareOutcome::Reused
|
||||
);
|
||||
assert_eq!(handed_off.info.leader_epoch, 8);
|
||||
assert_eq!(handed_off.validated_scan_frontier(), Some("bucket/static"));
|
||||
assert_eq!(retained(&handed_off), 3);
|
||||
|
||||
let mut unverified = cache.clone();
|
||||
unverified.info.scan_resume_after = None;
|
||||
unverified.info.scan_checkpoint = None;
|
||||
unverified.info.scan_coverage_receipt = None;
|
||||
assert_eq!(
|
||||
unverified.prepare_bucket_checkpoint("bucket", 11, 8, SOURCE, PLAN, identity),
|
||||
crate::DataUsageCachePrepareOutcome::Reset,
|
||||
"a cross-epoch cache without a durable frontier proof must rebuild"
|
||||
);
|
||||
assert!(unverified.cache.is_empty());
|
||||
for (cycle, epoch, expected) in [
|
||||
(10, 7, crate::DataUsageCachePrepareOutcome::RejectedNewerCycle),
|
||||
(11, 6, crate::DataUsageCachePrepareOutcome::RejectedNewerLeader),
|
||||
@@ -870,6 +888,7 @@ async fn run_checkpoint_fixture(change_digest: bool) {
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -951,6 +970,7 @@ async fn run_checkpoint_fixture(change_digest: bool) {
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -1293,6 +1293,7 @@ pub trait ScannerIOCache: Send + Sync + Debug + 'static {
|
||||
pub struct ScannerDiskScanOptions {
|
||||
pub scan_mode: HealScanMode,
|
||||
pub prefix_scan_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
pub checkpoint_tx: Option<mpsc::Sender<DataUsageCache>>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -1608,8 +1609,8 @@ use dirty_usage::*;
|
||||
use guards::*;
|
||||
|
||||
pub(crate) use cache::{
|
||||
DataUsageCacheReuseOptions, DataUsageCacheScanState, acquire_scanner_cache_locks,
|
||||
current_cache_root_or_prepare_with_generation,
|
||||
DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerCheckpointPersistResult, acquire_scanner_cache_locks,
|
||||
current_cache_root_or_prepare_with_generation, persist_scanner_checkpoint,
|
||||
};
|
||||
pub use dirty_usage::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageMutationObserver,
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
/// scanner cache locks and the cache snapshot persist/publish path.
|
||||
use super::*;
|
||||
use crate::{ScannerConfigObjectDelete, ScannerObjectIO};
|
||||
|
||||
pub(crate) fn scanner_cache_lock_resource(cache_name: &str, source: DataUsageCacheSource) -> String {
|
||||
let lock_name = format!("{SCANNER_CACHE_LOCK_SUFFIX}.pool-{}.set-{}", source.pool_index, source.set_index);
|
||||
@@ -23,6 +24,69 @@ pub(crate) fn scanner_cache_lock_timeout() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5))
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum ScannerCheckpointPersistResult {
|
||||
Saved,
|
||||
FenceChanged,
|
||||
Failed(StorageError),
|
||||
}
|
||||
|
||||
/// Persist one bounded checkpoint and refresh its CAS revisions.
|
||||
///
|
||||
/// Local and remote workers share the same publication/leader fencing and
|
||||
/// revision-refresh contract; only their lock/cancellation handling remains
|
||||
/// at the caller because those guards have different concrete types.
|
||||
pub(crate) async fn persist_scanner_checkpoint<S>(
|
||||
store: Arc<S>,
|
||||
cache_name: &str,
|
||||
checkpoint: &DataUsageCache,
|
||||
revisions: &mut DataUsageCacheRevisions,
|
||||
expected_publication_epoch: u64,
|
||||
cycle: u64,
|
||||
leader_epoch: u64,
|
||||
) -> ScannerCheckpointPersistResult
|
||||
where
|
||||
S: ScannerObjectIO + ScannerConfigObjectDelete,
|
||||
{
|
||||
if crate::remote_scanner::validate_remote_scanner_request_fence_with_store(cycle, leader_epoch, store.clone())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return ScannerCheckpointPersistResult::FenceChanged;
|
||||
}
|
||||
if scanner_publication_admission_for_epoch(store.clone(), expected_publication_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return ScannerCheckpointPersistResult::FenceChanged;
|
||||
}
|
||||
|
||||
if let Err(error) = checkpoint
|
||||
.save_with_revisions_for_epoch(store.clone(), cache_name, revisions, expected_publication_epoch)
|
||||
.await
|
||||
{
|
||||
return ScannerCheckpointPersistResult::Failed(error);
|
||||
}
|
||||
|
||||
if crate::remote_scanner::validate_remote_scanner_request_fence_with_store(cycle, leader_epoch, store.clone())
|
||||
.await
|
||||
.is_err()
|
||||
|| scanner_publication_admission_for_epoch(store.clone(), expected_publication_epoch)
|
||||
.await
|
||||
.is_none()
|
||||
{
|
||||
return ScannerCheckpointPersistResult::FenceChanged;
|
||||
}
|
||||
|
||||
match DataUsageCache::read_revisions(store, cache_name).await {
|
||||
Ok(next_revisions) => {
|
||||
*revisions = next_revisions;
|
||||
ScannerCheckpointPersistResult::Saved
|
||||
}
|
||||
Err(error) => ScannerCheckpointPersistResult::Failed(error),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScannerCacheLockGuards {
|
||||
scoped: NamespaceLockGuard,
|
||||
|
||||
@@ -981,7 +981,7 @@ impl ScannerIOCache for SetDisks {
|
||||
};
|
||||
|
||||
let mut cache = DataUsageCache::default();
|
||||
let revisions = match cache.load_with_revisions(store_clone_clone.clone(), &cache_name).await {
|
||||
let mut revisions = match cache.load_with_revisions(store_clone_clone.clone(), &cache_name).await {
|
||||
Ok(revisions) => revisions,
|
||||
Err(e) => {
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
@@ -1145,6 +1145,7 @@ impl ScannerIOCache for SetDisks {
|
||||
);
|
||||
|
||||
let before = cache.info.last_update;
|
||||
let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::<DataUsageCache>(1);
|
||||
|
||||
let scan_ctx = ctx_clone.child_token();
|
||||
let scan = disk.clone().nsscanner_disk(
|
||||
@@ -1156,11 +1157,13 @@ impl ScannerIOCache for SetDisks {
|
||||
ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope,
|
||||
checkpoint_tx: Some(checkpoint_tx),
|
||||
},
|
||||
);
|
||||
tokio::pin!(scan);
|
||||
let mut lock_watch = tokio::time::interval(SCANNER_CACHE_LOCK_POLL_INTERVAL);
|
||||
lock_watch.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
let mut checkpoint_channel_closed = false;
|
||||
let scan_result = loop {
|
||||
tokio::select! {
|
||||
result = &mut scan => break result,
|
||||
@@ -1171,6 +1174,60 @@ impl ScannerIOCache for SetDisks {
|
||||
break Err(Error::other("scanner bucket cache lock was lost during bucket scan"));
|
||||
}
|
||||
}
|
||||
checkpoint = checkpoint_rx.recv(), if !checkpoint_channel_closed => {
|
||||
let Some(checkpoint) = checkpoint else {
|
||||
checkpoint_channel_closed = true;
|
||||
continue;
|
||||
};
|
||||
if cache_guard.is_lock_lost() {
|
||||
scan_ctx.cancel();
|
||||
await_scanner_disk_shutdown(scan.as_mut()).await;
|
||||
break Err(Error::other("scanner bucket cache lock was lost before checkpoint save"));
|
||||
}
|
||||
if ctx_clone.is_cancelled() {
|
||||
scan_ctx.cancel();
|
||||
await_scanner_disk_shutdown(scan.as_mut()).await;
|
||||
break Err(Error::other("scanner leader fence changed before checkpoint save"));
|
||||
}
|
||||
match persist_scanner_checkpoint(
|
||||
store_clone_clone.clone(),
|
||||
cache_name.as_str(),
|
||||
&checkpoint,
|
||||
&mut revisions,
|
||||
expected_publication_epoch_clone,
|
||||
want_cycle,
|
||||
leader_epoch,
|
||||
)
|
||||
.await
|
||||
{
|
||||
ScannerCheckpointPersistResult::Saved => {
|
||||
if cache_guard.is_lock_lost() || ctx_clone.is_cancelled() {
|
||||
scan_ctx.cancel();
|
||||
await_scanner_disk_shutdown(scan.as_mut()).await;
|
||||
break Err(Error::other("scanner bucket cache fence changed after checkpoint save"));
|
||||
}
|
||||
}
|
||||
ScannerCheckpointPersistResult::FenceChanged => {
|
||||
scan_ctx.cancel();
|
||||
await_scanner_disk_shutdown(scan.as_mut()).await;
|
||||
break Err(Error::other("scanner bucket cache fence changed during checkpoint save"));
|
||||
}
|
||||
ScannerCheckpointPersistResult::Failed(error) => {
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
bucket = %bucket.name,
|
||||
cache_name = %cache_name,
|
||||
state = "periodic_checkpoint_save_failed",
|
||||
error = %error,
|
||||
"Scanner periodic checkpoint save failed"
|
||||
);
|
||||
checkpoint_channel_closed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Some(expected) = checkpoint_identity
|
||||
|
||||
@@ -159,6 +159,7 @@ impl ScannerIODisk for Disk {
|
||||
let ScannerDiskScanOptions {
|
||||
scan_mode,
|
||||
prefix_scan_scope,
|
||||
checkpoint_tx,
|
||||
} = options;
|
||||
let done_drive = Metrics::time(Metric::ScanBucketDrive);
|
||||
let drive_start = std::time::Instant::now();
|
||||
@@ -224,6 +225,7 @@ impl ScannerIODisk for Disk {
|
||||
scan_mode,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
prefix_scan_scope,
|
||||
checkpoint_tx,
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
@@ -29,8 +29,8 @@ pub(crate) use s3s::dto::{
|
||||
pub(crate) use s3s::dto::{ExpirationStatus as EcstoreExpirationStatus, LifecycleRule as EcstoreLifecycleRule};
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::{
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementReplica,
|
||||
register_scanner_pause_backlog_retirement_planner,
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan,
|
||||
ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::{NativeScannerPauseBacklogWriteFault, SourceCleanupDeleteBarrier};
|
||||
@@ -143,8 +143,8 @@ pub(crate) type EcstoreHealResultItem = <EcstoreStore as storage_contracts::Heal
|
||||
|
||||
pub(crate) mod owner {
|
||||
pub(crate) use super::{
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementReplica,
|
||||
register_scanner_pause_backlog_retirement_planner,
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementError, ScannerPauseBacklogRetirementPlan,
|
||||
ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{NativeScannerPauseBacklogWriteFault, SourceCleanupDeleteBarrier};
|
||||
|
||||
+45
-4
@@ -208,10 +208,10 @@ pub fn is_local_host(host: Host<&str>, port: u16, local_port: u16) -> std::io::R
|
||||
Host::Domain(domain) => {
|
||||
let ips = resolve_domain(domain)?.into_iter().collect::<Vec<_>>();
|
||||
|
||||
ips.iter().any(|ip| local_set.contains(ip))
|
||||
ips.iter().any(|ip| is_local_ip(&local_set, ip))
|
||||
}
|
||||
Host::Ipv4(ip) => local_set.contains(&IpAddr::V4(ip)),
|
||||
Host::Ipv6(ip) => local_set.contains(&IpAddr::V6(ip)),
|
||||
Host::Ipv4(ip) => is_local_ip(&local_set, &IpAddr::V4(ip)),
|
||||
Host::Ipv6(ip) => is_local_ip(&local_set, &IpAddr::V6(ip)),
|
||||
};
|
||||
|
||||
if port > 0 {
|
||||
@@ -221,6 +221,31 @@ pub fn is_local_host(host: Host<&str>, port: u16, local_port: u16) -> std::io::R
|
||||
Ok(is_local_host)
|
||||
}
|
||||
|
||||
/// Whether `ip` belongs to this host: it is either configured on a local
|
||||
/// interface or, where the platform routes the full loopback space, a loopback
|
||||
/// address.
|
||||
fn is_local_ip(local_set: &HashSet<IpAddr>, ip: &IpAddr) -> bool {
|
||||
if local_set.contains(ip) {
|
||||
return true;
|
||||
}
|
||||
is_loopback_local(ip)
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn is_loopback_local(ip: &IpAddr) -> bool {
|
||||
// Unix kernels route and bind the entire RFC 1122 loopback space
|
||||
// (127.0.0.0/8 plus ::1), so any loopback address is reachable on this
|
||||
// host even when only 127.0.0.1 is configured on `lo` (e.g. 127.0.5.9).
|
||||
ip.is_loopback()
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn is_loopback_local(_ip: &IpAddr) -> bool {
|
||||
// Windows only allows 127.0.0.1 by default, so loopback acceptance there
|
||||
// is limited to the interface-configured addresses already checked above.
|
||||
false
|
||||
}
|
||||
|
||||
fn get_custom_dns_resolver() -> Option<Arc<DynDnsResolver>> {
|
||||
match CUSTOM_DNS_RESOLVER.read() {
|
||||
Ok(guard) => guard.clone(),
|
||||
@@ -516,7 +541,16 @@ mod test {
|
||||
#[test]
|
||||
fn test_check_local_server_addr() {
|
||||
// Test valid local addresses
|
||||
let valid_cases = ["localhost:54321", "127.0.0.1:9000", "0.0.0.0:9000", "[::1]:8080", "::1:8080"];
|
||||
let valid_cases = [
|
||||
"localhost:54321",
|
||||
"127.0.0.1:9000",
|
||||
"0.0.0.0:9000",
|
||||
"[::1]:8080",
|
||||
"::1:8080",
|
||||
// Unix binds the whole 127.0.0.0/8 loopback range; Windows only allows 127.0.0.1.
|
||||
#[cfg(not(windows))]
|
||||
"127.0.5.9:9000",
|
||||
];
|
||||
|
||||
for addr in valid_cases {
|
||||
let result = check_local_server_addr(addr);
|
||||
@@ -557,6 +591,13 @@ mod test {
|
||||
let ipv4_loopback = Host::Ipv4(Ipv4Addr::new(127, 0, 0, 1));
|
||||
assert!(is_local_host(ipv4_loopback, 0, 0).unwrap());
|
||||
|
||||
// Any 127.0.0.0/8 address is loopback on Unix; Windows only allows 127.0.0.1.
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
let ipv4_loopback_octet = Host::Ipv4(Ipv4Addr::new(127, 0, 5, 9));
|
||||
assert!(is_local_host(ipv4_loopback_octet, 0, 0).unwrap());
|
||||
}
|
||||
|
||||
let ipv6_loopback = Host::Ipv6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1));
|
||||
assert!(is_local_host(ipv6_loopback, 0, 0).unwrap());
|
||||
|
||||
|
||||
@@ -36,3 +36,7 @@ drive ownership, snapshot freshness, counter queries, and rolling upgrades.
|
||||
|
||||
For optional shard commitments, see [Independent shard integrity rollout](operations/shard-integrity-rollout.md):
|
||||
activation, legacy repair results, multipart mode changes, and rollback limits.
|
||||
|
||||
For crates.io publication of workspace crates, see
|
||||
[Workspace Cargo Publish](operations/cargo-publish-workspace.md): dependency
|
||||
ordering, dry-run, publish, and failure handling.
|
||||
|
||||
@@ -43,6 +43,9 @@ an unknown or unsupported peer-health snapshot degrades readiness with
|
||||
- A blocked pool metadata writer degrades node and cluster-write readiness with
|
||||
`pool_meta_write_blocked`. Metadata save-gate inspection is bounded to 100 ms;
|
||||
contention reports `pool_metadata_check_timeout` without installing a block.
|
||||
Node readiness keeps the last confirmed writable observation for the normal
|
||||
readiness-cache TTL during a transient inspection timeout, while preserving a
|
||||
confirmed block and failing closed when no fresh observation exists.
|
||||
- The authenticated cluster snapshot extends its existing node-local metadata
|
||||
gate inspection with safe reason, failure phase, and original block time. It
|
||||
distinguishes timeout from a block and changes no admission or recovery
|
||||
@@ -76,7 +79,7 @@ For a healthy IAM and metadata writer in a four-node, one-drive-per-node EC 2+2
|
||||
| 1 | false | false | true | 503 / false |
|
||||
| All restored | true | true | true | 200 / true |
|
||||
|
||||
The node path reads local disk-handle health and reuses the same reachable-host observation as its lock dependency, including the existing `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS` cache. Only `Online` drives count; a reachable host with a `Returning` drive does not yet prove data I/O has recovered. It does not call cluster `storage_info`, local `disk_info`, or add disk-info RPCs. The entire storage inventory snapshot has a separate 100 ms wait budget; expiry reports `storage_readiness_check_timeout` and fails closed. Pool metadata inspection retains its own 100 ms budget. These observations are not an atomic cluster snapshot and do not bypass the existing lock-probe timing or cache policy.
|
||||
The node path reads local disk-handle health and reuses the same reachable-host observation as its lock dependency, including the existing `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS` cache. Only `Online` drives count; a reachable host with a `Returning` drive does not yet prove data I/O has recovered. It does not call cluster `storage_info`, local `disk_info`, or add disk-info RPCs. The entire storage inventory snapshot has a separate 100 ms wait budget; expiry reports `storage_readiness_check_timeout` and fails closed. Pool metadata inspection retains its own 100 ms budget; a timeout is counted by `rustfs_pool_metadata_check_timeouts_total` and uses the last confirmed node-local gate state only within the same cache TTL. These observations are not an atomic cluster snapshot and do not bypass the existing lock-probe timing or cache policy.
|
||||
|
||||
The new fields are additive. Their absence in an older response is not evidence of storage quorum. Minimal responses still contain only the existing top-level fields, liveness remains dependency-independent, and HEAD responses remain bodyless.
|
||||
|
||||
|
||||
@@ -39,9 +39,9 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
|
||||
| Client or engine | Status | Claim |
|
||||
|---|---|---|
|
||||
| PyIceberg | Automated | Namespace and table create, append, reload, scan, metadata-location, refs, views, maintenance, diagnostics, optional vended credentials with an exact-prefix data-plane scope check. |
|
||||
| Spark Iceberg REST catalog | Manual/live harness | Pinned package inputs, catalog properties, SQL, expected `row_count=2`, and a CI opt-in gate for create/append/refresh/count/cleanup. Live execution and commit-conflict probing remain manual unless enabled in the runner. |
|
||||
| Spark Iceberg REST catalog | Manual/live harness | Pinned package inputs, catalog properties, atomic CTAS SQL through staged create, expected `row_count=2`, and a CI opt-in gate for create/refresh/count/cleanup. Live execution and commit-conflict probing remain manual unless enabled in the runner. |
|
||||
| Trino Iceberg REST catalog | Manual/live harness | Catalog properties and a read-only `SELECT COUNT(*)` against a PyIceberg- or Spark-created table. Write compatibility not claimed. |
|
||||
| DuckDB Iceberg 1.5.5 | Automated | `duckdb_smoke.py` covers metadata-location read, single-table create/insert/update/delete/merge, schema evolution, snapshots, concurrent writers, drop, PyIceberg cross-read, and both signing profiles. Staged create, purge-on-drop, and format v3 are verified fail-closed. Two-table mode runs without claiming cross-table atomicity. AWS `ENDPOINT_TYPE S3_TABLES` and vended-credential integration not claimed. |
|
||||
| DuckDB Iceberg 1.5.5 | Automated | `duckdb_smoke.py` covers metadata-location read, single-table create/insert/update/delete/merge, schema evolution, snapshots, concurrent writers, drop, PyIceberg cross-read, and both signing profiles. Purge-on-drop and format v3 are verified fail-closed. Two-table mode runs without claiming cross-table atomicity. AWS `ENDPOINT_TYPE S3_TABLES`, staged-create client integration, and vended-credential integration are not claimed. |
|
||||
| StarRocks Iceberg REST catalog | Documented, not automated | External catalog read-path reference only. |
|
||||
| Databend | Manual/live harness | S3 stage read probe for table data files only; Iceberg REST integration not claimed. |
|
||||
| Snowflake Open Catalog / Iceberg integrations | Generated harness | Operator-adapted external volume/catalog SQL template only; live interop not claimed. |
|
||||
@@ -62,7 +62,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
|
||||
| Catalog config | Supported | `GET /v1/config` advertises defaults and only the supported OpenAPI REST paths in `endpoints`; RustFS extensions (administration, maintenance, migration, diagnostics, refs, metadata-location) are not presented as standard endpoints. |
|
||||
| Table bucket discovery | Supported | `PUT` / `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. |
|
||||
| Namespaces | Supported | Create, list, load, exists, drop on both prefixes; `pageSize`/`pageToken` pagination with context-bound tokens; identifiers limited to 512 ASCII characters. |
|
||||
| Tables | Supported | Create, register, list, load, exists, rename, commit, metadata-location get/update, drop on both prefixes. Rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication; the source name is reusable only via an ETag-conditional tombstone replacement. Commit identifiers must match the URL; unknown requirements/updates fail as bad requests; staged create, register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. |
|
||||
| Tables | Supported | Direct and staged create, register, list, load, exists, rename, commit, metadata-location get/update, and drop on both prefixes. Staged create is stateless until an `assert-create` commit atomically publishes the initial metadata and table entry. Rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication; the source name is reusable only via an ETag-conditional tombstone replacement. Commit identifiers must match the URL; unknown requirements/updates fail as bad requests; register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. |
|
||||
| Commit CAS | Supported | Single-table commits validate base metadata, version token, referenced object existence, warehouse scope, and Iceberg requirements before advancing the pointer; external metadata transitions preserve monotonic assignment watermarks and immutable retained definitions. `idempotency-key-lifetime` is not advertised; mutation-wide `Idempotency-Key` semantics are unsupported. |
|
||||
| Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose and repair finalization gaps without moving the pointer. |
|
||||
| Snapshot refs | Supported | List, create/replace, delete via commits; `main` is protected; refs with explicit retention need forced delete. |
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
# Workspace Cargo Publish Runbook
|
||||
|
||||
This runbook covers crates.io publication for the RustFS Cargo workspace. It is separate from the GitHub tag, binary asset, Docker, Helm, and package release pipeline.
|
||||
|
||||
Use [`scripts/cargo_publish_workspace.sh`](../../scripts/cargo_publish_workspace.sh) as the source of truth for the live publish order. The script derives the order from `cargo metadata`, skips workspace members marked `publish = false`, fails on dependency cycles, and runs crates before their workspace dependents.
|
||||
|
||||
## Scope
|
||||
|
||||
- Publish workspace crates with `cargo publish -p <package>` in dependency order.
|
||||
- Validate the package graph before any registry operation.
|
||||
- Check whether each crate version already exists in the target registry before dry-run or publish work.
|
||||
- Support an offline plan mode and an online dry-run mode before the final publish.
|
||||
- Keep the publish operation on the exact reviewed commit.
|
||||
|
||||
Out of scope:
|
||||
|
||||
- RustFS version-file bumping.
|
||||
- GitHub preview or final tag publication.
|
||||
- GitHub Release asset verification.
|
||||
- Docker, Helm, DEB, RPM, or latest-channel publication.
|
||||
|
||||
## Current Branch Snapshot
|
||||
|
||||
On this branch, `cargo metadata --format-version 1 --no-deps` reports 51 workspace members. None of the workspace package manifests set `publish = false`, so every member is eligible from Cargo's manifest perspective.
|
||||
|
||||
The script always filters out the test-only `e2e_test` package, leaving 50 packages in the default publish plan. If another test-only or internal crate should not be published, set `publish = false` in that crate's manifest before publication. The script also supports `--exclude <package>`, but it will fail if any selected publishable crate still depends on the excluded package.
|
||||
|
||||
The publish order below is the current topological order of workspace path dependencies:
|
||||
|
||||
1. `rustfs-checksums`
|
||||
2. `rustfs-common`
|
||||
3. `rustfs-config`
|
||||
4. `rustfs-credentials`
|
||||
5. `rustfs-crypto`
|
||||
6. `rustfs-extension-schema`
|
||||
7. `rustfs-heal-contracts`
|
||||
8. `rustfs-license`
|
||||
9. `rustfs-log-analyzer`
|
||||
10. `rustfs-object-data-cache`
|
||||
11. `rustfs-replication`
|
||||
12. `rustfs-s3-types`
|
||||
13. `rustfs-security-governance`
|
||||
14. `rustfs-utils`
|
||||
15. `rustfs-tls-runtime`
|
||||
16. `rustfs-policy`
|
||||
17. `rustfs-scanner-metrics`
|
||||
18. `rustfs-s3-ops`
|
||||
19. `rustfs-filemeta`
|
||||
20. `rustfs-kms`
|
||||
21. `rustfs-signer`
|
||||
22. `rustfs-trusted-proxies`
|
||||
23. `rustfs-targets`
|
||||
24. `rustfs-keystone`
|
||||
25. `rustfs-io-metrics`
|
||||
26. `rustfs-data-usage`
|
||||
27. `rustfs-storage-api`
|
||||
28. `rustfs-madmin`
|
||||
29. `rustfs-audit`
|
||||
30. `rustfs-io-core`
|
||||
31. `rustfs-lock`
|
||||
32. `rustfs-object-capacity`
|
||||
33. `rustfs-protos`
|
||||
34. `rustfs-rio`
|
||||
35. `rustfs-lifecycle`
|
||||
36. `rustfs-concurrency`
|
||||
37. `rustfs-rio-v2`
|
||||
38. `rustfs-s3-client`
|
||||
39. `rustfs-zip`
|
||||
40. `rustfs-ecstore`
|
||||
41. `rustfs-notify`
|
||||
42. `rustfs-test-utils`
|
||||
43. `rustfs-heal`
|
||||
44. `rustfs-iam`
|
||||
45. `rustfs-s3select-api`
|
||||
46. `rustfs-scanner`
|
||||
47. `rustfs-obs`
|
||||
48. `rustfs-protocols`
|
||||
49. `rustfs-s3select-query`
|
||||
50. `rustfs`
|
||||
|
||||
Regenerate this order before a real publication:
|
||||
|
||||
```bash
|
||||
scripts/cargo_publish_workspace.sh --mode plan
|
||||
```
|
||||
|
||||
## Preflight
|
||||
|
||||
1. Start from the exact commit that should be published.
|
||||
2. Confirm the worktree is clean:
|
||||
|
||||
```bash
|
||||
git status --short
|
||||
```
|
||||
|
||||
3. Confirm the branch contains the intended version numbers:
|
||||
|
||||
```bash
|
||||
cargo metadata --format-version 1 --no-deps
|
||||
```
|
||||
|
||||
4. Confirm crates.io authentication without printing tokens:
|
||||
|
||||
```bash
|
||||
cargo login
|
||||
```
|
||||
|
||||
5. Run the repository validation appropriate to the version change before publishing. At minimum, a real release should not rely only on `cargo publish --dry-run`; dry-run verifies packaging, not RustFS runtime acceptance.
|
||||
|
||||
## Dry Run
|
||||
|
||||
Run the complete ordered packaging check. The default registry is `crates-io`, and the script skips any crate version that already exists there. The script passes `--registry crates-io` explicitly so local source replacement does not redirect the publish command to a mirror.
|
||||
|
||||
```bash
|
||||
scripts/cargo_publish_workspace.sh --mode dry-run
|
||||
```
|
||||
|
||||
For an alternate registry:
|
||||
|
||||
```bash
|
||||
scripts/cargo_publish_workspace.sh --mode dry-run --registry <registry-name>
|
||||
```
|
||||
|
||||
Dry-run may need network access because the script checks whether versions already exist and Cargo validates registry dependency resolution. If the existence check cannot confirm the registry state, the script fails closed instead of publishing or dry-running that package. If dry-run fails for a workspace dependency, do not skip forward; fix the manifest/version state or publish the missing dependency first.
|
||||
|
||||
## Publish
|
||||
|
||||
The script requires an explicit environment confirmation for publish mode. The default registry is `crates-io`; crate versions that already exist in the registry are reported and skipped, not republished. Missing versions continue to `cargo publish --registry crates-io` by default.
|
||||
|
||||
```bash
|
||||
RUSTFS_CARGO_PUBLISH_CONFIRM=publish \
|
||||
scripts/cargo_publish_workspace.sh --mode publish --wait-seconds 60
|
||||
```
|
||||
|
||||
If the reviewed release/version bump is intentionally still uncommitted when packaging or publishing, pass `--allow-dirty`. The script then passes `--allow-dirty` through to `cargo publish`; without it, both the script and Cargo keep the dirty-worktree protection enabled.
|
||||
|
||||
For an alternate registry:
|
||||
|
||||
```bash
|
||||
RUSTFS_CARGO_PUBLISH_CONFIRM=publish \
|
||||
scripts/cargo_publish_workspace.sh --mode publish --registry <registry-name> --wait-seconds 60
|
||||
```
|
||||
|
||||
The wait between crates gives the registry index time to expose each newly published crate version before dependents are published. Increase `--wait-seconds` if dependents fail because the registry cannot resolve a just-published workspace crate.
|
||||
|
||||
## Failure Handling
|
||||
|
||||
- If a crate fails during dry-run, stop and fix the package before retrying from the beginning.
|
||||
- If the script reports that a crate version already exists, it skips that crate and continues with the next selected package.
|
||||
- If the registry existence check fails for network, service, authentication, or unexpected-response reasons, stop and rerun after the registry state can be verified.
|
||||
- If a dependent crate cannot resolve a just-published internal dependency, wait for registry propagation and retry the same package.
|
||||
- Do not bump versions, retag, or publish GitHub release assets as a workaround for a partial cargo publish. Treat that as a separate release decision.
|
||||
|
||||
## Post-Publish Checks
|
||||
|
||||
After the script finishes, spot-check the registry state:
|
||||
|
||||
```bash
|
||||
cargo info rustfs
|
||||
cargo info rustfs-ecstore
|
||||
cargo info rustfs-utils
|
||||
```
|
||||
|
||||
Record the final commit, crate version, script mode, registry, and any retry decisions in the release handoff.
|
||||
@@ -3,9 +3,8 @@
|
||||
**Use this when:** you need Tokio runtime-level evidence (which task held a worker, long polls, park/unpark behaviour) that Prometheus metrics and `tracing` spans cannot provide, or you are building or running the opt-in `dial9` profiling binary.
|
||||
**Source of truth:** `crates/obs/src/telemetry/dial9/mod.rs` (session setup), `crates/obs/src/metrics/collectors/dial9.rs` (metrics), `crates/config/src/constants/runtime.rs` (`RUSTFS_RUNTIME_DIAL9_*` and defaults), `.config/make/build.mak` (`build-profiling`), `crates/obs/build.rs` (feature/cfg pairing check).
|
||||
|
||||
`dial9-tokio-telemetry` records Tokio runtime-level events — poll start/end,
|
||||
worker park/unpark, task spawn/terminate, and optionally async backtraces of
|
||||
stalled tasks — into binary trace segments.
|
||||
`dial9` records Tokio runtime-level events — poll start/end, worker park/unpark,
|
||||
task spawn/terminate — into binary trace segments.
|
||||
|
||||
It answers questions that Prometheus metrics and `tracing` spans cannot:
|
||||
|
||||
@@ -29,7 +28,7 @@ For drive stalls use the `rustfs_io_*` metrics and the drive-stall budget. dial9
|
||||
answers a different question: which task held a worker, and for how long.
|
||||
|
||||
**It cannot tell you where a task was stuck.** That would need a task dump, and
|
||||
dial9 only captures those for futures spawned through `dial9_tokio_telemetry::spawn`.
|
||||
dial9 only captures those for futures spawned through `dial9::spawn`.
|
||||
RustFS spawns with `tokio::spawn` throughout, so no task dump is ever recorded and
|
||||
no configuration exposes one. Tracked as D9-16 in rustfs/backlog#1157.
|
||||
|
||||
|
||||
@@ -13,7 +13,9 @@ Every remote lock call (`lock`, `lock_batch`, `release`, `refresh`, `force_relea
|
||||
| Any failure while the last eviction is younger than the cooldown | `cooling_down` | Channel kept so the fresh dial can prove itself; no re-dial burst. |
|
||||
| Transport failure (refused, reset, `GOAWAY`) outside the cooldown | `evict` | Cached channel evicted once. |
|
||||
|
||||
A timed-out request is no longer cancelled. Cancelling sends `RST_STREAM`, and enough resets against a server that is slow to accept streams make it answer `GOAWAY too_many_resets`, which kills every stream on the connection and restarts the loop. Instead the stream is detached: it keeps running in the background (bounded by the internode RPC timeout), the caller still gets its timeout error, and if the peer grants a lock after the caller gave up the client releases it immediately instead of leaving an orphan for the lease to expire.
|
||||
A timed-out request is no longer cancelled. Cancelling sends `RST_STREAM`, and enough resets against a server that is slow to accept streams make it answer `GOAWAY too_many_resets`, which kills every stream on the connection and restarts the loop. Instead the stream is detached: it keeps running in the background (bounded by the internode RPC timeout), the caller still gets its timeout error, and if the peer grants a lock after the caller gave up the client performs bounded release retries. A per-peer cleanup budget prevents late-release retries from becoming a second request storm; entries beyond that budget remain protected by the server lease.
|
||||
|
||||
Three consecutive lock-acquisition timeouts open a request-level per-peer breaker. While open, new `lock` and `lock_batch` calls fail fast as retryable contention; after the backoff window one half-open probe is admitted. The breaker does not block release or refresh traffic, and it is separate from channel eviction. A per-peer in-flight admission limit also bounds requests that were already admitted before the breaker opened.
|
||||
|
||||
Unlocks that fail three quick retries no longer stop there. The background task continues with a deferred schedule (1s, 2s, 4s, 8s, 16s) before it gives up and leaves the entry to the server-side lease.
|
||||
|
||||
@@ -24,6 +26,7 @@ Unlocks that fail three quick retries no longer stop there. The background task
|
||||
| `RUSTFS_OBJECT_LOCK_RPC_TIMEOUT_MS` | `3000` | Per-request deadline for remote lock RPCs. |
|
||||
| `RUSTFS_OBJECT_LOCK_RPC_EVICTION_COOLDOWN_MS` | `5000` | Minimum interval between channel evictions per peer. `0` restores eviction on every qualifying failure. |
|
||||
| `RUSTFS_OBJECT_LOCK_RPC_DETACHED_LIMIT` | `256` | How many timed-out lock RPCs per peer may keep running in the background. Beyond the budget a timed-out stream is cancelled as before. |
|
||||
| `RUSTFS_OBJECT_LOCK_RPC_REQUEST_LIMIT` | `128` | Maximum in-flight lock acquisition RPCs admitted to a peer while its request breaker is degraded. Excess calls fail fast as retryable contention; healthy peers are not capped by this guard. |
|
||||
|
||||
## Metrics
|
||||
|
||||
@@ -35,6 +38,7 @@ Unlocks that fail three quick retries no longer stop there. The background task
|
||||
| `rustfs_remote_lock_rpc_detached_total` | `op`, `outcome` | Timed-out RPCs left running (`detached`) or cancelled for budget (`aborted`). |
|
||||
| `rustfs_remote_lock_rpc_late_completions_total` | `op`, `outcome` | How detached RPCs ended (`success`, `error`, `join_error`). |
|
||||
| `rustfs_remote_lock_late_releases_total` | `outcome` | Releases of locks granted after their caller timed out (`released`, `partial`, `failed`). |
|
||||
| `rustfs_remote_lock_request_suppressed_total` | `peer`, `op`, `reason` | Lock acquisitions rejected before dialing because the peer breaker is open or its degraded in-flight limit is reached. |
|
||||
|
||||
## Reading an incident
|
||||
|
||||
|
||||
@@ -292,13 +292,13 @@ Heal knobs are environment-only and read by `HealConfig::default` (`crates/heal/
|
||||
|---|---|---|
|
||||
| `RUSTFS_HEAL_ENABLED` (deprecated alias `RUSTFS_ENABLE_HEAL`) | `true` (`heal_enabled_from_env`, `rustfs/src/module_switches.rs`) | Master switch for the background heal manager. |
|
||||
| `RUSTFS_HEAL_AUTO_HEAL_ENABLE` | `true` (`DEFAULT_HEAL_AUTO_HEAL_ENABLE`) | Enables automatic healing of detected issues; `false` leaves healing to manual admin requests. |
|
||||
| `RUSTFS_HEAL_QUEUE_SIZE` | `10000` (`DEFAULT_HEAL_QUEUE_SIZE`) | Heal request queue capacity. |
|
||||
| `RUSTFS_HEAL_QUEUE_SIZE` | `10000` (`DEFAULT_HEAL_QUEUE_SIZE`) | Heal request queue capacity. The manager reserves at least one slot per Scanner/ReadRepair/AutoHeal source, or 10% for larger queues, so sustained normal-priority MRF or internal background work cannot consume every slot. |
|
||||
| `RUSTFS_HEAL_INTERVAL_SECS` | `10` (`DEFAULT_HEAL_INTERVAL_SECS`) | Heal manager polling interval. |
|
||||
| `RUSTFS_HEAL_TASK_TIMEOUT_SECS` | `300` (`DEFAULT_HEAL_TASK_TIMEOUT_SECS`) | Per-task timeout. |
|
||||
| `RUSTFS_HEAL_MAX_CONCURRENT_HEALS` | `4` (`DEFAULT_HEAL_MAX_CONCURRENT_HEALS`) | Global concurrent heal task limit. |
|
||||
| `RUSTFS_HEAL_MAX_CONCURRENT_PER_SET` | `1` (`DEFAULT_HEAL_MAX_CONCURRENT_PER_SET`) | Per-erasure-set limit; effective value is `min(global, per_set)`, each floored at `1`. |
|
||||
| `RUSTFS_HEAL_LOW_PRIORITY_MERGE_ENABLE` | `true` (`DEFAULT_HEAL_LOW_PRIORITY_MERGE_ENABLE`) | Merge duplicate low-priority requests with the same dedup key. |
|
||||
| `RUSTFS_HEAL_LOW_PRIORITY_DROP_WHEN_FULL` | `true` (`DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL`) | Drop, rather than block on, low-priority requests when the queue is full. |
|
||||
| `RUSTFS_HEAL_LOW_PRIORITY_DROP_WHEN_FULL` | `true` (`DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL`) | Drop, rather than block on, low-priority requests only after the reserved best-effort capacity is occupied and the queue is actually full. |
|
||||
| `RUSTFS_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE` | `true` (`DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE`) | Notify-driven scheduler wakeups. |
|
||||
| `RUSTFS_HEAL_SET_BULKHEAD_ENABLE` | `true` (`DEFAULT_HEAL_SET_BULKHEAD_ENABLE`) | Per-set bulkhead scheduling. |
|
||||
| `RUSTFS_HEAL_PAGE_PARALLEL_ENABLE` | `true` (`DEFAULT_HEAL_PAGE_PARALLEL_ENABLE`) | Page-level parallel object healing during erasure-set repair. |
|
||||
@@ -314,6 +314,18 @@ Heal knobs are environment-only and read by `HealConfig::default` (`crates/heal/
|
||||
| `RUSTFS_HEAL_MRF_REPLAY_BATCH` | `256` (`DEFAULT_HEAL_MRF_REPLAY_BATCH`) | Intents per replay push round. |
|
||||
| `RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS` | `3600` (`DEFAULT_HEAL_DANGLING_DELETE_GRACE_SECS`, `crates/ecstore/src/set_disk/core/io_primitives.rs`) | A recently modified object is never deleted as dangling inside this window; `0` disables the grace window. |
|
||||
|
||||
The heal scheduler preserves priority ordering for urgent, high, and ordinary
|
||||
work, but applies a bounded fairness quantum to a continuously refilled MRF
|
||||
normal backlog: after four MRF/normal dispatches, a runnable Scanner,
|
||||
ReadRepair, or AutoHeal request gets a service opportunity. This protects
|
||||
integrity progress without weakening durable MRF admission or receipt
|
||||
requirements. Queue capacities from 4 through 9 reserve three slots, one per
|
||||
best-effort source, while still leaving one slot for MRF; capacities below 4
|
||||
cannot isolate all work classes and
|
||||
should only be used for diagnostic tests. Normal-priority non-best-effort submissions are deferred at the
|
||||
reserve boundary; high and urgent requests, including admin or MRF recovery,
|
||||
retain their priority and may use that space.
|
||||
|
||||
### Admin heal start, retries, and budgets
|
||||
|
||||
Admin heal has three separate budgets. Increasing one does not extend the others:
|
||||
|
||||
@@ -73,7 +73,7 @@
|
||||
rustc = rustToolchain;
|
||||
};
|
||||
|
||||
clientVersion = "0.1.32";
|
||||
clientVersion = "0.1.36";
|
||||
|
||||
rustfs = rustPlatform.buildRustPackage {
|
||||
pname = "rustfs";
|
||||
|
||||
@@ -43,7 +43,7 @@ spec:
|
||||
service:
|
||||
name: {{ include "rustfs.fullname" $ }}-svc
|
||||
port:
|
||||
name: console
|
||||
name: {{ .servicePort }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -335,10 +335,16 @@ ingress:
|
||||
customAnnotations: # Deprecated: use ingress.annotations instead
|
||||
{}
|
||||
hosts:
|
||||
- host: example.rustfs.com
|
||||
- host: console.rustfs.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
servicePort: console
|
||||
- host: endpoint.rustfs.com
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
servicePort: endpoint
|
||||
tls:
|
||||
enabled: false # Enable tls and access rustfs via https.
|
||||
|
||||
|
||||
@@ -910,6 +910,16 @@ struct RestLoadTableResponse {
|
||||
storage_credentials: Vec<RestStorageCredential>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RestCreateTableResponse {
|
||||
#[serde(rename = "metadata-location")]
|
||||
metadata_location: Option<String>,
|
||||
metadata: serde_json::Value,
|
||||
config: BTreeMap<String, String>,
|
||||
#[serde(rename = "storage-credentials")]
|
||||
storage_credentials: Vec<RestStorageCredential>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct RestLoadCredentialsResponse {
|
||||
config: BTreeMap<String, String>,
|
||||
@@ -2706,6 +2716,25 @@ fn load_table_response_from_entry(entry: crate::table_catalog::TableEntry, metad
|
||||
}
|
||||
}
|
||||
|
||||
fn create_table_response_from_entry(
|
||||
entry: crate::table_catalog::TableEntry,
|
||||
metadata: serde_json::Value,
|
||||
staged: bool,
|
||||
) -> RestCreateTableResponse {
|
||||
let RestLoadTableResponse {
|
||||
metadata_location,
|
||||
metadata,
|
||||
config,
|
||||
storage_credentials,
|
||||
} = load_table_response_from_entry(entry, metadata);
|
||||
RestCreateTableResponse {
|
||||
metadata_location: (!staged).then_some(metadata_location),
|
||||
metadata,
|
||||
config,
|
||||
storage_credentials,
|
||||
}
|
||||
}
|
||||
|
||||
fn load_view_response_from_entry(entry: crate::table_catalog::ViewEntry, metadata: serde_json::Value) -> RestLoadViewResponse {
|
||||
let mut config = BTreeMap::new();
|
||||
let warehouse_location = entry.warehouse_location.clone();
|
||||
@@ -3104,16 +3133,9 @@ fn table_entry_from_create_table_request(
|
||||
mut schema,
|
||||
mut partition_spec,
|
||||
mut write_order,
|
||||
stage_create,
|
||||
stage_create: _,
|
||||
mut properties,
|
||||
} = request;
|
||||
if stage_create {
|
||||
return Err(iceberg_rest_error(
|
||||
ICEBERG_ERROR_UNSUPPORTED_OPERATION,
|
||||
StatusCode::NOT_ACCEPTABLE,
|
||||
"stage-create is not supported",
|
||||
));
|
||||
}
|
||||
|
||||
let table = crate::table_catalog::IdentifierSegment::parse(name)
|
||||
.map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?;
|
||||
@@ -3490,6 +3512,26 @@ fn validate_table_commit_requirements(metadata: &serde_json::Value, requirements
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn request_has_assert_create_requirement(request: &RestCommitTableRequest) -> bool {
|
||||
request
|
||||
.requirements
|
||||
.iter()
|
||||
.any(|requirement| requirement.get("type").and_then(serde_json::Value::as_str) == Some("assert-create"))
|
||||
}
|
||||
|
||||
fn validate_create_table_commit_requirements(requirements: &[serde_json::Value]) -> S3Result<()> {
|
||||
if requirements.is_empty()
|
||||
|| requirements
|
||||
.iter()
|
||||
.any(|requirement| requirement.get("type").and_then(serde_json::Value::as_str) != Some("assert-create"))
|
||||
{
|
||||
return Err(S3Error::from(ApiError::invalid_request(
|
||||
"create table commit requires only assert-create requirements",
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_i64_requirement(
|
||||
metadata: &serde_json::Value,
|
||||
requirement: &serde_json::Value,
|
||||
@@ -3555,20 +3597,104 @@ fn apply_table_commit_updates(
|
||||
}
|
||||
|
||||
fn apply_table_commit_updates_at(
|
||||
mut metadata: serde_json::Value,
|
||||
metadata: serde_json::Value,
|
||||
updates: &[serde_json::Value],
|
||||
previous_metadata_location: &str,
|
||||
commit_timestamp_ms: i64,
|
||||
) -> S3Result<serde_json::Value> {
|
||||
apply_table_updates_at(
|
||||
metadata,
|
||||
updates,
|
||||
TableMetadataUpdateMode::Existing {
|
||||
previous_metadata_location,
|
||||
},
|
||||
commit_timestamp_ms,
|
||||
)
|
||||
}
|
||||
|
||||
fn apply_table_create_updates_at(updates: &[serde_json::Value], commit_timestamp_ms: i64) -> S3Result<serde_json::Value> {
|
||||
let format_version = updates
|
||||
.iter()
|
||||
.find(|update| update.get("action").and_then(serde_json::Value::as_str) == Some("upgrade-format-version"))
|
||||
.and_then(|update| update.get("format-version"))
|
||||
.and_then(serde_json::Value::as_i64)
|
||||
.unwrap_or(2);
|
||||
let mut metadata = serde_json::json!({
|
||||
"format-version": format_version,
|
||||
"last-updated-ms": commit_timestamp_ms,
|
||||
"last-column-id": 0,
|
||||
"schemas": [],
|
||||
"current-schema-id": -1,
|
||||
"partition-specs": [],
|
||||
"default-spec-id": -1,
|
||||
"last-partition-id": 999,
|
||||
"sort-orders": [],
|
||||
"default-sort-order-id": -1,
|
||||
"properties": {},
|
||||
"current-snapshot-id": -1,
|
||||
"snapshots": [],
|
||||
"snapshot-log": [],
|
||||
"metadata-log": [],
|
||||
"refs": {}
|
||||
});
|
||||
if format_version == 2 {
|
||||
metadata_object_mut(&mut metadata)?.insert("last-sequence-number".to_string(), serde_json::Value::from(0));
|
||||
}
|
||||
apply_table_updates_at(metadata, updates, TableMetadataUpdateMode::Create, commit_timestamp_ms)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum TableMetadataUpdateMode<'a> {
|
||||
Create,
|
||||
Existing { previous_metadata_location: &'a str },
|
||||
}
|
||||
|
||||
impl TableMetadataUpdateMode<'_> {
|
||||
fn initial_catalog_id(self, array_key: &str) -> Option<i64> {
|
||||
match (self, array_key) {
|
||||
(Self::Create, "schemas" | "partition-specs") => Some(0),
|
||||
(Self::Create, "sort-orders") => Some(1),
|
||||
(Self::Create, _) | (Self::Existing { .. }, _) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_table_updates_at(
|
||||
mut metadata: serde_json::Value,
|
||||
updates: &[serde_json::Value],
|
||||
mode: TableMetadataUpdateMode<'_>,
|
||||
commit_timestamp_ms: i64,
|
||||
) -> S3Result<serde_json::Value> {
|
||||
if !metadata.is_object() {
|
||||
return Err(s3_error!(InvalidRequest, "current table metadata must be a JSON object"));
|
||||
}
|
||||
if metadata.get("format-version").is_some() {
|
||||
if matches!(mode, TableMetadataUpdateMode::Existing { .. }) && metadata.get("format-version").is_some() {
|
||||
crate::table_catalog::synchronize_table_metadata_version_fields(&mut metadata).map_err(catalog_store_error)?;
|
||||
}
|
||||
let mut next_schema_id = next_catalog_id_for_updates(&metadata, updates, "add-schema", "schemas", "schema-id")?;
|
||||
let mut next_spec_id = next_catalog_id_for_updates(&metadata, updates, "add-spec", "partition-specs", "spec-id")?;
|
||||
let mut next_sort_order_id = next_catalog_id_for_updates(&metadata, updates, "add-sort-order", "sort-orders", "order-id")?;
|
||||
let mut next_schema_id = next_catalog_id_for_updates(
|
||||
&metadata,
|
||||
updates,
|
||||
"add-schema",
|
||||
"schemas",
|
||||
"schema-id",
|
||||
mode.initial_catalog_id("schemas"),
|
||||
)?;
|
||||
let mut next_spec_id = next_catalog_id_for_updates(
|
||||
&metadata,
|
||||
updates,
|
||||
"add-spec",
|
||||
"partition-specs",
|
||||
"spec-id",
|
||||
mode.initial_catalog_id("partition-specs"),
|
||||
)?;
|
||||
let mut next_sort_order_id = next_catalog_id_for_updates(
|
||||
&metadata,
|
||||
updates,
|
||||
"add-sort-order",
|
||||
"sort-orders",
|
||||
"order-id",
|
||||
mode.initial_catalog_id("sort-orders"),
|
||||
)?;
|
||||
let mut last_added_schema_id = None;
|
||||
let mut last_added_spec_id = None;
|
||||
let mut last_added_sort_order_id = None;
|
||||
@@ -3658,8 +3784,16 @@ fn apply_table_commit_updates_at(
|
||||
if metadata.get("format-version").is_some() {
|
||||
crate::table_catalog::synchronize_table_metadata_version_fields(&mut metadata).map_err(catalog_store_error)?;
|
||||
}
|
||||
append_previous_metadata_log(&mut metadata, previous_metadata_location)?;
|
||||
if let TableMetadataUpdateMode::Existing {
|
||||
previous_metadata_location,
|
||||
} = mode
|
||||
{
|
||||
append_previous_metadata_log(&mut metadata, previous_metadata_location)?;
|
||||
}
|
||||
metadata_object_mut(&mut metadata)?.insert("last-updated-ms".to_string(), serde_json::Value::from(commit_timestamp_ms));
|
||||
if matches!(mode, TableMetadataUpdateMode::Create) {
|
||||
crate::table_catalog::validate_supported_table_metadata(&metadata).map_err(catalog_store_error)?;
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
@@ -3726,7 +3860,7 @@ fn apply_view_commit_updates_at(
|
||||
if !metadata.is_object() {
|
||||
return Err(s3_error!(InvalidRequest, "current view metadata must be a JSON object"));
|
||||
}
|
||||
let mut next_schema_id = next_catalog_id_for_updates(&metadata, updates, "add-schema", "schemas", "schema-id")?;
|
||||
let mut next_schema_id = next_catalog_id_for_updates(&metadata, updates, "add-schema", "schemas", "schema-id", None)?;
|
||||
let mut last_added_schema_id = None;
|
||||
let mut last_added_view_version_id = None;
|
||||
let mut added_view_version_timestamps = BTreeMap::new();
|
||||
@@ -5084,11 +5218,12 @@ fn next_catalog_id_for_updates(
|
||||
action: &str,
|
||||
array_key: &str,
|
||||
id_key: &str,
|
||||
initial_id: Option<i64>,
|
||||
) -> S3Result<Option<i64>> {
|
||||
updates
|
||||
.iter()
|
||||
.any(|update| update.get("action").and_then(serde_json::Value::as_str) == Some(action))
|
||||
.then(|| next_array_object_i64(metadata, array_key, id_key))
|
||||
.then(|| next_array_object_i64(metadata, array_key, id_key, initial_id))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
@@ -5103,10 +5238,22 @@ fn take_catalog_assigned_id(next_id: &mut Option<i64>, label: &str) -> S3Result<
|
||||
Ok(assigned_id)
|
||||
}
|
||||
|
||||
fn next_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result<i64> {
|
||||
last_array_object_i64(metadata, array_key, id_key)?
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} {id_key} exceeds the signed 64-bit range"))
|
||||
fn next_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str, initial_id: Option<i64>) -> S3Result<i64> {
|
||||
match last_array_object_i64(metadata, array_key, id_key) {
|
||||
Ok(last_id) => last_id
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} {id_key} exceeds the signed 64-bit range")),
|
||||
Err(_)
|
||||
if metadata
|
||||
.get(array_key)
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.is_some_and(Vec::is_empty) =>
|
||||
{
|
||||
initial_id
|
||||
.ok_or_else(|| S3Error::from(ApiError::invalid_request(format!("metadata field {array_key} has no {id_key}"))))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn last_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result<i64> {
|
||||
@@ -5435,11 +5582,40 @@ async fn create_table_response<S>(
|
||||
namespace: &crate::table_catalog::Namespace,
|
||||
request: CreateTableRequest,
|
||||
table_bucket_enabled: bool,
|
||||
) -> S3Result<RestLoadTableResponse>
|
||||
) -> S3Result<RestCreateTableResponse>
|
||||
where
|
||||
S: crate::table_catalog::TableCatalogStore + ?Sized,
|
||||
{
|
||||
let staged = request.stage_create;
|
||||
let (entry, metadata) = table_entry_from_create_table_request(bucket, namespace, request)?;
|
||||
if staged {
|
||||
if !table_bucket_enabled {
|
||||
return Err(S3Error::from(ApiError::invalid_request(format!("bucket {bucket} is not table-enabled"))));
|
||||
}
|
||||
get_namespace_response(store, bucket, namespace).await?;
|
||||
if store
|
||||
.load_table(bucket, &namespace.public_name(), &entry.table)
|
||||
.await
|
||||
.map_err(catalog_store_error)?
|
||||
.is_some()
|
||||
|| store
|
||||
.load_view(bucket, &namespace.public_name(), &entry.table)
|
||||
.await
|
||||
.map_err(catalog_store_error)?
|
||||
.is_some()
|
||||
{
|
||||
return Err(iceberg_rest_error(
|
||||
ICEBERG_ERROR_ALREADY_EXISTS,
|
||||
StatusCode::CONFLICT,
|
||||
format!("catalog object already exists: {}/{}/{}", bucket, namespace.public_name(), entry.table),
|
||||
));
|
||||
}
|
||||
store
|
||||
.ensure_table_warehouse_location_available(&entry)
|
||||
.await
|
||||
.map_err(catalog_store_already_exists_error)?;
|
||||
return Ok(create_table_response_from_entry(entry, metadata, true));
|
||||
}
|
||||
ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?;
|
||||
crate::table_catalog::TableCommitPublication::begin_table_bucket(metadata_backend, bucket)
|
||||
.await
|
||||
@@ -5465,7 +5641,7 @@ where
|
||||
.register_table_with_publication(entry.clone(), metadata_backend)
|
||||
.await
|
||||
.map_err(catalog_store_already_exists_error)?;
|
||||
Ok(load_table_response_from_entry(entry, metadata))
|
||||
Ok(create_table_response_from_entry(entry, metadata, false))
|
||||
}
|
||||
|
||||
async fn create_view_response<S>(
|
||||
@@ -6080,11 +6256,14 @@ async fn standard_commit_table_response<S>(
|
||||
where
|
||||
S: crate::table_catalog::TableCatalogStore + ?Sized,
|
||||
{
|
||||
let Some(current) = store
|
||||
let current = store
|
||||
.load_table(bucket, &namespace.public_name(), table)
|
||||
.await
|
||||
.map_err(catalog_store_error)?
|
||||
else {
|
||||
.map_err(catalog_store_error)?;
|
||||
let Some(current) = current else {
|
||||
if request_has_assert_create_requirement(&request) {
|
||||
return commit_staged_table_create_response(store, metadata_backend, bucket, namespace, table, request).await;
|
||||
}
|
||||
return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found"));
|
||||
};
|
||||
if let Some(response) =
|
||||
@@ -6194,6 +6373,104 @@ where
|
||||
Ok(commit_table_response_from_result(result, next_metadata))
|
||||
}
|
||||
|
||||
async fn commit_staged_table_create_response<S>(
|
||||
store: &S,
|
||||
metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend,
|
||||
bucket: &str,
|
||||
namespace: &crate::table_catalog::Namespace,
|
||||
table: &str,
|
||||
request: RestCommitTableRequest,
|
||||
) -> S3Result<RestCommitTableResponse>
|
||||
where
|
||||
S: crate::table_catalog::TableCatalogStore + ?Sized,
|
||||
{
|
||||
validate_create_table_commit_requirements(&request.requirements)?;
|
||||
get_namespace_response(store, bucket, namespace).await?;
|
||||
if store
|
||||
.load_view(bucket, &namespace.public_name(), table)
|
||||
.await
|
||||
.map_err(catalog_store_error)?
|
||||
.is_some()
|
||||
{
|
||||
return Err(iceberg_rest_error(
|
||||
ICEBERG_ERROR_ALREADY_EXISTS,
|
||||
StatusCode::CONFLICT,
|
||||
format!("catalog object already exists: {}/{}/{}", bucket, namespace.public_name(), table),
|
||||
));
|
||||
}
|
||||
let commit_timestamp_ms = current_time_millis();
|
||||
let metadata = apply_table_create_updates_at(&request.updates, commit_timestamp_ms)?;
|
||||
validate_metadata_table_location_in_bucket(bucket, &metadata)?;
|
||||
|
||||
let table = crate::table_catalog::IdentifierSegment::parse(table.to_string())
|
||||
.map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid table name: {err}"))))?;
|
||||
let (commit_id, _) = standard_commit_ids(request.commit_id.or(request.idempotency_key));
|
||||
let table_id = Uuid::new_v4().to_string();
|
||||
let metadata_location =
|
||||
crate::table_catalog::default_table_metadata_file_path(namespace, &table, &next_metadata_file_name(1, &table_id));
|
||||
let properties = serde_json::from_value::<BTreeMap<String, String>>(
|
||||
metadata
|
||||
.get("properties")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| serde_json::Value::Object(serde_json::Map::new())),
|
||||
)
|
||||
.map_err(|_| S3Error::from(ApiError::invalid_request("table property values must be strings")))?;
|
||||
let mut entry = crate::table_catalog::TableEntry {
|
||||
version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION,
|
||||
table_bucket: bucket.to_string(),
|
||||
namespace: namespace.public_name(),
|
||||
table: table.as_str().to_string(),
|
||||
table_id,
|
||||
table_uuid: Uuid::new_v4().to_string(),
|
||||
format: "ICEBERG".to_string(),
|
||||
format_version: 1,
|
||||
warehouse_location: metadata_table_location(&metadata)?.to_string(),
|
||||
metadata_location,
|
||||
version_token: format!("token-{}", Uuid::new_v4()),
|
||||
generation: 1,
|
||||
state: crate::table_catalog::TableCatalogEntryState::Active,
|
||||
properties,
|
||||
created_at: None,
|
||||
updated_at: None,
|
||||
};
|
||||
adopt_registered_metadata_identity(&mut entry, &metadata)?;
|
||||
validate_table_metadata_snapshot_graph(metadata_backend, bucket, &entry, None, &metadata).await?;
|
||||
|
||||
crate::table_catalog::TableCommitPublication::begin_table_bucket(metadata_backend, bucket)
|
||||
.await
|
||||
.map_err(catalog_store_error)?;
|
||||
if !crate::table_catalog::TableCommitPublication::holds_table_bucket(metadata_backend, bucket) {
|
||||
return Err(catalog_store_error(crate::table_catalog::TableCatalogStoreError::Internal(
|
||||
"staged table creation requires a table-bucket publication fence".to_string(),
|
||||
)));
|
||||
}
|
||||
let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend);
|
||||
let metadata_data = serde_json::to_vec(&metadata).map_err(|err| {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize initial table metadata: {err}"))
|
||||
})?;
|
||||
metadata_backend
|
||||
.put_object(
|
||||
bucket,
|
||||
&entry.metadata_location,
|
||||
metadata_data,
|
||||
crate::table_catalog::TableCatalogPutPrecondition::IfAbsent,
|
||||
)
|
||||
.await
|
||||
.map_err(catalog_store_already_exists_error)?;
|
||||
store
|
||||
.register_table_with_publication(entry.clone(), metadata_backend)
|
||||
.await
|
||||
.map_err(catalog_store_already_exists_error)?;
|
||||
|
||||
Ok(RestCommitTableResponse {
|
||||
metadata_location: table_metadata_location_for_client(bucket, &entry.metadata_location),
|
||||
metadata: table_metadata_for_client(bucket, metadata),
|
||||
version_token: entry.version_token,
|
||||
generation: entry.generation,
|
||||
commit_id,
|
||||
})
|
||||
}
|
||||
|
||||
async fn table_commit_for_retry<S>(
|
||||
store: &S,
|
||||
bucket: &str,
|
||||
|
||||
@@ -203,6 +203,10 @@ impl Operation for RestCommitTableHandler {
|
||||
install_table_catalog_s3_request_info(&mut req, &principal)?;
|
||||
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
|
||||
let request = read_rest_commit_table_request(std::mem::take(&mut req.input)).await?;
|
||||
if request_has_assert_create_requirement(&request) {
|
||||
let namespace_resource = TableCatalogResource::namespace(&warehouse, &namespace);
|
||||
authorize_table_catalog_resource_request(&req, &namespace_resource, AdminAction::CreateTableAction).await?;
|
||||
}
|
||||
let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?;
|
||||
let store = table_catalog_store_from_backend(metadata_backend.clone())?;
|
||||
let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req);
|
||||
|
||||
@@ -668,6 +668,11 @@ fn table_catalog_handlers_require_table_admin_actions() {
|
||||
);
|
||||
}
|
||||
|
||||
let commit_table_block = operation_block(&src, "RestCommitTableHandler");
|
||||
assert!(commit_table_block.contains("request_has_assert_create_requirement(&request)"));
|
||||
assert!(commit_table_block.contains("TableCatalogResource::namespace(&warehouse, &namespace)"));
|
||||
assert!(commit_table_block.contains("AdminAction::CreateTableAction"));
|
||||
|
||||
let sync_bridge_block = operation_block(&src, "SyncExternalCatalogBridgeHandler");
|
||||
assert!(
|
||||
sync_bridge_block.contains("AdminAction::RegisterTableAction"),
|
||||
@@ -2825,7 +2830,7 @@ async fn commit_request_readers_require_standard_arrays_and_preserve_legacy_poin
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_create_and_register_modes_return_iceberg_errors() {
|
||||
fn register_overwrite_returns_iceberg_unsupported_error() {
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
|
||||
let register_error = table_entry_from_register_request(
|
||||
"warehouse",
|
||||
@@ -2839,17 +2844,6 @@ fn unsupported_create_and_register_modes_return_iceberg_errors() {
|
||||
.expect_err("register overwrite should remain unsupported");
|
||||
assert_eq!(register_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_UNSUPPORTED_OPERATION.into()));
|
||||
assert_eq!(register_error.status_code(), Some(StatusCode::NOT_ACCEPTABLE));
|
||||
|
||||
let create_request: CreateTableRequest = serde_json::from_value(serde_json::json!({
|
||||
"name": "events",
|
||||
"schema": {"type": "struct", "schema-id": 0, "fields": []},
|
||||
"stage-create": true
|
||||
}))
|
||||
.expect("stage-create request should parse");
|
||||
let create_error = table_entry_from_create_table_request("warehouse", &namespace, create_request)
|
||||
.expect_err("staged create should remain unsupported");
|
||||
assert_eq!(create_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_UNSUPPORTED_OPERATION.into()));
|
||||
assert_eq!(create_error.status_code(), Some(StatusCode::NOT_ACCEPTABLE));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2973,10 +2967,10 @@ async fn create_table_response_writes_initial_metadata_for_standard_request() {
|
||||
.expect("table should exist");
|
||||
assert_eq!(
|
||||
response.metadata_location,
|
||||
format!(
|
||||
Some(format!(
|
||||
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001-{}.metadata.json",
|
||||
entry.table_id
|
||||
)
|
||||
))
|
||||
);
|
||||
assert_eq!(response.metadata["table-uuid"], entry.table_uuid);
|
||||
assert!(
|
||||
@@ -3380,7 +3374,7 @@ async fn concurrent_create_table_responses_keep_one_catalog_winner_with_distinct
|
||||
let winner_entry = &tables[0];
|
||||
assert_eq!(
|
||||
winner.metadata_location,
|
||||
table_metadata_location_for_client("warehouse", &winner_entry.metadata_location)
|
||||
Some(table_metadata_location_for_client("warehouse", &winner_entry.metadata_location))
|
||||
);
|
||||
let metadata_prefix = winner_entry
|
||||
.metadata_location
|
||||
@@ -3415,6 +3409,423 @@ async fn concurrent_create_table_responses_keep_one_catalog_winner_with_distinct
|
||||
assert!(table_uuids.contains(&winner_entry.table_uuid));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_create_is_invisible_until_assert_create_commit_across_backings() {
|
||||
for mode in [
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
crate::table_catalog::TableCatalogBackingMode::DurableStrong,
|
||||
] {
|
||||
let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let store = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(metadata_backend.clone(), mode);
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
|
||||
ensure_table_bucket_entry(&store, "warehouse", true)
|
||||
.await
|
||||
.expect("table bucket entry should be seeded");
|
||||
create_namespace_response(
|
||||
&store,
|
||||
"warehouse",
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("namespace should be created");
|
||||
let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
|
||||
|
||||
let staged =
|
||||
create_table_response(&store, &commit_backend, "warehouse", &namespace, staged_events_create_request(), true)
|
||||
.await
|
||||
.expect("stage-create should return initialized metadata");
|
||||
|
||||
assert_eq!(staged.metadata_location, None, "{mode:?}");
|
||||
let staged_json = serde_json::to_value(&staged).expect("staged response should serialize");
|
||||
assert!(staged_json["metadata-location"].is_null(), "{mode:?}");
|
||||
assert!(
|
||||
store
|
||||
.load_table("warehouse", "analytics", "events")
|
||||
.await
|
||||
.expect("table lookup should succeed")
|
||||
.is_none(),
|
||||
"{mode:?}"
|
||||
);
|
||||
let metadata_prefix = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/";
|
||||
assert!(
|
||||
metadata_backend
|
||||
.list_objects("warehouse", metadata_prefix)
|
||||
.await
|
||||
.expect("metadata listing should succeed")
|
||||
.is_empty(),
|
||||
"{mode:?}"
|
||||
);
|
||||
|
||||
let committed = commit_table_response(
|
||||
&store,
|
||||
&commit_backend,
|
||||
"warehouse",
|
||||
&namespace,
|
||||
"events",
|
||||
staged_create_commit_request(&staged.metadata, []),
|
||||
)
|
||||
.await
|
||||
.expect("assert-create commit should publish the table");
|
||||
|
||||
assert_eq!(committed.generation, 1, "{mode:?}");
|
||||
assert_eq!(committed.metadata["table-uuid"], staged.metadata["table-uuid"], "{mode:?}");
|
||||
assert_eq!(committed.metadata["metadata-log"], serde_json::json!([]), "{mode:?}");
|
||||
let entry = store
|
||||
.load_table("warehouse", "analytics", "events")
|
||||
.await
|
||||
.expect("table lookup should succeed")
|
||||
.expect("committed table should be visible");
|
||||
assert_eq!(entry.table_uuid, staged.metadata["table-uuid"].as_str().unwrap(), "{mode:?}");
|
||||
assert_eq!(
|
||||
committed.metadata_location,
|
||||
table_metadata_location_for_client("warehouse", &entry.metadata_location),
|
||||
"{mode:?}"
|
||||
);
|
||||
assert!(
|
||||
metadata_backend
|
||||
.object_exists("warehouse", &entry.metadata_location)
|
||||
.await
|
||||
.expect("metadata lookup should succeed"),
|
||||
"{mode:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_create_commit_accepts_initial_snapshot_for_spark_ctas() {
|
||||
let store = TestTableCatalogStore::default();
|
||||
let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
|
||||
ensure_table_bucket_entry(&store, "warehouse", true)
|
||||
.await
|
||||
.expect("table bucket entry should be seeded");
|
||||
create_namespace_response(
|
||||
&store,
|
||||
"warehouse",
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("namespace should be created");
|
||||
let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
|
||||
let staged = create_table_response(&store, &commit_backend, "warehouse", &namespace, staged_events_create_request(), true)
|
||||
.await
|
||||
.expect("stage-create should succeed");
|
||||
let table_location = staged.metadata["location"]
|
||||
.as_str()
|
||||
.expect("table location should be present");
|
||||
let manifest_list = format!("{table_location}/metadata/snap-10.avro");
|
||||
let data_file = format!("{table_location}/data/part-10.parquet");
|
||||
seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await;
|
||||
|
||||
let committed = commit_table_response(
|
||||
&store,
|
||||
&commit_backend,
|
||||
"warehouse",
|
||||
&namespace,
|
||||
"events",
|
||||
staged_create_commit_request(
|
||||
&staged.metadata,
|
||||
[
|
||||
serde_json::json!({
|
||||
"action": "add-snapshot",
|
||||
"snapshot": {
|
||||
"snapshot-id": 10,
|
||||
"sequence-number": 1,
|
||||
"timestamp-ms": 1234,
|
||||
"manifest-list": manifest_list,
|
||||
"summary": {"operation": "append"}
|
||||
}
|
||||
}),
|
||||
serde_json::json!({
|
||||
"action": "set-snapshot-ref",
|
||||
"ref-name": "main",
|
||||
"snapshot-id": 10,
|
||||
"type": "branch"
|
||||
}),
|
||||
],
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("Spark CTAS-style create commit should succeed");
|
||||
|
||||
assert_eq!(committed.metadata["current-snapshot-id"], 10);
|
||||
assert_eq!(committed.metadata["last-sequence-number"], 1);
|
||||
assert_eq!(committed.metadata["refs"]["main"]["snapshot-id"], 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn staged_create_commit_defaults_to_format_version_two_without_upgrade() {
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
|
||||
let (_, staged_metadata) = table_entry_from_create_table_request("warehouse", &namespace, staged_events_create_request())
|
||||
.expect("staged create metadata should initialize");
|
||||
let mut request = staged_create_commit_request(&staged_metadata, []);
|
||||
request
|
||||
.updates
|
||||
.retain(|update| update.get("action").and_then(serde_json::Value::as_str) != Some("upgrade-format-version"));
|
||||
|
||||
let metadata = apply_table_create_updates_at(&request.updates, 1234)
|
||||
.expect("create updates without an explicit upgrade should use the Iceberg default");
|
||||
|
||||
assert_eq!(metadata["format-version"], 2);
|
||||
assert_eq!(metadata["last-sequence-number"], 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn staged_create_rejects_an_active_table_warehouse_location() {
|
||||
for mode in [
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
crate::table_catalog::TableCatalogBackingMode::DurableStrong,
|
||||
] {
|
||||
let catalog_backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let store = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(catalog_backend, mode);
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
|
||||
ensure_table_bucket_entry(&store, "warehouse", true)
|
||||
.await
|
||||
.expect("table bucket entry should be seeded");
|
||||
create_namespace_response(
|
||||
&store,
|
||||
"warehouse",
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("namespace should be created");
|
||||
let commit_backend = TableCommitObjectBackend::trusted(metadata_backend);
|
||||
let active_request = serde_json::from_value::<CreateTableRequest>(serde_json::json!({
|
||||
"name": "active_events",
|
||||
"location": "s3://warehouse/shared/events",
|
||||
"schema": {"type": "struct", "fields": []}
|
||||
}))
|
||||
.expect("active create request should parse");
|
||||
create_table_response(&store, &commit_backend, "warehouse", &namespace, active_request, true)
|
||||
.await
|
||||
.expect("active table should be created");
|
||||
let staged_request = serde_json::from_value::<CreateTableRequest>(serde_json::json!({
|
||||
"name": "staged_events",
|
||||
"location": "s3://warehouse/shared/events/child",
|
||||
"schema": {"type": "struct", "fields": []},
|
||||
"stage-create": true
|
||||
}))
|
||||
.expect("staged create request should parse");
|
||||
|
||||
let error = create_table_response(&store, &commit_backend, "warehouse", &namespace, staged_request, true)
|
||||
.await
|
||||
.expect_err("stage-create must reject a location owned by an active table");
|
||||
|
||||
assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_ALREADY_EXISTS.into()), "{mode:?}");
|
||||
assert_eq!(error.status_code(), Some(StatusCode::CONFLICT), "{mode:?}");
|
||||
assert!(
|
||||
store
|
||||
.load_table("warehouse", "analytics", "staged_events")
|
||||
.await
|
||||
.expect("table lookup should succeed")
|
||||
.is_none(),
|
||||
"{mode:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_staged_create_commits_publish_exactly_one_table() {
|
||||
for mode in [
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
crate::table_catalog::TableCatalogBackingMode::DurableStrong,
|
||||
] {
|
||||
let catalog_backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let barrier = Arc::new(tokio::sync::Barrier::new(2));
|
||||
let metadata_backend = TestTableCatalogObjectBackend {
|
||||
put_object_barrier: Some(Arc::clone(&barrier)),
|
||||
..TestTableCatalogObjectBackend::content_addressed()
|
||||
};
|
||||
let store = Arc::new(crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(catalog_backend, mode));
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
|
||||
ensure_table_bucket_entry(store.as_ref(), "warehouse", true)
|
||||
.await
|
||||
.expect("table bucket entry should be seeded");
|
||||
create_namespace_response(
|
||||
store.as_ref(),
|
||||
"warehouse",
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("namespace should be created");
|
||||
let staged = create_table_response(
|
||||
store.as_ref(),
|
||||
&TableCommitObjectBackend::trusted(metadata_backend.clone()),
|
||||
"warehouse",
|
||||
&namespace,
|
||||
staged_events_create_request(),
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("stage-create should succeed");
|
||||
let first_store = Arc::clone(&store);
|
||||
let first_namespace = namespace.clone();
|
||||
let first_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
|
||||
let first_request = staged_create_commit_request(&staged.metadata, []);
|
||||
let first = tokio::spawn(async move {
|
||||
commit_table_response(
|
||||
first_store.as_ref(),
|
||||
&first_backend,
|
||||
"warehouse",
|
||||
&first_namespace,
|
||||
"events",
|
||||
first_request,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(2), async {
|
||||
while metadata_backend.state.lock().await.objects.is_empty() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("first metadata write should reach the publication pause");
|
||||
metadata_backend.lock_attempts.lock().await.clear();
|
||||
let second_store = Arc::clone(&store);
|
||||
let second_namespace = namespace.clone();
|
||||
let second_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
|
||||
let second_request = staged_create_commit_request(&staged.metadata, []);
|
||||
let second = tokio::spawn(async move {
|
||||
commit_table_response(
|
||||
second_store.as_ref(),
|
||||
&second_backend,
|
||||
"warehouse",
|
||||
&second_namespace,
|
||||
"events",
|
||||
second_request,
|
||||
)
|
||||
.await
|
||||
});
|
||||
metadata_backend.wait_for_lock_attempts(1).await;
|
||||
assert!(
|
||||
!second.is_finished(),
|
||||
"second create must wait after observing the table as absent: {mode:?}"
|
||||
);
|
||||
|
||||
barrier.wait().await;
|
||||
let _winner = tokio::time::timeout(StdDuration::from_secs(2), first)
|
||||
.await
|
||||
.expect("first assert-create commit should complete")
|
||||
.expect("first assert-create task should join")
|
||||
.expect("first assert-create commit should win");
|
||||
tokio::time::timeout(StdDuration::from_secs(2), async {
|
||||
while metadata_backend.state.lock().await.objects.len() < 2 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second metadata write should reach the publication pause");
|
||||
barrier.wait().await;
|
||||
let loser = tokio::time::timeout(StdDuration::from_secs(2), second)
|
||||
.await
|
||||
.expect("second assert-create commit should complete")
|
||||
.expect("second assert-create task should join")
|
||||
.expect_err("second assert-create commit must lose at atomic registration");
|
||||
|
||||
assert_eq!(loser.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_ALREADY_EXISTS.into()), "{mode:?}");
|
||||
assert_eq!(loser.status_code(), Some(StatusCode::CONFLICT), "{mode:?}");
|
||||
assert_eq!(
|
||||
store
|
||||
.list_tables("warehouse", "analytics")
|
||||
.await
|
||||
.expect("table listing should succeed")
|
||||
.len(),
|
||||
1,
|
||||
"{mode:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_or_conflicting_assert_create_does_not_replace_a_table() {
|
||||
let store = TestTableCatalogStore::default();
|
||||
let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
|
||||
ensure_table_bucket_entry(&store, "warehouse", true)
|
||||
.await
|
||||
.expect("table bucket entry should be seeded");
|
||||
create_namespace_response(
|
||||
&store,
|
||||
"warehouse",
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("namespace should be created");
|
||||
let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
|
||||
let malformed = serde_json::from_value(serde_json::json!({
|
||||
"requirements": [{"type": "assert-create"}],
|
||||
"updates": []
|
||||
}))
|
||||
.expect("malformed create commit should parse");
|
||||
let error = commit_table_response(&store, &commit_backend, "warehouse", &namespace, "events", malformed)
|
||||
.await
|
||||
.expect_err("incomplete create updates must fail");
|
||||
assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST));
|
||||
assert!(
|
||||
store
|
||||
.load_table("warehouse", "analytics", "events")
|
||||
.await
|
||||
.expect("table lookup should succeed")
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let staged = create_table_response(&store, &commit_backend, "warehouse", &namespace, staged_events_create_request(), true)
|
||||
.await
|
||||
.expect("stage-create should succeed");
|
||||
let first = commit_table_response(
|
||||
&store,
|
||||
&commit_backend,
|
||||
"warehouse",
|
||||
&namespace,
|
||||
"events",
|
||||
staged_create_commit_request(&staged.metadata, []),
|
||||
)
|
||||
.await
|
||||
.expect("first assert-create should succeed");
|
||||
let error = commit_table_response(
|
||||
&store,
|
||||
&commit_backend,
|
||||
"warehouse",
|
||||
&namespace,
|
||||
"events",
|
||||
staged_create_commit_request(&staged.metadata, []),
|
||||
)
|
||||
.await
|
||||
.expect_err("assert-create must fail after the table exists");
|
||||
assert_eq!(error.status_code(), Some(StatusCode::CONFLICT));
|
||||
let current = store
|
||||
.load_table("warehouse", "analytics", "events")
|
||||
.await
|
||||
.expect("table lookup should succeed")
|
||||
.expect("table should remain visible");
|
||||
assert_eq!(current.generation, first.generation);
|
||||
assert_eq!(
|
||||
table_metadata_location_for_client("warehouse", ¤t.metadata_location),
|
||||
first.metadata_location
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn standard_commit_applies_updates_and_writes_next_metadata() {
|
||||
let store = TestTableCatalogStore::default();
|
||||
@@ -10868,6 +11279,68 @@ async fn seed_test_manifest_data_files(
|
||||
}
|
||||
}
|
||||
|
||||
fn staged_events_create_request() -> CreateTableRequest {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"name": "events",
|
||||
"schema": {
|
||||
"type": "struct",
|
||||
"schema-id": 0,
|
||||
"fields": [
|
||||
{"id": 1, "name": "id", "required": true, "type": "long"},
|
||||
{"id": 2, "name": "payload", "required": false, "type": "string"}
|
||||
]
|
||||
},
|
||||
"stage-create": true,
|
||||
"properties": {"write.format.default": "parquet"}
|
||||
}))
|
||||
.expect("staged create table request should parse")
|
||||
}
|
||||
|
||||
fn staged_create_commit_request(
|
||||
staged_metadata: &serde_json::Value,
|
||||
additional_updates: impl IntoIterator<Item = serde_json::Value>,
|
||||
) -> RestCommitTableRequest {
|
||||
let mut updates = vec![
|
||||
serde_json::json!({
|
||||
"action": "assign-uuid",
|
||||
"uuid": staged_metadata["table-uuid"]
|
||||
}),
|
||||
serde_json::json!({
|
||||
"action": "upgrade-format-version",
|
||||
"format-version": staged_metadata["format-version"]
|
||||
}),
|
||||
serde_json::json!({
|
||||
"action": "add-schema",
|
||||
"schema": staged_metadata["schemas"][0]
|
||||
}),
|
||||
serde_json::json!({"action": "set-current-schema", "schema-id": -1}),
|
||||
serde_json::json!({
|
||||
"action": "add-spec",
|
||||
"spec": staged_metadata["partition-specs"][0]
|
||||
}),
|
||||
serde_json::json!({"action": "set-default-spec", "spec-id": -1}),
|
||||
serde_json::json!({
|
||||
"action": "add-sort-order",
|
||||
"sort-order": staged_metadata["sort-orders"][0]
|
||||
}),
|
||||
serde_json::json!({"action": "set-default-sort-order", "sort-order-id": -1}),
|
||||
serde_json::json!({
|
||||
"action": "set-location",
|
||||
"location": staged_metadata["location"]
|
||||
}),
|
||||
serde_json::json!({
|
||||
"action": "set-properties",
|
||||
"updates": staged_metadata["properties"]
|
||||
}),
|
||||
];
|
||||
updates.extend(additional_updates);
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"requirements": [{"type": "assert-create"}],
|
||||
"updates": updates
|
||||
}))
|
||||
.expect("staged create commit request should parse")
|
||||
}
|
||||
|
||||
async fn create_standard_events_table<S>(
|
||||
store: &S,
|
||||
metadata_backend: &TestTableCatalogObjectBackend,
|
||||
@@ -10907,9 +11380,17 @@ where
|
||||
}))
|
||||
.expect("standard create table request should parse");
|
||||
let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
|
||||
create_table_response(store, &commit_backend, "warehouse", namespace, create_request, true)
|
||||
let response = create_table_response(store, &commit_backend, "warehouse", namespace, create_request, true)
|
||||
.await
|
||||
.expect("table should be created")
|
||||
.expect("table should be created");
|
||||
RestLoadTableResponse {
|
||||
metadata_location: response
|
||||
.metadata_location
|
||||
.expect("direct create should return metadata location"),
|
||||
metadata: response.metadata,
|
||||
config: response.config,
|
||||
storage_credentials: response.storage_credentials,
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_standard_recent_events_view<S>(
|
||||
|
||||
@@ -92,7 +92,7 @@ use rustfs_policy::policy::{
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_targets::{
|
||||
EventName,
|
||||
arn::{ARN, TargetIDError},
|
||||
arn::{ARN, TargetID, TargetIDError},
|
||||
};
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use rustfs_utils::http::{SUFFIX_FORCE_DELETE, get_header};
|
||||
@@ -508,6 +508,46 @@ fn validate_notification_configuration_filters(notification_configuration: &Noti
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_notification_target_id(arn_str: &str) -> Result<TargetID, TargetIDError> {
|
||||
ARN::parse(arn_str)
|
||||
.map(|arn| arn.target_id)
|
||||
.map_err(|e| TargetIDError::InvalidFormat(e.to_string()))
|
||||
}
|
||||
|
||||
type NotificationEventRule = (Vec<EventName>, String, String, Vec<TargetID>);
|
||||
|
||||
/// Builds the notify runtime rules for a bucket notification configuration
|
||||
/// without touching the store or the runtime rule state.
|
||||
fn build_notification_event_rules(
|
||||
notification_configuration: &NotificationConfiguration,
|
||||
) -> S3Result<Vec<NotificationEventRule>> {
|
||||
let mut event_rules = Vec::new();
|
||||
let invalid_arn = |e: TargetIDError| {
|
||||
S3Error::with_message(S3ErrorCode::InvalidArgument, format!("Invalid ARN in notification configuration: {e}"))
|
||||
};
|
||||
|
||||
process_queue_configurations(
|
||||
&mut event_rules,
|
||||
notification_configuration.queue_configurations.clone(),
|
||||
parse_notification_target_id,
|
||||
)
|
||||
.map_err(invalid_arn)?;
|
||||
process_topic_configurations(
|
||||
&mut event_rules,
|
||||
notification_configuration.topic_configurations.clone(),
|
||||
parse_notification_target_id,
|
||||
)
|
||||
.map_err(invalid_arn)?;
|
||||
process_lambda_configurations(
|
||||
&mut event_rules,
|
||||
notification_configuration.lambda_function_configurations.clone(),
|
||||
parse_notification_target_id,
|
||||
)
|
||||
.map_err(invalid_arn)?;
|
||||
|
||||
Ok(event_rules)
|
||||
}
|
||||
|
||||
fn sr_bucket_meta_item(bucket: String, item_type: &str) -> SRBucketMeta {
|
||||
SRBucketMeta {
|
||||
bucket,
|
||||
@@ -2440,6 +2480,17 @@ impl DefaultBucketUsecase {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let region = resolve_notification_region(self.global_region(), request_region);
|
||||
let notify = current_notify_interface_for_context(self.context.as_deref());
|
||||
let event_rules = build_notification_event_rules(¬ification_configuration)?;
|
||||
|
||||
// Reject the request before the store write so a failure cannot leave a
|
||||
// persisted configuration that the notify runtime refused to activate.
|
||||
notify
|
||||
.validate_event_specific_rules(&bucket, region.as_str(), &event_rules)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))?;
|
||||
|
||||
let data = serialize_config(¬ification_configuration)?;
|
||||
update_bucket_config_for_incarnation(&bucket, BUCKET_NOTIFICATION_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
@@ -2447,40 +2498,10 @@ impl DefaultBucketUsecase {
|
||||
|
||||
notify_bucket_metadata_reload(bucket.clone(), "put bucket notification", request_context, false).await;
|
||||
|
||||
let region = resolve_notification_region(self.global_region(), request_region);
|
||||
let notify = current_notify_interface_for_context(self.context.as_deref());
|
||||
let clear_rules = notify.clear_bucket_notification_rules(&bucket);
|
||||
let parse_rules = async {
|
||||
let mut event_rules = Vec::new();
|
||||
|
||||
process_queue_configurations(&mut event_rules, notification_configuration.queue_configurations.clone(), |arn_str| {
|
||||
ARN::parse(arn_str)
|
||||
.map(|arn| arn.target_id)
|
||||
.map_err(|e| TargetIDError::InvalidFormat(e.to_string()))
|
||||
})?;
|
||||
process_topic_configurations(&mut event_rules, notification_configuration.topic_configurations.clone(), |arn_str| {
|
||||
ARN::parse(arn_str)
|
||||
.map(|arn| arn.target_id)
|
||||
.map_err(|e| TargetIDError::InvalidFormat(e.to_string()))
|
||||
})?;
|
||||
process_lambda_configurations(
|
||||
&mut event_rules,
|
||||
notification_configuration.lambda_function_configurations.clone(),
|
||||
|arn_str| {
|
||||
ARN::parse(arn_str)
|
||||
.map(|arn| arn.target_id)
|
||||
.map_err(|e| TargetIDError::InvalidFormat(e.to_string()))
|
||||
},
|
||||
)?;
|
||||
|
||||
Ok::<_, TargetIDError>(event_rules)
|
||||
};
|
||||
|
||||
let (clear_result, event_rules_result) = tokio::join!(clear_rules, parse_rules);
|
||||
|
||||
clear_result.map_err(|e| s3_error!(InternalError, "Failed to clear rules: {e}"))?;
|
||||
let event_rules =
|
||||
event_rules_result.map_err(|e| s3_error!(InvalidArgument, "Invalid ARN in notification configuration: {e}"))?;
|
||||
notify
|
||||
.clear_bucket_notification_rules(&bucket)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "Failed to clear rules: {e}"))?;
|
||||
warn!("notify event rules: {:?}", &event_rules);
|
||||
notify
|
||||
.add_event_specific_rules(&bucket, region.as_str(), &event_rules)
|
||||
|
||||
@@ -167,6 +167,15 @@ impl NotifyInterface for NotifyHandle {
|
||||
runtime_sources::notify(args).await;
|
||||
}
|
||||
|
||||
async fn validate_event_specific_rules(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> Result<(), NotificationError> {
|
||||
runtime_sources::validate_event_specific_rules(bucket_name, region, event_rules).await
|
||||
}
|
||||
|
||||
async fn add_event_specific_rules(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
|
||||
@@ -73,6 +73,13 @@ pub trait OutboundTlsRuntimeInterface: Send + Sync {
|
||||
pub trait NotifyInterface: Send + Sync {
|
||||
async fn notify(&self, args: EventArgs);
|
||||
|
||||
async fn validate_event_specific_rules(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> Result<(), NotificationError>;
|
||||
|
||||
async fn add_event_specific_rules(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
|
||||
@@ -71,6 +71,14 @@ pub async fn notify(args: EventArgs) {
|
||||
notifier_global::notify(args).await;
|
||||
}
|
||||
|
||||
pub async fn validate_event_specific_rules(
|
||||
bucket_name: &str,
|
||||
region: &str,
|
||||
event_rules: &[(Vec<EventName>, String, String, Vec<TargetID>)],
|
||||
) -> Result<(), NotificationError> {
|
||||
notifier_global::validate_event_specific_rules(bucket_name, region, event_rules).await
|
||||
}
|
||||
|
||||
pub async fn add_event_specific_rules(
|
||||
bucket_name: &str,
|
||||
region: &str,
|
||||
|
||||
@@ -2747,6 +2747,15 @@ mod tests {
|
||||
let _ = self.events.send(args.version_id);
|
||||
}
|
||||
|
||||
async fn validate_event_specific_rules(
|
||||
&self,
|
||||
_bucket_name: &str,
|
||||
_region: &str,
|
||||
_event_rules: &[(Vec<rustfs_targets::EventName>, String, String, Vec<rustfs_targets::arn::TargetID>)],
|
||||
) -> Result<(), rustfs_notify::NotificationError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn add_event_specific_rules(
|
||||
&self,
|
||||
_bucket_name: &str,
|
||||
|
||||
@@ -90,36 +90,42 @@ struct SignedRequest {
|
||||
/// Constructs a real SigV4 fixture using the production crypto primitives.
|
||||
/// S3S verifies both the request authorization and every signed chunk.
|
||||
fn signed_request(payload: &[u8], unsigned_trailer: bool) -> SignedRequest {
|
||||
use rustfs_utils::{hex_sha256, hmac_sha256};
|
||||
|
||||
const DATE: &str = "20130524T000000Z";
|
||||
const SCOPE: &str = "20130524/us-east-1/s3/aws4_request";
|
||||
const REGION: &str = "us-east-1";
|
||||
const SERVICE: &str = "s3";
|
||||
const SECRET_KEY: &str = "test-secret";
|
||||
let mode = if unsigned_trailer {
|
||||
"STREAMING-UNSIGNED-PAYLOAD-TRAILER"
|
||||
} else {
|
||||
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||
};
|
||||
let signed_headers = "host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length";
|
||||
let headers = format!(
|
||||
"host:s3.amazonaws.com\nx-amz-content-sha256:{mode}\nx-amz-date:{DATE}\nx-amz-decoded-content-length:{}\n",
|
||||
payload.len()
|
||||
);
|
||||
let canonical = format!("PUT\n/test-bucket/test-key\npartNumber=1&uploadId=test-upload\n{headers}\n{signed_headers}\n{mode}");
|
||||
let key = hmac_sha256("AWS4test-secret", "20130524");
|
||||
let key = hmac_sha256(key, "us-east-1");
|
||||
let key = hmac_sha256(key, "s3");
|
||||
let key = hmac_sha256(key, "aws4_request");
|
||||
let digest = |data: &[u8]| hex_sha256(data, str::to_owned);
|
||||
let encode = |data: [u8; 32]| hex_simd::encode_to_string(data, hex_simd::AsciiCase::Lower);
|
||||
let seed = encode(hmac_sha256(
|
||||
key,
|
||||
format!("AWS4-HMAC-SHA256\n{DATE}\n{SCOPE}\n{}", digest(canonical.as_bytes())),
|
||||
));
|
||||
let decoded_length = payload.len().to_string();
|
||||
let mut signed_headers = vec![
|
||||
("content-encoding", "aws-chunked"),
|
||||
("host", "s3.amazonaws.com"),
|
||||
("x-amz-content-sha256", mode),
|
||||
("x-amz-date", DATE),
|
||||
("x-amz-decoded-content-length", decoded_length.as_str()),
|
||||
];
|
||||
if unsigned_trailer {
|
||||
signed_headers.push(("x-amz-trailer", "x-amz-checksum-crc32"));
|
||||
}
|
||||
signed_headers.sort_unstable_by(|lhs, rhs| lhs.0.cmp(rhs.0));
|
||||
let signed_header_names = signed_headers.iter().map(|(name, _)| *name).collect::<Vec<_>>().join(";");
|
||||
let query = [("partNumber", "1"), ("uploadId", "test-upload")];
|
||||
let payload_mode = if unsigned_trailer {
|
||||
s3s_sigv4::Payload::UnsignedMultipleChunksWithTrailer
|
||||
} else {
|
||||
s3s_sigv4::Payload::MultipleChunks
|
||||
};
|
||||
let date = s3s_sigv4::AmzDate::parse(DATE).expect("fixture date");
|
||||
let canonical = s3s_sigv4::create_canonical_request("PUT", "/test-bucket/test-key", &query, &signed_headers, payload_mode);
|
||||
let string_to_sign = s3s_sigv4::create_string_to_sign(&canonical, &date, REGION, SERVICE);
|
||||
let seed = s3s_sigv4::calculate_signature(&string_to_sign, SECRET_KEY, &date, REGION, SERVICE);
|
||||
let chunk_signature = |previous: &str, data: &[u8]| {
|
||||
encode(hmac_sha256(
|
||||
key,
|
||||
format!("AWS4-HMAC-SHA256-PAYLOAD\n{DATE}\n{SCOPE}\n{previous}\n{}\n{}", digest(b""), digest(data)),
|
||||
))
|
||||
let string_to_sign = s3s_sigv4::create_chunk_string_to_sign(&date, REGION, SERVICE, previous, &[data]);
|
||||
s3s_sigv4::calculate_signature(&string_to_sign, SECRET_KEY, &date, REGION, SERVICE)
|
||||
};
|
||||
let (prefix, suffix) = if unsigned_trailer {
|
||||
(
|
||||
@@ -147,7 +153,7 @@ fn signed_request(payload: &[u8], unsigned_trailer: bool) -> SignedRequest {
|
||||
.header("x-amz-decoded-content-length", payload.len())
|
||||
.header(
|
||||
"authorization",
|
||||
format!("AWS4-HMAC-SHA256 Credential=test-access/{SCOPE}, SignedHeaders={signed_headers}, Signature={seed}"),
|
||||
format!("AWS4-HMAC-SHA256 Credential=test-access/{SCOPE}, SignedHeaders={signed_header_names}, Signature={seed}"),
|
||||
);
|
||||
if unsigned_trailer {
|
||||
builder = builder.header("x-amz-trailer", "x-amz-checksum-crc32");
|
||||
|
||||
@@ -64,6 +64,7 @@ const SIGNATURE_PATH: &str = "envelope.sig";
|
||||
const RESULT_PATH: &str = "result.json";
|
||||
const MAX_OBJECT_RESPONSE_BYTES: usize = 16_384;
|
||||
const CLEANUP_RESERVE_MAX: Duration = Duration::from_millis(250);
|
||||
const CLEANUP_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
const OUTPUT_MODE: u32 = 0o600;
|
||||
|
||||
static OBJECT_COLLECTOR_ACTIVE: AtomicBool = AtomicBool::new(false);
|
||||
@@ -374,7 +375,7 @@ impl S3ObjectProbe {
|
||||
let mut builder = Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.timeout(timeout);
|
||||
.timeout(timeout.max(CLEANUP_TIMEOUT));
|
||||
if let Some(root_ca_pem) = root_ca_pem {
|
||||
let certificate =
|
||||
reqwest::Certificate::from_pem(root_ca_pem).map_err(|_| ObjectPerformanceError::InvalidRootCertificate)?;
|
||||
@@ -403,7 +404,6 @@ impl S3ObjectProbe {
|
||||
let started = Instant::now();
|
||||
let reserve = CLEANUP_RESERVE_MAX.min(request.duration / 10);
|
||||
let operation_deadline = started + request.duration.saturating_sub(reserve);
|
||||
let cleanup_deadline = started + request.duration;
|
||||
let bucket = format!("rustfs-connect-perf-{}", request.artifact_uid.replace('-', ""));
|
||||
let object = "synthetic-object";
|
||||
let bucket_url = self.object_url(&bucket, None)?;
|
||||
@@ -427,6 +427,9 @@ impl S3ObjectProbe {
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
// Cleanup owns two sequential requests and must not inherit only the
|
||||
// fractional tail of the caller's measurement window.
|
||||
let cleanup_deadline = Instant::now() + CLEANUP_TIMEOUT;
|
||||
let cleanup = self.cleanup(object_url, bucket_url, cleanup_deadline).await;
|
||||
if cleanup.is_err() {
|
||||
return Err(ObjectProbeError::CleanupFailed);
|
||||
|
||||
@@ -531,6 +531,22 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn writable_node_readiness_keeps_200_for_transient_pool_metadata_timeout() {
|
||||
let mut report = ready_report();
|
||||
report.degraded_reasons = vec![ReadinessDegradedReason::PoolMetadataCheckTimeout];
|
||||
|
||||
with_var(rustfs_config::ENV_HEALTH_MINIMAL_RESPONSE_ENABLE, Some("false"), || {
|
||||
let parts = build_health_response_parts(Method::GET, HealthProbe::Readiness, Some(&report), "rustfs", None, None);
|
||||
|
||||
assert_eq!(parts.status_code, StatusCode::OK);
|
||||
let payload = parts.payload.expect("GET readiness body");
|
||||
assert_eq!(payload["ready"], true);
|
||||
assert_eq!(payload["degradedReasons"], json!(["pool_metadata_check_timeout"]));
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn readiness_collects_object_stalls_and_recovers_on_completion() {
|
||||
let object_traffic_health = ObjectTrafficHealth::enabled_for_test(Duration::ZERO);
|
||||
|
||||
@@ -38,7 +38,7 @@ use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
sync::OnceLock,
|
||||
sync::{Mutex as StdMutex, OnceLock},
|
||||
time::Instant,
|
||||
};
|
||||
use tokio::sync::Mutex;
|
||||
@@ -75,6 +75,7 @@ fn startup_runtime_readiness_max_wait() -> Duration {
|
||||
}
|
||||
const METRIC_RUNTIME_READINESS_READY: &str = "rustfs_runtime_readiness_ready";
|
||||
const METRIC_RUNTIME_READINESS_DEGRADED_TOTAL: &str = "rustfs_runtime_readiness_degraded_total";
|
||||
const METRIC_POOL_METADATA_CHECK_TIMEOUT_TOTAL: &str = "rustfs_pool_metadata_check_timeouts_total";
|
||||
|
||||
pub use crate::shared_types::{DependencyReadiness, DependencyReadinessReport, ReadinessDegradedReason, StorageReadinessDetails};
|
||||
|
||||
@@ -266,6 +267,66 @@ struct StorageWriteReadinessStatus {
|
||||
pool_metadata_reason: Option<ReadinessDegradedReason>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
struct NodePoolMetadataReadinessCache {
|
||||
entry: Option<NodePoolMetadataReadinessCacheEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct NodePoolMetadataReadinessCacheEntry {
|
||||
cached_at: Instant,
|
||||
observed_at: Instant,
|
||||
status: StorageWriteReadinessStatus,
|
||||
}
|
||||
|
||||
impl NodePoolMetadataReadinessCache {
|
||||
fn observe(
|
||||
&mut self,
|
||||
observed: StorageWriteReadinessStatus,
|
||||
observed_at: Instant,
|
||||
now: Instant,
|
||||
ttl: Duration,
|
||||
) -> StorageWriteReadinessStatus {
|
||||
if ttl.is_zero() {
|
||||
self.entry = None;
|
||||
return observed;
|
||||
}
|
||||
|
||||
if observed.pool_metadata_reason == Some(ReadinessDegradedReason::PoolMetadataCheckTimeout) {
|
||||
let Some(previous) = self.entry else {
|
||||
return observed;
|
||||
};
|
||||
if now.saturating_duration_since(previous.cached_at) > ttl {
|
||||
return observed;
|
||||
}
|
||||
|
||||
// A confirmed write block must never be hidden by a later timeout.
|
||||
if previous.status.pool_metadata_reason == Some(ReadinessDegradedReason::PoolMetaWriteBlocked) {
|
||||
return previous.status;
|
||||
}
|
||||
|
||||
// A transient inspection outage is diagnostic, not proof that the
|
||||
// writer became unsafe. Keep the last confirmed writable state.
|
||||
if previous.status.ready {
|
||||
return StorageWriteReadinessStatus {
|
||||
ready: true,
|
||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
||||
};
|
||||
}
|
||||
return observed;
|
||||
}
|
||||
|
||||
if self.entry.is_none_or(|entry| entry.observed_at <= observed_at) {
|
||||
self.entry = Some(NodePoolMetadataReadinessCacheEntry {
|
||||
cached_at: now,
|
||||
observed_at,
|
||||
status: observed,
|
||||
});
|
||||
}
|
||||
observed
|
||||
}
|
||||
}
|
||||
|
||||
fn pool_metadata_write_readiness(result: Result<(), StorageError>) -> StorageWriteReadinessStatus {
|
||||
match result {
|
||||
Ok(()) => StorageWriteReadinessStatus {
|
||||
@@ -342,6 +403,11 @@ fn health_cluster_timeout() -> Duration {
|
||||
)
|
||||
}
|
||||
|
||||
fn node_pool_metadata_readiness_cache() -> &'static StdMutex<NodePoolMetadataReadinessCache> {
|
||||
static CACHE: OnceLock<StdMutex<NodePoolMetadataReadinessCache>> = OnceLock::new();
|
||||
CACHE.get_or_init(|| StdMutex::new(NodePoolMetadataReadinessCache::default()))
|
||||
}
|
||||
|
||||
fn storage_readiness_cache() -> &'static Mutex<Option<StorageReadinessCacheEntry>> {
|
||||
static CACHE: OnceLock<Mutex<Option<StorageReadinessCacheEntry>>> = OnceLock::new();
|
||||
CACHE.get_or_init(|| Mutex::new(None))
|
||||
@@ -446,6 +512,26 @@ async fn update_storage_readiness_cache(status: StorageWriteReadinessStatus) {
|
||||
});
|
||||
}
|
||||
|
||||
fn apply_node_pool_metadata_timeout_policy(
|
||||
observed: StorageWriteReadinessStatus,
|
||||
observed_at: Instant,
|
||||
) -> StorageWriteReadinessStatus {
|
||||
if observed.pool_metadata_reason == Some(ReadinessDegradedReason::PoolMetadataCheckTimeout) {
|
||||
counter!(METRIC_POOL_METADATA_CHECK_TIMEOUT_TOTAL).increment(1);
|
||||
}
|
||||
let mut cache = node_pool_metadata_readiness_cache()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
cache.observe(observed, observed_at, Instant::now(), health_readiness_cache_ttl())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn reset_node_pool_metadata_readiness_cache() {
|
||||
*node_pool_metadata_readiness_cache()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = NodePoolMetadataReadinessCache::default();
|
||||
}
|
||||
|
||||
async fn load_cached_lock_quorum_status() -> Option<LockQuorumObservation> {
|
||||
let ttl = health_readiness_cache_ttl();
|
||||
if ttl.is_zero() {
|
||||
@@ -724,6 +810,9 @@ fn record_readiness_report(report: &DependencyReadinessReport) {
|
||||
&& report.readiness.peer_health_ready;
|
||||
gauge!(METRIC_RUNTIME_READINESS_READY).set(if ready { 1.0 } else { 0.0 });
|
||||
for reason in &report.degraded_reasons {
|
||||
if ready && *reason == ReadinessDegradedReason::PoolMetadataCheckTimeout {
|
||||
continue;
|
||||
}
|
||||
counter!(METRIC_RUNTIME_READINESS_DEGRADED_TOTAL, "reason" => reason.as_str()).increment(1);
|
||||
}
|
||||
}
|
||||
@@ -788,7 +877,8 @@ pub async fn collect_node_readiness_report() -> DependencyReadinessReport {
|
||||
let mut details = StorageReadinessDetails::default();
|
||||
let mut storage_check_timed_out = false;
|
||||
if let Some(store) = runtime_sources::current_object_store_handle() {
|
||||
storage = pool_metadata_write_readiness(store.pool_meta_write_status().await);
|
||||
let pool_metadata_status = store.pool_meta_write_status().await;
|
||||
storage = apply_node_pool_metadata_timeout_policy(pool_metadata_write_readiness(pool_metadata_status), Instant::now());
|
||||
details.pool_metadata_write_ready = storage.ready;
|
||||
match node_storage_snapshot(store.as_ref(), &lock_observation.online_hosts).await {
|
||||
Ok(info) => {
|
||||
@@ -1172,6 +1262,8 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api::server::readiness::{DiskOption, new_disk};
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_madmin::{BackendInfo, Disk};
|
||||
use serial_test::serial;
|
||||
use std::future;
|
||||
@@ -2276,6 +2368,207 @@ mod tests {
|
||||
assert_eq!(report.degraded_reasons[0].as_str(), "pool_metadata_check_timeout");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn sticky_pool_metadata_timeout_uses_dedicated_metric_without_degraded_count() {
|
||||
reset_node_pool_metadata_readiness_cache();
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
with_var(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"), || {
|
||||
let writable = StorageWriteReadinessStatus {
|
||||
ready: true,
|
||||
pool_metadata_reason: None,
|
||||
};
|
||||
assert_eq!(apply_node_pool_metadata_timeout_policy(writable, Instant::now()), writable);
|
||||
let sticky = apply_node_pool_metadata_timeout_policy(
|
||||
StorageWriteReadinessStatus {
|
||||
ready: false,
|
||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
||||
},
|
||||
Instant::now(),
|
||||
);
|
||||
assert!(sticky.ready);
|
||||
record_readiness_report(&DependencyReadinessReport {
|
||||
readiness: DependencyReadiness {
|
||||
storage_ready: true,
|
||||
iam_ready: true,
|
||||
lock_quorum_ready: true,
|
||||
peer_health_ready: true,
|
||||
},
|
||||
degraded_reasons: vec![ReadinessDegradedReason::PoolMetadataCheckTimeout],
|
||||
storage_details: None,
|
||||
});
|
||||
});
|
||||
});
|
||||
reset_node_pool_metadata_readiness_cache();
|
||||
|
||||
let entries = snapshotter.snapshot().into_vec();
|
||||
let ready = entries.iter().find_map(|(composite, _, _, value)| {
|
||||
(composite.kind() == MetricKind::Gauge && composite.key().name() == METRIC_RUNTIME_READINESS_READY).then_some(value)
|
||||
});
|
||||
assert!(matches!(ready, Some(DebugValue::Gauge(value)) if value.into_inner() == 1.0));
|
||||
|
||||
let degraded = entries.iter().find_map(|(composite, _, _, value)| {
|
||||
(composite.kind() == MetricKind::Counter
|
||||
&& composite.key().name() == METRIC_RUNTIME_READINESS_DEGRADED_TOTAL
|
||||
&& composite
|
||||
.key()
|
||||
.labels()
|
||||
.any(|label| label.key() == "reason" && label.value() == "pool_metadata_check_timeout"))
|
||||
.then_some(value)
|
||||
});
|
||||
assert!(degraded.is_none());
|
||||
|
||||
let timeout = entries.iter().find_map(|(composite, _, _, value)| {
|
||||
(composite.kind() == MetricKind::Counter && composite.key().name() == METRIC_POOL_METADATA_CHECK_TIMEOUT_TOTAL)
|
||||
.then_some(value)
|
||||
});
|
||||
assert!(matches!(timeout, Some(DebugValue::Counter(1))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn node_pool_metadata_timeout_preserves_recent_writable_observation() {
|
||||
reset_node_pool_metadata_readiness_cache();
|
||||
async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"))], async {
|
||||
let writable = StorageWriteReadinessStatus {
|
||||
ready: true,
|
||||
pool_metadata_reason: None,
|
||||
};
|
||||
assert_eq!(apply_node_pool_metadata_timeout_policy(writable, Instant::now()), writable);
|
||||
|
||||
let timed_out = apply_node_pool_metadata_timeout_policy(
|
||||
StorageWriteReadinessStatus {
|
||||
ready: false,
|
||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
||||
},
|
||||
Instant::now(),
|
||||
);
|
||||
|
||||
assert!(timed_out.ready);
|
||||
assert_eq!(timed_out.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetadataCheckTimeout));
|
||||
})
|
||||
.await;
|
||||
reset_node_pool_metadata_readiness_cache();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn node_pool_metadata_timeout_fails_closed_with_empty_cache() {
|
||||
reset_node_pool_metadata_readiness_cache();
|
||||
async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"))], async {
|
||||
let timed_out = apply_node_pool_metadata_timeout_policy(
|
||||
StorageWriteReadinessStatus {
|
||||
ready: false,
|
||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
||||
},
|
||||
Instant::now(),
|
||||
);
|
||||
|
||||
assert!(!timed_out.ready);
|
||||
assert_eq!(timed_out.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetadataCheckTimeout));
|
||||
})
|
||||
.await;
|
||||
reset_node_pool_metadata_readiness_cache();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_pool_metadata_timeout_fails_closed_after_cache_ttl() {
|
||||
let mut cache = NodePoolMetadataReadinessCache::default();
|
||||
let observed_at = Instant::now();
|
||||
let ttl = Duration::from_secs(60);
|
||||
let writable = StorageWriteReadinessStatus {
|
||||
ready: true,
|
||||
pool_metadata_reason: None,
|
||||
};
|
||||
assert_eq!(cache.observe(writable, observed_at, observed_at, ttl), writable);
|
||||
|
||||
let timed_out = cache.observe(
|
||||
StorageWriteReadinessStatus {
|
||||
ready: false,
|
||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
||||
},
|
||||
observed_at + Duration::from_secs(61),
|
||||
observed_at + Duration::from_secs(61),
|
||||
ttl,
|
||||
);
|
||||
|
||||
assert!(!timed_out.ready);
|
||||
assert_eq!(timed_out.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetadataCheckTimeout));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn node_pool_metadata_timeout_never_hides_confirmed_block() {
|
||||
reset_node_pool_metadata_readiness_cache();
|
||||
async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"))], async {
|
||||
let blocked = StorageWriteReadinessStatus {
|
||||
ready: false,
|
||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetaWriteBlocked),
|
||||
};
|
||||
let newer_observation = Instant::now();
|
||||
assert_eq!(apply_node_pool_metadata_timeout_policy(blocked, newer_observation), blocked);
|
||||
assert_eq!(
|
||||
apply_node_pool_metadata_timeout_policy(
|
||||
StorageWriteReadinessStatus {
|
||||
ready: true,
|
||||
pool_metadata_reason: None,
|
||||
},
|
||||
newer_observation
|
||||
.checked_sub(Duration::from_secs(1))
|
||||
.expect("test instant has elapsed time"),
|
||||
),
|
||||
StorageWriteReadinessStatus {
|
||||
ready: true,
|
||||
pool_metadata_reason: None,
|
||||
}
|
||||
);
|
||||
|
||||
let timed_out = apply_node_pool_metadata_timeout_policy(
|
||||
StorageWriteReadinessStatus {
|
||||
ready: false,
|
||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
||||
},
|
||||
Instant::now(),
|
||||
);
|
||||
|
||||
assert_eq!(timed_out, blocked);
|
||||
})
|
||||
.await;
|
||||
reset_node_pool_metadata_readiness_cache();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn node_pool_metadata_timeout_fails_closed_without_fresh_observation() {
|
||||
reset_node_pool_metadata_readiness_cache();
|
||||
async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("60000"))], async {
|
||||
let writable = StorageWriteReadinessStatus {
|
||||
ready: true,
|
||||
pool_metadata_reason: None,
|
||||
};
|
||||
assert_eq!(apply_node_pool_metadata_timeout_policy(writable, Instant::now()), writable);
|
||||
|
||||
async_with_vars([(rustfs_config::ENV_HEALTH_READINESS_CACHE_TTL_MS, Some("0"))], async {
|
||||
let timed_out = apply_node_pool_metadata_timeout_policy(
|
||||
StorageWriteReadinessStatus {
|
||||
ready: false,
|
||||
pool_metadata_reason: Some(ReadinessDegradedReason::PoolMetadataCheckTimeout),
|
||||
},
|
||||
Instant::now(),
|
||||
);
|
||||
|
||||
assert!(!timed_out.ready);
|
||||
assert_eq!(timed_out.pool_metadata_reason, Some(ReadinessDegradedReason::PoolMetadataCheckTimeout));
|
||||
})
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
reset_node_pool_metadata_readiness_cache();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degraded_reasons_report_pool_meta_write_blocked() {
|
||||
let readiness = DependencyReadiness {
|
||||
|
||||
@@ -245,6 +245,26 @@ pub(crate) trait TableCatalogStore: Send + Sync {
|
||||
|
||||
async fn list_all_tables(&self, table_bucket: &str) -> TableCatalogStoreResult<Vec<TableEntry>>;
|
||||
|
||||
/// Checks whether an active table already owns any part of the candidate warehouse location.
|
||||
///
|
||||
/// This is a preflight for clients that write data before catalog registration. Registration
|
||||
/// remains the authoritative atomic check for concurrent creators.
|
||||
async fn ensure_table_warehouse_location_available(&self, candidate: &TableEntry) -> TableCatalogStoreResult<()> {
|
||||
let candidate_prefix = table_warehouse_object_prefix(candidate)?;
|
||||
for existing in self.list_all_tables(&candidate.table_bucket).await? {
|
||||
if existing.state != TableCatalogEntryState::Active || existing.table_id == candidate.table_id {
|
||||
continue;
|
||||
}
|
||||
let existing_prefix = table_warehouse_object_prefix(&existing)?;
|
||||
if warehouse_object_prefixes_overlap(&existing_prefix, &candidate_prefix) {
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"table warehouse location overlaps an active table: {candidate_prefix}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_tables_page(
|
||||
&self,
|
||||
table_bucket: &str,
|
||||
@@ -1242,6 +1262,13 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn ensure_table_warehouse_location_available(&self, candidate: &TableEntry) -> TableCatalogStoreResult<()> {
|
||||
match self {
|
||||
Self::ObjectBacked(store) => store.ensure_table_warehouse_location_available(candidate).await,
|
||||
Self::DurableStrong(store) => store.ensure_table_warehouse_location_available(candidate).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_tables_page(
|
||||
&self,
|
||||
table_bucket: &str,
|
||||
|
||||
@@ -5168,6 +5168,10 @@ where
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
async fn ensure_table_warehouse_location_available(&self, candidate: &TableEntry) -> TableCatalogStoreResult<()> {
|
||||
self.ensure_table_warehouse_prefix_available(candidate).await
|
||||
}
|
||||
|
||||
async fn list_tables_page(
|
||||
&self,
|
||||
table_bucket: &str,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user