mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 20:06:37 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0c3d26f208 | |||
| ed07f2dc03 |
@@ -1,2 +0,0 @@
|
||||
sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
|
||||
sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
|
||||
@@ -1 +0,0 @@
|
||||
sha256=9b9bc336b43b70d0e06e0adb5455bf035bb18945d85d60936eb6fe4d48e0e680
|
||||
@@ -1,2 +0,0 @@
|
||||
sha256-darwin=55534a97fbd376f64c8f6c341d319017d11ff77cad6da8629a1a7f6a874e0315
|
||||
sha256-linux=c06fb8c19aed6f388b9dc61cb8251b7a44f8561a9bf764ad2b9e635598f8dc17
|
||||
@@ -1 +0,0 @@
|
||||
sha256=655a3f3c1d042e694339d15caba7580518320322d1bac0f09450b37e6c09e2e7
|
||||
@@ -1 +0,0 @@
|
||||
sha256=ec27cde6ce6400723c4b372bfbd2ac61709c744294e4810af765e8a808d8e31d
|
||||
@@ -75,11 +75,6 @@ embedded-secrets-check: ## Check no private key material or credential literal i
|
||||
@echo "🔑 Checking embedded secret material guard..."
|
||||
./scripts/check_embedded_secrets.sh
|
||||
|
||||
.PHONY: test-wiring-check
|
||||
test-wiring-check: ## Check tests stay registered and selected by their intended runners
|
||||
@echo "🧪 Checking test wiring..."
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
||||
@echo "🩺 Checking log-analyzer rule anchors..."
|
||||
|
||||
@@ -19,13 +19,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
@echo "✅ Fast development checks passed!"
|
||||
|
||||
@@ -35,8 +35,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_pinned_paired_abba_bench.sh
|
||||
./scripts/test_manual_transition_runbooks.sh
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/s3-tests/test_report_compat.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
|
||||
|
||||
+14
-48
@@ -38,11 +38,10 @@ e2e-vault = { max-threads = 1 }
|
||||
# replacement_privileged_e2e_test when explicitly run as root on Linux). They
|
||||
# are correct in isolation but resource-heavy; serialize them under nextest's
|
||||
# process boundary (serial_test's #[serial] does not cross it) so several 4-disk
|
||||
# servers never run at once. The e2e-full merge/main lane picks these up;
|
||||
# servers never run at once. ci-7's nightly picks these up via the e2e suite;
|
||||
# they are deliberately NOT in the fast PR `e2e-smoke` filter.
|
||||
e2e-reliability = { max-threads = 1 }
|
||||
e2e-inline-boundaries = { max-threads = 1 }
|
||||
e2e-cluster-nightly = { max-threads = 1 }
|
||||
|
||||
# --- default profile (local): serialize the flaky groups, never retry --------
|
||||
[[profile.default.overrides]]
|
||||
@@ -162,7 +161,7 @@ retries = 2
|
||||
# Serialize the 4-disk reliability / degraded-read e2e tests under the ci
|
||||
# profile too (see the e2e-reliability test-group note near the top). Not a
|
||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||
# run concurrently when e2e-full runs the suite.
|
||||
# run concurrently when ci-7's nightly runs the full e2e suite.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
@@ -231,8 +230,8 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the nightly profile derives its set as "the replication module MINUS this
|
||||
# allowlist", so any new replication test lands in nightly by default (never
|
||||
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
||||
# regexes byte-identical. The committed profile selection digests make changes
|
||||
# visible in CI; current counts live in docs/testing/e2e-suite-inventory.md.
|
||||
# regexes byte-identical. Count invariant: 20 here + 49 nightly = 69 total
|
||||
# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
|
||||
# HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
|
||||
# (#4724) because they set a loopback (127.0.0.1) replication target that the
|
||||
# SSRF egress guard rejected on every PR after repl-1 (#4712). That is fixed —
|
||||
@@ -328,8 +327,9 @@ slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||
# the STS dual-node test actually exercises its path (it skips gracefully with
|
||||
# a visible log line when awscurl is absent), and routes scheduled failures
|
||||
# through .github/actions/schedule-failure-issue (ci-8). Explicit division of
|
||||
# labor with e2e-full: these tests run only in the consolidated nightly
|
||||
# workflow, not in the merge/main lane.
|
||||
# labor with ci-5's future e2e-full merge gate: these tests run ONLY here, not
|
||||
# double-run there. TODO(ci-7): fold this interim repl-owned lane into the ci
|
||||
# domain's consolidated scheduled e2e workflow once it exists.
|
||||
[profile.e2e-repl-nightly]
|
||||
default-filter = """
|
||||
package(e2e_test)
|
||||
@@ -343,60 +343,26 @@ fail-fast = false
|
||||
# workflow as the failure-triage artifact.
|
||||
path = "junit.xml"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-nightly profile — destructive multi-process cluster fault domains
|
||||
# ---------------------------------------------------------------------------
|
||||
# These seven modules are deliberately outside e2e-full's merge budget. Each
|
||||
# starts a real multi-process or multi-disk topology and exercises node/disk
|
||||
# loss, quorum, cleanup, notification fan-in, or admin-timeout behavior. The
|
||||
# consolidated nightly workflow runs them serially to avoid resource
|
||||
# starvation; failures are never retried.
|
||||
[profile.e2e-nightly]
|
||||
default-filter = """
|
||||
package(e2e_test)
|
||||
& test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
|
||||
"""
|
||||
fail-fast = false
|
||||
|
||||
[profile.e2e-nightly.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
[[profile.e2e-nightly.overrides]]
|
||||
filter = 'package(e2e_test)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-protocols profile — serial protocol lane
|
||||
# ---------------------------------------------------------------------------
|
||||
# The suite owns fixed ports, so the nightly workflow runs this exact profile
|
||||
# with one nextest worker.
|
||||
[profile.e2e-protocols]
|
||||
default-filter = 'package(e2e_test) & test(/^protocols::/)'
|
||||
fail-fast = false
|
||||
|
||||
[profile.e2e-protocols.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-full profile — merge-gate full single-node e2e lane (backlog#1149 ci-5)
|
||||
# ---------------------------------------------------------------------------
|
||||
# The merge gate (ci.yml `e2e-full` job: push main + merge_group +
|
||||
# workflow_dispatch). Runs the user-visible KMS, object-lock, multipart-auth,
|
||||
# quota, checksum, encryption,
|
||||
# workflow_dispatch). Runs the never-automated user-visible suites — KMS (40),
|
||||
# object_lock (33), multipart_auth (109), quota, checksum, encryption,
|
||||
# security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately
|
||||
# skips. Budget <= 45 min; authority for the suite count is `cargo nextest list
|
||||
# --profile e2e-full` (see docs/testing/e2e-suite-inventory.md).
|
||||
#
|
||||
# The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
|
||||
# * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile
|
||||
# with one worker because the suite owns fixed ports.
|
||||
# * protocols:: — FTPS/SFTP/WebDAV, still pinned to --test-threads=1 by fixed
|
||||
# ports; they join a scheduled lane once ci-6 randomises the ports (ci-7).
|
||||
# * the 7 cluster suites that spin up a RustFSTestClusterEnvironment
|
||||
# (cluster_concurrency, cluster_multidrive_pool, stale_multipart_cleanup_cluster,
|
||||
# namespace_lock_quorum, heal_erasure_disk_rebuild, admin_timeout_regression,
|
||||
# object_lambda) — too heavy for the merge budget; they run in the
|
||||
# e2e-nightly serial cluster-fault lane.
|
||||
# object_lambda) — too heavy for the merge budget; they run in ci-7's
|
||||
# nightly 4-node lane.
|
||||
# * replication_extension_test — repl-1 already splits it into the PR
|
||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (55 slow) lanes and reserves
|
||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (49 slow) lanes and reserves
|
||||
# it for those, so e2e-full does not double-run it.
|
||||
# * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
|
||||
# manual-localhost:9000 reliant/policy tests are ci-13's migration.
|
||||
|
||||
@@ -46,9 +46,10 @@ lists when upstream changes.
|
||||
the PR.
|
||||
- **Weekly + manual**: `.github/workflows/e2e-s3tests.yml` runs the full
|
||||
upstream suite (`TEST_SCOPE=all`) against a Docker deployment (single node
|
||||
or a 4-node distributed cluster behind HAProxy). The canonical gate policy
|
||||
and compatibility-report behavior are documented in
|
||||
[`scripts/s3-tests/README.md`](../../scripts/s3-tests/README.md).
|
||||
or a 4-node distributed cluster behind HAProxy). It fails only on
|
||||
regressions in the implemented whitelist and publishes a classification
|
||||
report (`compat-report.md`, also shown in the job summary) listing promotion
|
||||
candidates and unclassified tests.
|
||||
|
||||
## Running Tests Locally
|
||||
|
||||
|
||||
@@ -53,9 +53,9 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install ripgrep
|
||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
||||
with:
|
||||
tool: ripgrep@15.2.0
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ripgrep
|
||||
|
||||
- name: Check architecture migration rules
|
||||
run: ./scripts/check_architecture_migration_rules.sh
|
||||
|
||||
@@ -83,9 +83,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install ripgrep
|
||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
||||
with:
|
||||
tool: ripgrep@15.2.0
|
||||
run: sudo apt-get update && sudo apt-get install -y ripgrep
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
@@ -125,9 +123,6 @@ jobs:
|
||||
- name: Check no embedded secret material
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Check test wiring
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
|
||||
@@ -118,9 +118,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install ripgrep
|
||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
||||
with:
|
||||
tool: ripgrep@15.2.0
|
||||
run: sudo apt-get update && sudo apt-get install -y ripgrep
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
@@ -160,9 +158,6 @@ jobs:
|
||||
- name: Check no embedded secret material
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Check test wiring
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
@@ -689,9 +684,9 @@ jobs:
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
# Build the e2e test graph once. The archive is reused by the smoke
|
||||
# selection guard, security exact-count check, and run below, avoiding a
|
||||
# second compile of the same e2e_test target on cold runners (backlog#1645).
|
||||
# Build the e2e test graph once. The archive is reused by the security
|
||||
# count-floor check and the smoke run below, avoiding a second compile of
|
||||
# the same e2e_test target on cold runners (backlog#1645).
|
||||
- name: Archive e2e smoke test binaries
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
@@ -699,7 +694,6 @@ jobs:
|
||||
run: |
|
||||
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||
@@ -764,7 +758,7 @@ jobs:
|
||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
||||
# dispatch. protocols / the 6 cluster suites / replication / #[ignore] are
|
||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
@@ -824,13 +818,6 @@ jobs:
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
- name: Verify e2e full membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-full -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-full "${NEXTEST_LISTING}"
|
||||
|
||||
# Full single-node e2e lane (backlog#1149 ci-5). The e2e-full
|
||||
# default-filter in .config/nextest.toml is the single wiring mechanism —
|
||||
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
|
||||
@@ -843,9 +830,7 @@ jobs:
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-full-junit-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-full/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
path: target/nextest/e2e-full/junit.xml
|
||||
retention-days: 7
|
||||
|
||||
e2e-tests-rio-v2:
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Consolidated nightly e2e lane for replication, cluster faults, and protocols.
|
||||
# Nightly full replication e2e lane (backlog#1147 repl-1, deps: ci-4).
|
||||
#
|
||||
# The per-PR gate (ci.yml `e2e-tests` job, `--profile e2e-smoke`) runs the
|
||||
# FAST replication tests. This scheduled lane runs the remaining heavier
|
||||
@@ -28,12 +28,15 @@
|
||||
# add ad-hoc cargo-test steps here; change the filterset instead. The
|
||||
# authoritative membership and count come from
|
||||
# `cargo nextest list -p e2e_test --profile e2e-repl-nightly`; the PR/nightly
|
||||
# selection digest is committed under .config/.
|
||||
# count invariant is maintained next to the filtersets in .config/nextest.toml
|
||||
# (deliberately not duplicated here).
|
||||
#
|
||||
# Explicit division of labor: these subsets run only here and never double-run
|
||||
# in the e2e-full merge gate.
|
||||
# Explicit division of labor: the nightly subset runs ONLY here, never double-run
|
||||
# in ci-5's future e2e-full merge gate. TODO(ci-7): once the ci domain's
|
||||
# consolidated scheduled e2e workflow exists, fold this interim repl-owned lane
|
||||
# into it rather than growing a second scheduled entrypoint.
|
||||
|
||||
name: e2e-nightly
|
||||
name: e2e-replication-nightly
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -47,10 +50,6 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
repl-nightly:
|
||||
name: Replication e2e (nightly)
|
||||
@@ -98,20 +97,9 @@ jobs:
|
||||
# demand otherwise, but a single explicit build avoids several parallel
|
||||
# nextest test processes racing to build it at once.
|
||||
- name: Build rustfs binary
|
||||
run: |
|
||||
cargo build -p rustfs --bins
|
||||
: > target/debug/rustfs.features
|
||||
|
||||
- name: Verify replication e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-repl-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-repl-nightly "${NEXTEST_LISTING}"
|
||||
run: cargo build -p rustfs --bins
|
||||
|
||||
- name: Run replication e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
run: cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
|
||||
- name: Upload nextest junit report
|
||||
@@ -119,112 +107,13 @@ jobs:
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-repl-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-logs/
|
||||
path: target/nextest/e2e-repl-nightly/junit.xml
|
||||
retention-days: 7
|
||||
if-no-files-found: ignore
|
||||
|
||||
cluster-nightly:
|
||||
name: Cluster fault e2e (nightly)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-e2e-nightly
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Build rustfs binary
|
||||
run: |
|
||||
cargo build -p rustfs --bins --features e2e-test-hooks
|
||||
: > target/debug/rustfs.features
|
||||
|
||||
- name: Verify cluster fault e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-nightly -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-nightly "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run cluster fault e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
run: cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
- name: Upload cluster fault diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
${{ runner.temp }}/rustfs-e2e-nightly-logs/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
protocols-nightly:
|
||||
name: Protocol e2e (nightly)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-e2e-protocols
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# The suite owns fixed protocol ports and serializes its internal cases.
|
||||
- name: Verify protocol e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-protocols -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-protocols "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run protocol e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
run: >-
|
||||
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
|
||||
|
||||
- name: Upload protocol diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-protocols/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
||||
${{ runner.temp }}/rustfs-protocol-e2e-logs/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
needs: [repl-nightly]
|
||||
# Only scheduled runs open/append the tracking issue (backlog#1149 ci-8);
|
||||
# manual workflow_dispatch runs stay quiet so a debugging run never files a
|
||||
# spurious alert.
|
||||
|
||||
@@ -18,9 +18,10 @@
|
||||
# runs only the implemented_tests.txt whitelist. This workflow complements it:
|
||||
#
|
||||
# - Scheduled weekly full sweep (TEST_SCOPE=all): runs the ENTIRE upstream
|
||||
# suite and reports promotion candidates. Regressions, unclassified tests,
|
||||
# incomplete execution, and infrastructure errors fail the job; classified
|
||||
# failures for not-yet-implemented features remain informational.
|
||||
# suite and reports promotion candidates (tests that newly pass) and
|
||||
# unclassified tests. The job fails only on regressions in the implemented
|
||||
# whitelist or on infrastructure errors — expected failures from
|
||||
# not-yet-implemented features do not turn the run red.
|
||||
# - Manual runs (workflow_dispatch): same, with configurable mode/scope.
|
||||
#
|
||||
# All test execution is delegated to scripts/s3-tests/run.sh (single source of
|
||||
@@ -44,6 +45,13 @@
|
||||
# The PR gate (ci.yml s3-implemented-tests) is unaffected: it avoids Docker
|
||||
# via DEPLOY_MODE=binary and defers all pip setup to run.sh's self-bootstrap.
|
||||
|
||||
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||
# reading this file, which has already misled at least one audit — hence this
|
||||
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||
#
|
||||
name: e2e-s3tests
|
||||
|
||||
on:
|
||||
@@ -73,19 +81,6 @@ on:
|
||||
description: "Stop after N failures. '0' to run everything."
|
||||
required: false
|
||||
default: "0"
|
||||
shard-count:
|
||||
description: "Exact-node-ID shard count for a targeted manual run"
|
||||
required: false
|
||||
default: "1"
|
||||
type: choice
|
||||
options:
|
||||
- "1"
|
||||
- "2"
|
||||
- "4"
|
||||
shard-index:
|
||||
description: "Zero-based shard index for a targeted manual run"
|
||||
required: false
|
||||
default: "0"
|
||||
markexpr:
|
||||
description: "Optional pytest -m expression"
|
||||
required: false
|
||||
@@ -116,8 +111,6 @@ env:
|
||||
XDIST: ${{ github.event.inputs.xdist || '4' }}
|
||||
MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
|
||||
MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
|
||||
S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }}
|
||||
TEST_TIMEOUT: "300"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs['test-mode'] || 'single' }}
|
||||
@@ -134,22 +127,19 @@ defaults:
|
||||
|
||||
jobs:
|
||||
s3tests:
|
||||
name: s3tests (${{ matrix.test-mode }}, shard ${{ matrix.shard-index }})
|
||||
# GitHub-hosted: reliably provides Docker + docker compose + python3/pip.
|
||||
# See the header note (ci-1) for why the self-hosted sm-standard-4 label
|
||||
# was abandoned. Scheduled failures are handled by alert-on-failure below.
|
||||
# was abandoned. TODO(ci-8): scheduled-failure alerting (auto-open issue)
|
||||
# is added by the ci-8 composite action; do not implement it here.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 2
|
||||
matrix:
|
||||
# Scheduled sweeps cover both topologies; manual runs use the input.
|
||||
test-mode: ${{ github.event_name == 'schedule' && fromJSON('["single", "multi"]') || fromJSON(format('["{0}"]', github.event.inputs.test-mode || 'single')) }}
|
||||
shard-index: ${{ github.event_name == 'schedule' && fromJSON('[0, 1, 2, 3]') || fromJSON(format('[{0}]', github.event.inputs.shard-index || '0')) }}
|
||||
env:
|
||||
TEST_MODE: ${{ matrix.test-mode }}
|
||||
S3_SHARD_INDEX: ${{ matrix.shard-index }}
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
@@ -191,7 +181,6 @@ jobs:
|
||||
- name: Start single RustFS
|
||||
if: env.TEST_MODE == 'single'
|
||||
run: |
|
||||
SSE_KEY="$(head -c 32 /dev/zero | base64 -w0)"
|
||||
docker network inspect rustfs-net >/dev/null 2>&1 || docker network create rustfs-net
|
||||
docker rm -f rustfs-single >/dev/null 2>&1 || true
|
||||
# The four disks share one physical device on the runner (a single
|
||||
@@ -204,7 +193,6 @@ jobs:
|
||||
-e RUSTFS_ADDRESS=0.0.0.0:9000 \
|
||||
-e RUSTFS_ACCESS_KEY="${S3_ACCESS_KEY}" \
|
||||
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
|
||||
-e RUSTFS_SSE_S3_MASTER_KEY="${SSE_KEY}" \
|
||||
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-v /tmp/rustfs-single:/data \
|
||||
@@ -213,7 +201,6 @@ jobs:
|
||||
- name: Start 4-node distributed cluster
|
||||
if: env.TEST_MODE == 'multi'
|
||||
run: |
|
||||
SSE_KEY="$(head -c 32 /dev/zero | base64 -w0)"
|
||||
# A real distributed deployment: every node lists all endpoints in
|
||||
# RUSTFS_VOLUMES so data is erasure-coded ACROSS nodes. Do not use
|
||||
# node-local volume paths here — that would create four independent
|
||||
@@ -226,7 +213,6 @@ jobs:
|
||||
RUSTFS_ADDRESS: "0.0.0.0:9000"
|
||||
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY}
|
||||
RUSTFS_SECRET_KEY: ${S3_SECRET_KEY}
|
||||
RUSTFS_SSE_S3_MASTER_KEY: "${SSE_KEY}"
|
||||
RUSTFS_VOLUMES: "http://rustfs{1...4}:9000/data/rustfs{0...3}"
|
||||
# Each node's four disks share one physical device inside its
|
||||
# container, so bypass the local physical-disk-independence guard
|
||||
@@ -308,6 +294,7 @@ jobs:
|
||||
|
||||
- name: Run ceph s3-tests
|
||||
run: |
|
||||
set +e
|
||||
DEPLOY_MODE=existing \
|
||||
TEST_MODE="${TEST_MODE}" \
|
||||
TEST_SCOPE="${TEST_SCOPE}" \
|
||||
@@ -315,6 +302,26 @@ jobs:
|
||||
MAXFAIL="${MAXFAIL}" \
|
||||
MARKEXPR="${MARKEXPR}" \
|
||||
./scripts/s3-tests/run.sh
|
||||
RC=$?
|
||||
set -e
|
||||
|
||||
if [ "${TEST_SCOPE}" = "implemented" ]; then
|
||||
# Whitelist run: every failure is a regression.
|
||||
exit "${RC}"
|
||||
fi
|
||||
|
||||
# Full sweep: failures outside the implemented whitelist are
|
||||
# inventory (promotion candidates / unimplemented features), not a
|
||||
# gate. Fail only on whitelist regressions or infrastructure errors.
|
||||
JUNIT="artifacts/s3tests-${TEST_MODE}/junit.xml"
|
||||
if [ ! -f "${JUNIT}" ]; then
|
||||
echo "No junit.xml produced — infrastructure failure (exit ${RC})" >&2
|
||||
exit "${RC}"
|
||||
fi
|
||||
python3 scripts/s3-tests/report_compat.py \
|
||||
--junit "${JUNIT}" \
|
||||
--lists-dir scripts/s3-tests \
|
||||
--fail-on-regression
|
||||
|
||||
- name: Publish compatibility report
|
||||
if: always()
|
||||
@@ -339,7 +346,7 @@ jobs:
|
||||
if: always() && env.ACT != 'true'
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: s3tests-${{ env.TEST_MODE }}-shard-${{ matrix.shard-index }}
|
||||
name: s3tests-${{ env.TEST_MODE }}
|
||||
path: artifacts/**
|
||||
|
||||
alert-on-failure:
|
||||
|
||||
+24
-11
@@ -12,22 +12,27 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||
# reading this file, which has already misled at least one audit — hence this
|
||||
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||
#
|
||||
name: Fuzz
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
# Run when the harness or any directly fuzzed production crate changes.
|
||||
# PR trigger is intentionally narrow: only changes to the fuzz harness
|
||||
# itself gate a PR. Broad crate paths (ecstore/filemeta/utils/policy/…)
|
||||
# are covered by the nightly `schedule` run below, which fuzzes against
|
||||
# whatever landed on main. Widening these paths previously queued a
|
||||
# ~45min fuzz-build on nearly every PR and is why this workflow was
|
||||
# disabled; do not re-add crate paths here.
|
||||
paths:
|
||||
- "fuzz/**"
|
||||
- "scripts/fuzz/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/policy/**"
|
||||
- "crates/security-governance/**"
|
||||
- "crates/utils/**"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- ".github/workflows/fuzz.yml"
|
||||
schedule:
|
||||
- cron: "0 2 * * *"
|
||||
@@ -76,7 +81,7 @@ jobs:
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -116,7 +121,12 @@ jobs:
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: fuzz-prebuilt-binaries-${{ github.run_number }}
|
||||
path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/
|
||||
path: |
|
||||
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/archive_extract
|
||||
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/bucket_validation
|
||||
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/local_metadata
|
||||
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/path_containment
|
||||
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/policy_ingress
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
compression-level: 0
|
||||
@@ -182,7 +192,10 @@ jobs:
|
||||
nightly-fuzz-corpus:
|
||||
name: "Nightly / ${{ matrix.target }}"
|
||||
needs: fuzz-build
|
||||
# Scheduled failures are handled by alert-on-failure below.
|
||||
# TODO(ci-8): when the schedule-failure-issue composite action lands,
|
||||
# add a step here (or a dependent job) that opens/updates a GitHub issue
|
||||
# on nightly failure. ci-8 is the single alerting mechanism for all
|
||||
# scheduled workflows; do not self-roll alerting in this workflow.
|
||||
if: >
|
||||
github.event_name == 'schedule' ||
|
||||
(github.event_name == 'workflow_dispatch' &&
|
||||
|
||||
@@ -189,7 +189,6 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
include:
|
||||
- arch: x86_64
|
||||
@@ -511,13 +510,15 @@ jobs:
|
||||
|
||||
CHECKSUM_DIR="$(mktemp -d)"
|
||||
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
||||
-D "$CHECKSUM_DIR" --clobber
|
||||
-D "$CHECKSUM_DIR" --clobber 2>/dev/null || true
|
||||
|
||||
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
||||
asset="${spec%%:*}"
|
||||
checksum_cmd="${spec##*:}"
|
||||
checksum_file="${CHECKSUM_DIR}/${asset}"
|
||||
|
||||
touch "$checksum_file"
|
||||
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
base="$(basename "$f")"
|
||||
@@ -530,8 +531,7 @@ jobs:
|
||||
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
||||
grep -Fv -- "$github_base" "${checksum_file}.tmp" > "${checksum_file}.tmp2" || true
|
||||
mv "${checksum_file}.tmp2" "$checksum_file"
|
||||
digest=$("$checksum_cmd" -- "$f" | awk '{print $1}')
|
||||
printf '%s %s\n' "$digest" "$github_base" >> "$checksum_file"
|
||||
(cd "$(dirname "$f")" && "$checksum_cmd" -- "$github_base") >> "$checksum_file"
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
@@ -127,9 +127,8 @@ the broadest gate. Inspect only the final task-owned diff, classify it by
|
||||
behavioral impact rather than line count or path alone, and run the smallest
|
||||
set of checks that provides meaningful coverage. Do not let unrelated
|
||||
worktree changes or a generic contributor checklist expand the scope.
|
||||
For non-exempt changes, complete the applicable multi-role adversarial review
|
||||
before running `make pre-pr` (or an equivalent full gate). Resolve or rebut
|
||||
every finding first, then run the gate against the reviewed final diff.
|
||||
Non-exempt changes must also pass Adversarial Validation (next section) before
|
||||
the checks below count as completion.
|
||||
|
||||
### Validation floor
|
||||
|
||||
@@ -167,9 +166,8 @@ every finding first, then run the gate against the reviewed final diff.
|
||||
dependency set is identifiable, validate those packages and known
|
||||
dependents instead of the whole workspace. Use `make pre-commit` only when
|
||||
a repository-wide fast gate adds useful confidence beyond those checks.
|
||||
4. **Broad or high-risk change:** After the applicable adversarial review has
|
||||
completed, run `make pre-pr` only when targeted coverage cannot bound the
|
||||
impact, including:
|
||||
4. **Broad or high-risk change:** Run `make pre-pr` only when targeted coverage
|
||||
cannot bound the impact, including:
|
||||
- dependency, feature, build-script, procedural-macro, code-generation,
|
||||
toolchain, or CI changes that alter compilation or the test matrix;
|
||||
- cross-crate public APIs, shared foundational code, or broad refactors with
|
||||
@@ -289,9 +287,8 @@ High risk: all seven roles.
|
||||
- Every testable behavior change has a focused regression check. Exceptions
|
||||
follow the validation floor and state why a check is impractical and what
|
||||
risk remains.
|
||||
- After the applicable adversarial review has completed, the Verification
|
||||
Before PR gates pass; adversarial review supplements those gates, never
|
||||
replaces them.
|
||||
- The Verification Before PR gates pass — adversarial review supplements
|
||||
those gates, never replaces them.
|
||||
- High risk only: record a one-line verdict per role in the PR description.
|
||||
|
||||
## Git and PR Baseline
|
||||
|
||||
+4
-6
@@ -91,9 +91,8 @@ A green `make pre-commit` is not enough to open a pull request.
|
||||
`make pre-pr` is the **full** gate: it runs all of the guard checks above,
|
||||
then `clippy-check` (`cargo clippy --all-targets --all-features -- -D warnings`)
|
||||
and `test` (shell script tests, workspace tests excluding `e2e_test`, and doc
|
||||
tests). Complete the applicable multi-role adversarial review described in
|
||||
`AGENTS.md` before running `make pre-pr`; then run the gate before opening or
|
||||
updating a pull request. This is what CI enforces.
|
||||
tests). Run `make pre-pr` before opening or updating a pull request — this is
|
||||
what CI enforces.
|
||||
|
||||
### 🔒 Git Pre-commit Hooks (optional)
|
||||
|
||||
@@ -151,9 +150,8 @@ Example output when formatting fails:
|
||||
2. **Format your code**: `make fmt` or `cargo fmt --all`
|
||||
3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
|
||||
4. **Commit your changes**: `git commit -m "your message"`
|
||||
5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
|
||||
6. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
|
||||
7. **Push to your branch**: `git push`
|
||||
5. **Run the full gate before opening/updating a PR**: `make pre-pr` (clippy + tests)
|
||||
6. **Push to your branch**: `git push`
|
||||
|
||||
### 🛠️ IDE Integration
|
||||
|
||||
|
||||
Generated
+144
-176
@@ -1198,9 +1198,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-client"
|
||||
version = "1.4.0"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1"
|
||||
checksum = "3c1c8a04cb31ba74d0115af5a890bb8c0d48fba64b52812fa13929a6ef0cc83c"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-protocol-test",
|
||||
@@ -1280,9 +1280,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.14.0"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606"
|
||||
checksum = "483b858ff67522011c4786310c5cd8fd88d0be7ea3d5f1a48328446300c4269e"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -1306,9 +1306,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.15.0"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "954c563ce84507722d2679f07a35d21b9c6466b3872d513020d0281fc8112ac9"
|
||||
checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -1598,7 +1598,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1617,7 +1617,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1968,7 +1968,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -2428,7 +2428,7 @@ version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -2453,11 +2453,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -2703,9 +2703,8 @@ checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"
|
||||
|
||||
[[package]]
|
||||
name = "datafusion"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96f76f0167ed0842b29a3d1e41be3c034c0a46409a3a703cc4cc84ee8c24abf4"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -2752,9 +2751,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-catalog"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d79ec3460f6ed5c58f9b3f2d873fbc77748b82653bff1b4cdaf06de33bb4e05f"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2777,9 +2775,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-catalog-listing"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b48cef241e2efcfd496fe05ae4d0d5de20793451862faefe406c397a467e12d4"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2801,9 +2798,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-common"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f72810485975c258f1b4d00baab31728470676c60c5546f366ebd0d99f05ab6"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -2828,9 +2824,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-common-runtime"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "533c28e75dba52f41bde187d23a1cb24ab91c7c097966824fa471e67b60320ea"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"log",
|
||||
@@ -2839,9 +2834,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b00a1fa0da26f6087136a82fea7f13c76a672cbab452d4086952a7cf770a19b"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2869,9 +2863,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-arrow"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ad17ec881bff2ed7768b4bfe971d3efbf3473f2fd1f9d365447bccbdf908678"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -2893,9 +2886,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-csv"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5345285b0c3eaab412e7539b706973c083bd7e5bce575de5e0a3da488d08d1d"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2916,9 +2908,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-json"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da02fb9324f56bd8c53f1ee2e949547425cb66f76adc6832b10d44f80a1221d2"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2939,9 +2930,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-parquet"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c0b0dc1453952952fd5c69ad1c7f6042176e69ed233011d47e07cf74ed0949e"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -2971,15 +2961,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-doc"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a88fd985bc0550c36f557db69543cc9d6393b1509783520b30e902f23c555da6"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-execution"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a98f1052f91b4991f0bf2ce1e4e36dfbdcda454a956b8c8d562c7c845e8fce1d"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-buffer",
|
||||
@@ -3003,9 +2991,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-expr"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "464625a1f0e4b9df552d894fafcc8aac953ebbc8b0fa0acdaf20975fd615040e"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -3026,9 +3013,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-expr-common"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3038,9 +3024,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "051e97533e6af53e4aa0a0667cadc886abcaf36c4a5925019c55c0aa4c218fde"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-buffer",
|
||||
@@ -3066,9 +3051,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-aggregate"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d0f1bb166d3572b6ed40e1afb2faaacade962abc08c2fcf04babee74681c56b"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3087,9 +3071,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-aggregate-common"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ed756770f5f98369e181d692fd5ee6b1127ffd7322caba92f3730f9f5c92333"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3099,9 +3082,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-nested"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91173fdb5c0ff2a41169a8ffa1b385b8844f18728747bb0a37e35ad7d5772a4f"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ord",
|
||||
@@ -3124,9 +3106,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-table"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1bcdfb286a745461b126719c32700777e83df4f17cc44db5d71ebce5731e840"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -3140,9 +3121,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-window"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ec4b508f1f93f00038ba3e737e894ec6c775528b4369413386655ae6125f0fc"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3157,9 +3137,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-window-common"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b352020834140073fbf5b46ee0ceb926e5074a9d0bcae1dbd91d0586d999cde"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"datafusion-common",
|
||||
"datafusion-physical-expr-common",
|
||||
@@ -3167,9 +3146,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-macros"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"datafusion-doc",
|
||||
"quote",
|
||||
@@ -3178,9 +3156,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-optimizer"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "854445d9f7847e1e46089cf61b8d341a64382f14484e912c83a0f23b31216896"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"chrono",
|
||||
@@ -3198,9 +3175,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "671558dad1d2aa253c39c0a4c52515958b99eb91abf649f4b88d5e69cc55282f"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3220,9 +3196,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr-adapter"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffae3d78c2da80ecc829cb58536cc5aca2e99cf1365eda694fc75bfe288861e0"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3235,9 +3210,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr-common"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d9092ed15e7203fbd0903215172f7c9d18f10d94cba35137f3b3836f7c46f16"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"chrono",
|
||||
@@ -3252,9 +3226,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-optimizer"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9005b6cf50b57b72d476c6ed4662b04be7ca6be5320ba9127c6d0b7e4218095b"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3272,9 +3245,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-plan"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5787e4fcff4adc4fce8948441103a99705018b49c8dff0720b650bd7a15da112"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-data",
|
||||
@@ -3307,9 +3279,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-pruning"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e651c8df0b90daed6a7be5921ec0ee379e6909705f063eeff70fd4e35010e4c"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3323,9 +3294,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-session"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb56667ee38217efab19b895d9a936052cfb47ed438a19663351bdc42a6214a1"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow-schema",
|
||||
"async-trait",
|
||||
@@ -3338,9 +3308,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-sql"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c29067cb9d32f8e603c45e15d61ea18f1069f96ceafeceb4e18466b8e5b31d9"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"bigdecimal",
|
||||
@@ -3695,7 +3664,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3795,7 +3764,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -3955,7 +3924,7 @@ dependencies = [
|
||||
"crypto-bigint 0.5.5",
|
||||
"digest 0.10.7",
|
||||
"ff 0.13.1",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"group 0.13.0",
|
||||
"hkdf 0.12.4",
|
||||
"pem-rfc7468 0.7.0",
|
||||
@@ -4400,9 +4369,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.9"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4415,7 +4384,7 @@ version = "1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -4757,9 +4726,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.18"
|
||||
version = "0.4.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228"
|
||||
checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -5449,7 +5418,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8659,18 +8628,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.27"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3"
|
||||
checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.27"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -9125,7 +9094,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9173,7 +9142,6 @@ dependencies = [
|
||||
"mime_guess",
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
"p256 0.13.2",
|
||||
"parking_lot",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
@@ -9264,7 +9232,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"const-str",
|
||||
@@ -9287,7 +9255,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9303,13 +9271,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"hotpath",
|
||||
"jiff",
|
||||
"metrics",
|
||||
"rmp-serde",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
@@ -9321,7 +9290,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
@@ -9334,7 +9303,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"hotpath",
|
||||
@@ -9344,7 +9313,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9358,7 +9327,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9379,7 +9348,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
@@ -9389,12 +9358,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
"aws-config",
|
||||
"aws-credential-types",
|
||||
"aws-sdk-s3",
|
||||
"aws-smithy-http-client",
|
||||
@@ -9432,7 +9402,6 @@ dependencies = [
|
||||
"md-5 0.11.0",
|
||||
"memmap2",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"moka",
|
||||
"num_cpus",
|
||||
"opentelemetry",
|
||||
@@ -9529,7 +9498,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -9539,7 +9508,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"byteorder",
|
||||
@@ -9566,7 +9535,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
@@ -9599,7 +9568,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9640,7 +9609,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
@@ -9652,7 +9621,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"hotpath",
|
||||
@@ -9716,7 +9685,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -9743,7 +9712,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9792,7 +9761,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -9815,7 +9784,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
@@ -9838,7 +9807,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -9857,7 +9826,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
@@ -9877,7 +9846,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9912,7 +9881,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -9932,7 +9901,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -9949,7 +9918,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -9964,7 +9933,6 @@ dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"num_cpus",
|
||||
"nvml-wrapper",
|
||||
"opentelemetry",
|
||||
@@ -10005,7 +9973,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -10036,7 +10004,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10098,7 +10066,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
@@ -10122,7 +10090,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -10140,7 +10108,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -10178,7 +10146,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -10201,7 +10169,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-s3-types",
|
||||
@@ -10209,7 +10177,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -10218,7 +10186,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10248,7 +10216,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
@@ -10267,7 +10235,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10308,7 +10276,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"thiserror 2.0.20",
|
||||
@@ -10316,7 +10284,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10334,7 +10302,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10349,7 +10317,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -10403,7 +10371,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-data-usage",
|
||||
@@ -10419,7 +10387,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -10440,7 +10408,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -10477,7 +10445,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
@@ -10519,7 +10487,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"hotpath",
|
||||
@@ -10709,8 +10677,8 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.15.0-alpha.1"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a#ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a"
|
||||
version = "0.14.1"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=d358a68783096df1db0c3e314127f2704603b29e#d358a68783096df1db0c3e314127f2704603b29e"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -10864,7 +10832,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct 0.2.0",
|
||||
"der 0.7.10",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -11649,9 +11617,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "suppaftp"
|
||||
version = "10.0.2"
|
||||
version = "10.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "821001051ea3d12a60fb790b8c7cb9a6f5f8698dcfdca4cd533a025fefb0b5b8"
|
||||
checksum = "9c890e698eaf58526b6e7105d74c5d91ebe76a4da1faac2e20ff10e8e5c8bcff"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -11842,7 +11810,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -13392,9 +13360,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerovec"
|
||||
version = "0.11.8"
|
||||
version = "0.11.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"
|
||||
checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8"
|
||||
dependencies = [
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
|
||||
+53
-54
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.3"
|
||||
version = "1.0.0-rc.2"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.3" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.3" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.3" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.3" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.3" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.3" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.3" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.3" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.3" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.3" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.3" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.3" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.3" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.3" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.3" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.3" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.3" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.3" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.3" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.3" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.3" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.3" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.3", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.3" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.3" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.3" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.3" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.3" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.3" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.3" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.3" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.3" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.3" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.3" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.3" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.3" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.3" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.3" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.3" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.3" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.3" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.3" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.3" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.2" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.2" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.2" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.2" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.2" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.2" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.2" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.2" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.2" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.2" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.2" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.2" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.2" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.2" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.2" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.2" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.2" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.2", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.2" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.2" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.2" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.2" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.2" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.2" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.2" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.2" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.2" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.2" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.2" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.2" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.2" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.2" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.2" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.2" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.2" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -231,8 +231,8 @@ aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.115.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.111.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
|
||||
aws-smithy-runtime-api = { version = "1.15.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||
aws-smithy-types = { version = "1.6.2" }
|
||||
base64 = "0.23.1"
|
||||
base64-simd = "0.8.0"
|
||||
@@ -245,7 +245,8 @@ crossbeam-queue = "0.3.13"
|
||||
crossbeam-channel = "0.5.16"
|
||||
crossbeam-deque = "0.8.7"
|
||||
crossbeam-utils = "0.8.22"
|
||||
datafusion = { default-features = false, version = "55.0.0" }
|
||||
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
|
||||
#datafusion = { default-features = false, version = "54.1.0" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
faster-hex = "0.10.0"
|
||||
@@ -263,7 +264,6 @@ lazy_static = "1.5.0"
|
||||
libc = "0.2.189"
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.11"
|
||||
lz4 = "1.28.1"
|
||||
matchit = "0.9.2"
|
||||
@@ -290,7 +290,7 @@ rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "d358a68783096df1db0c3e314127f2704603b29e" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
@@ -326,7 +326,6 @@ zstd = "0.13.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9-tokio-telemetry = "0.3"
|
||||
opentelemetry = { version = "0.32.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.32.0" }
|
||||
@@ -340,7 +339,7 @@ pyroscope = { version = "2.1.1" }
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.2" }
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.7" }
|
||||
russh-sftp = "2.4.0"
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.3
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.3
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -236,19 +236,12 @@ async fn audit_pipeline_reports_empty_runtime_snapshots() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stopping_audit_replay_workers_is_a_no_op_when_there_are_none() {
|
||||
async fn audit_runtime_facade_stops_empty_replay_workers() {
|
||||
let registry = Arc::new(Mutex::new(AuditRegistry::new()));
|
||||
let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new()));
|
||||
let facade = AuditRuntimeFacade::new(registry, Arc::clone(&replay_workers));
|
||||
let facade = AuditRuntimeFacade::new(registry, replay_workers);
|
||||
|
||||
facade.stop_replay_workers().await;
|
||||
|
||||
// The stop path takes the manager's workers and hands them to the adapter,
|
||||
// so an empty facade must leave it empty rather than wedge it, and a second
|
||||
// call — which shutdown paths make — must stay harmless (rustfs/backlog#1836).
|
||||
assert!(replay_workers.read().await.is_empty());
|
||||
facade.stop_replay_workers().await;
|
||||
assert!(replay_workers.read().await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -44,6 +44,7 @@ metrics = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
smallvec = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fmt::{self, Display},
|
||||
@@ -632,6 +633,104 @@ pub fn create_heal_response(
|
||||
}
|
||||
}
|
||||
|
||||
fn lc_get_prefix(rule: &LifecycleRule) -> String {
|
||||
if let Some(p) = &rule.prefix {
|
||||
return p.to_string();
|
||||
} else if let Some(filter) = &rule.filter {
|
||||
if let Some(p) = &filter.prefix {
|
||||
return p.to_string();
|
||||
} else if let Some(and) = &filter.and
|
||||
&& let Some(p) = &and.prefix
|
||||
{
|
||||
return p.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
"".into()
|
||||
}
|
||||
|
||||
pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) -> bool {
|
||||
if config.rules.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
for rule in config.rules.iter() {
|
||||
if rule.status == ExpirationStatus::from_static(ExpirationStatus::DISABLED) {
|
||||
continue;
|
||||
}
|
||||
let rule_prefix = lc_get_prefix(rule);
|
||||
if !prefix.is_empty() && !rule_prefix.is_empty() && !prefix.starts_with(&rule_prefix) && !rule_prefix.starts_with(prefix)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(e) = &rule.noncurrent_version_expiration {
|
||||
if e.noncurrent_days.is_some() {
|
||||
return true;
|
||||
}
|
||||
if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if rule.noncurrent_version_transitions.is_some() {
|
||||
return true;
|
||||
}
|
||||
if let Some(true) = rule.expiration.as_ref().map(|e| e.date.is_some()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Some(true) = rule.expiration.as_ref().map(|e| e.days.is_some()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Some(true) = rule.transitions.as_ref().map(|t| !t.is_empty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if rule.transitions.is_some() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool {
|
||||
if config.rules.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
for rule in config.rules.iter() {
|
||||
if rule
|
||||
.status
|
||||
.eq(&ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if !prefix.is_empty()
|
||||
&& let Some(filter) = &rule.filter
|
||||
&& let Some(r_prefix) = &filter.prefix
|
||||
&& !r_prefix.is_empty()
|
||||
{
|
||||
// incoming prefix must be in rule prefix
|
||||
if !recursive && !prefix.starts_with(r_prefix) {
|
||||
continue;
|
||||
}
|
||||
// If recursive, we can skip this rule if it doesn't match the tested prefix or level below prefix
|
||||
// does not match
|
||||
if recursive && !r_prefix.starts_with(prefix) && !prefix.starts_with(r_prefix) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPriority>) -> Result<(), String> {
|
||||
let req = HealChannelRequest {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
|
||||
@@ -13,6 +13,82 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default)]
|
||||
struct TimedAction {
|
||||
count: u64,
|
||||
acc_time: u64,
|
||||
min_time: Option<u64>,
|
||||
max_time: Option<u64>,
|
||||
bytes: u64,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TimedAction {
|
||||
// Avg returns the average time spent on the action.
|
||||
pub fn avg(&self) -> Option<Duration> {
|
||||
if self.count == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(Duration::from_nanos(self.acc_time / self.count))
|
||||
}
|
||||
|
||||
// AvgBytes returns the average bytes processed.
|
||||
pub fn avg_bytes(&self) -> u64 {
|
||||
if self.count == 0 {
|
||||
return 0;
|
||||
}
|
||||
self.bytes / self.count
|
||||
}
|
||||
|
||||
// Merge other into t.
|
||||
pub fn merge(&mut self, other: TimedAction) {
|
||||
self.count += other.count;
|
||||
self.acc_time += other.acc_time;
|
||||
self.bytes += other.bytes;
|
||||
|
||||
if self.count == 0 {
|
||||
self.min_time = other.min_time;
|
||||
}
|
||||
if let Some(other_min) = other.min_time {
|
||||
self.min_time = self.min_time.map_or(Some(other_min), |min| Some(min.min(other_min)));
|
||||
}
|
||||
|
||||
self.max_time = self
|
||||
.max_time
|
||||
.map_or(other.max_time, |max| Some(max.max(other.max_time.unwrap_or(0))));
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug)]
|
||||
enum SizeCategory {
|
||||
SizeLessThan1KiB = 0,
|
||||
SizeLessThan1MiB,
|
||||
SizeLessThan10MiB,
|
||||
SizeLessThan100MiB,
|
||||
SizeLessThan1GiB,
|
||||
SizeGreaterThan1GiB,
|
||||
// Add new entries here
|
||||
SizeLastElemMarker,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SizeCategory {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let s = match *self {
|
||||
SizeCategory::SizeLessThan1KiB => "SizeLessThan1KiB",
|
||||
SizeCategory::SizeLessThan1MiB => "SizeLessThan1MiB",
|
||||
SizeCategory::SizeLessThan10MiB => "SizeLessThan10MiB",
|
||||
SizeCategory::SizeLessThan100MiB => "SizeLessThan100MiB",
|
||||
SizeCategory::SizeLessThan1GiB => "SizeLessThan1GiB",
|
||||
SizeCategory::SizeGreaterThan1GiB => "SizeGreaterThan1GiB",
|
||||
SizeCategory::SizeLastElemMarker => "SizeLastElemMarker",
|
||||
};
|
||||
write!(f, "{s}")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Copy)]
|
||||
pub struct AccElem {
|
||||
pub total: u64,
|
||||
|
||||
+25
-110
@@ -729,7 +729,7 @@ fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
u64::try_from(duration.as_secs()).unwrap_or(u64::MAX)
|
||||
u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
@@ -781,19 +781,6 @@ struct ScannerBucketDriveResultValue {
|
||||
last_seen: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
|
||||
struct ScannerActiveBucketDriveKey {
|
||||
source: String,
|
||||
bucket: String,
|
||||
drive: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ScannerActiveBucketDriveValue {
|
||||
count: u64,
|
||||
started_at: Timestamp,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metrics
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -826,7 +813,6 @@ pub struct Metrics {
|
||||
scanner_set_scans_active: AtomicU64,
|
||||
scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
|
||||
scanner_bucket_drive_results: Mutex<ScannerBucketDriveResults>,
|
||||
scanner_active_bucket_drive_scans: Mutex<HashMap<ScannerActiveBucketDriveKey, ScannerActiveBucketDriveValue>>,
|
||||
scanner_bucket_drive_result_clock: AtomicU64,
|
||||
current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
|
||||
last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
|
||||
@@ -1059,15 +1045,6 @@ pub struct ScannerBucketDriveResultSnapshot {
|
||||
pub count: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerActiveBucketDriveSnapshot {
|
||||
pub source: String,
|
||||
pub bucket: String,
|
||||
pub drive: String,
|
||||
pub count: u64,
|
||||
pub age_seconds: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerReplicationRepairSnapshot {
|
||||
pub source: String,
|
||||
@@ -1410,8 +1387,6 @@ pub struct ScannerRuntimeDetailsReport {
|
||||
pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||
#[serde(default)]
|
||||
pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||
#[serde(default)]
|
||||
pub active_bucket_drive_scans: Vec<ScannerActiveBucketDriveSnapshot>,
|
||||
}
|
||||
|
||||
impl CurrentCycle {
|
||||
@@ -1426,11 +1401,25 @@ impl CurrentCycle {
|
||||
}
|
||||
|
||||
/// OTEL metric name constants for scanner metrics
|
||||
const OTEL_SCANNER_OBJECTS_SCANNED: &str = "rustfs_scanner_objects_scanned_total";
|
||||
const OTEL_SCANNER_DIRECTORIES_SCANNED: &str = "rustfs_scanner_directories_scanned_total";
|
||||
const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
|
||||
const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
|
||||
const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
|
||||
const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
|
||||
|
||||
fn emit_otel_counter(metric: usize, count: u64) {
|
||||
match Metric::from_index(metric) {
|
||||
Some(Metric::ScanObject) => {
|
||||
metrics::counter!(OTEL_SCANNER_OBJECTS_SCANNED).increment(count);
|
||||
}
|
||||
Some(Metric::ScanFolder) => {
|
||||
metrics::counter!(OTEL_SCANNER_DIRECTORIES_SCANNED).increment(count);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_cycle_result_label(result: u8) -> &'static str {
|
||||
match result {
|
||||
SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
|
||||
@@ -1771,7 +1760,7 @@ pub fn emit_scan_cycle_deferred(duration: Duration) {
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
|
||||
}
|
||||
|
||||
pub fn emit_scan_bucket_drive_complete(_source: ScannerWorkSource, success: bool, bucket: &str, disk: &str, duration: Duration) {
|
||||
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
|
||||
let result = if success { "success" } else { "error" };
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
|
||||
metrics::counter!(
|
||||
@@ -1789,7 +1778,7 @@ pub fn emit_scan_bucket_drive_complete(_source: ScannerWorkSource, success: bool
|
||||
.record(duration.as_secs_f64());
|
||||
}
|
||||
|
||||
pub fn emit_scan_bucket_drive_partial(_source: ScannerWorkSource, bucket: &str, disk: &str, duration: Duration) {
|
||||
pub fn emit_scan_bucket_drive_partial(bucket: &str, disk: &str, duration: Duration) {
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
|
||||
metrics::counter!(
|
||||
OTEL_SCANNER_BUCKETS_SCANNED,
|
||||
@@ -1842,7 +1831,6 @@ impl Metrics {
|
||||
scanner_set_scans_active: AtomicU64::new(0),
|
||||
scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
|
||||
scanner_bucket_drive_results: Mutex::new(ScannerBucketDriveResults::default()),
|
||||
scanner_active_bucket_drive_scans: Mutex::new(HashMap::new()),
|
||||
scanner_bucket_drive_result_clock: AtomicU64::new(0),
|
||||
current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
|
||||
last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
|
||||
@@ -1972,6 +1960,7 @@ impl Metrics {
|
||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||
global_metrics().record_source_work_for_metric(metric, 1);
|
||||
emit_otel_counter(metric_idx, 1);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
global_metrics().latency[metric_idx].add(duration);
|
||||
}
|
||||
@@ -1987,6 +1976,7 @@ impl Metrics {
|
||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||
global_metrics().record_source_work_for_metric(metric, 1);
|
||||
emit_otel_counter(metric_idx, 1);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
global_metrics().latency[metric_idx].add_size(duration, size);
|
||||
}
|
||||
@@ -2002,6 +1992,7 @@ impl Metrics {
|
||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||
global_metrics().record_source_work_for_metric(metric, 1);
|
||||
emit_otel_counter(metric_idx, 1);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
global_metrics().latency[metric_idx].add(duration);
|
||||
}
|
||||
@@ -2019,6 +2010,7 @@ impl Metrics {
|
||||
let count = usize_to_u64_saturated(count);
|
||||
global_metrics().operations[metric_idx].fetch_add(count, Ordering::Relaxed);
|
||||
global_metrics().record_source_work_for_metric(metric, count);
|
||||
emit_otel_counter(metric_idx, count);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
global_metrics().latency[metric_idx].add(duration);
|
||||
}
|
||||
@@ -2039,6 +2031,7 @@ impl Metrics {
|
||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||
let metric_idx = Metric::Ilm as usize;
|
||||
global_metrics().operations[metric_idx].fetch_add(versions, Ordering::Relaxed);
|
||||
emit_otel_counter(metric_idx, versions);
|
||||
global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
|
||||
global_metrics().actions_latency[a_idx].add(duration);
|
||||
})
|
||||
@@ -2051,6 +2044,7 @@ impl Metrics {
|
||||
let metric_idx = metric as usize;
|
||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||
global_metrics().record_source_work_for_metric(metric, 1);
|
||||
emit_otel_counter(metric_idx, 1);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
global_metrics().latency[metric_idx].add(duration);
|
||||
}
|
||||
@@ -2334,45 +2328,8 @@ impl Metrics {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_scan_bucket_drive_start(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
|
||||
pub fn record_scan_bucket_drive_start(&self) {
|
||||
self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
|
||||
if bucket.is_empty() || drive.is_empty() {
|
||||
return;
|
||||
}
|
||||
let key = ScannerActiveBucketDriveKey {
|
||||
source: source.as_str().to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
drive: drive.to_string(),
|
||||
};
|
||||
let mut active = self
|
||||
.scanner_active_bucket_drive_scans
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
active
|
||||
.entry(key)
|
||||
.and_modify(|value| value.count = value.count.saturating_add(1))
|
||||
.or_insert(ScannerActiveBucketDriveValue {
|
||||
count: 1,
|
||||
started_at: Timestamp::now(),
|
||||
});
|
||||
}
|
||||
|
||||
pub fn record_scan_bucket_drive_end(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
|
||||
let key = ScannerActiveBucketDriveKey {
|
||||
source: source.as_str().to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
drive: drive.to_string(),
|
||||
};
|
||||
let mut active = self
|
||||
.scanner_active_bucket_drive_scans
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(value) = active.get_mut(&key) {
|
||||
value.count = value.count.saturating_sub(1);
|
||||
if value.count == 0 {
|
||||
active.remove(&key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_scan_bucket_drive_failure(&self) {
|
||||
@@ -2845,26 +2802,6 @@ impl Metrics {
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let now = Timestamp::now();
|
||||
let mut active_bucket_drive_scans = self
|
||||
.scanner_active_bucket_drive_scans
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.iter()
|
||||
.map(|(key, value)| ScannerActiveBucketDriveSnapshot {
|
||||
source: key.source.clone(),
|
||||
bucket: key.bucket.clone(),
|
||||
drive: key.drive.clone(),
|
||||
count: value.count,
|
||||
age_seconds: timestamp_elapsed_seconds_since(now, value.started_at),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
active_bucket_drive_scans.sort_by(|left, right| {
|
||||
left.source
|
||||
.cmp(&right.source)
|
||||
.then_with(|| left.bucket.cmp(&right.bucket))
|
||||
.then_with(|| left.drive.cmp(&right.drive))
|
||||
});
|
||||
ScannerRuntimeDetailsReport {
|
||||
disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
|
||||
bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
|
||||
@@ -2874,7 +2811,6 @@ impl Metrics {
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clone(),
|
||||
active_bucket_drive_scans,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4455,7 +4391,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn report_includes_bucket_drive_scan_starts() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
||||
metrics.record_scan_bucket_drive_start();
|
||||
metrics.record_scan_bucket_drive_failure();
|
||||
|
||||
let report = metrics.report().await;
|
||||
@@ -4464,27 +4400,6 @@ mod tests {
|
||||
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn active_bucket_drive_snapshot_is_structured_and_retired_on_end() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
||||
let active = metrics.scanner_runtime_details_report().active_bucket_drive_scans;
|
||||
assert_eq!(active.len(), 1);
|
||||
assert_eq!(active[0].source, ScannerWorkSource::Usage.as_str());
|
||||
assert_eq!(active[0].bucket, "bucket-a");
|
||||
assert_eq!(active[0].drive, "/mnt/data/1");
|
||||
assert_eq!(active[0].count, 2);
|
||||
|
||||
metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
||||
assert_eq!(metrics.scanner_runtime_details_report().active_bucket_drive_scans[0].count, 1);
|
||||
metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
||||
assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
|
||||
|
||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "", "/mnt/data/1");
|
||||
assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_structured_bucket_drive_results() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -148,62 +148,6 @@ fn unix_now_ms() -> u64 {
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// A repair the MRF consumer landed, fanned out so retry ledgers can drop
|
||||
/// entries the journal no longer tracks (backlog#1894 axis B). The payload
|
||||
/// mirrors the intent identity so consumers match without re-parsing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MrfRepairedEvent {
|
||||
pub bucket: Arc<str>,
|
||||
pub object: Arc<str>,
|
||||
pub version_id: Option<[u8; 16]>,
|
||||
}
|
||||
|
||||
/// Bound on the repaired-event backlog. Notices are best-effort hints; when
|
||||
/// the ring is full the oldest are dropped and the affected ledger entries
|
||||
/// simply expire through their own attempts/age limits.
|
||||
const MRF_REPAIRED_EVENT_CAP: usize = 4096;
|
||||
|
||||
static MRF_REPAIRED_EVENTS: OnceLock<std::sync::Mutex<std::collections::VecDeque<MrfRepairedEvent>>> = OnceLock::new();
|
||||
|
||||
/// Record that the MRF consumer landed a repair. Never blocks: the critical
|
||||
/// section is a deque push under a std mutex.
|
||||
pub fn note_mrf_repaired(bucket: &str, object: &str, version_id: Option<[u8; 16]>) {
|
||||
let registry = MRF_REPAIRED_EVENTS.get_or_init(|| std::sync::Mutex::new(std::collections::VecDeque::new()));
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
return;
|
||||
};
|
||||
if events.len() >= MRF_REPAIRED_EVENT_CAP {
|
||||
events.pop_front();
|
||||
}
|
||||
events.push_back(MrfRepairedEvent {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
version_id,
|
||||
});
|
||||
}
|
||||
|
||||
/// Take the repair notices recorded for `bucket`, leaving other buckets'
|
||||
/// notices in place for their own scanners.
|
||||
pub fn take_mrf_repaired_events_for(bucket: &str) -> Vec<MrfRepairedEvent> {
|
||||
let Some(registry) = MRF_REPAIRED_EVENTS.get() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut taken = Vec::new();
|
||||
let mut retained = std::collections::VecDeque::with_capacity(events.len());
|
||||
while let Some(event) = events.pop_front() {
|
||||
if event.bucket.as_ref() == bucket {
|
||||
taken.push(event);
|
||||
} else {
|
||||
retained.push_back(event);
|
||||
}
|
||||
}
|
||||
*events = retained;
|
||||
taken
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -256,32 +200,4 @@ mod tests {
|
||||
assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None));
|
||||
set_mrf_delivery_enabled(true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repaired_events_take_is_bucket_scoped_and_cap_bounded() {
|
||||
// Distinct buckets keep their notices until their own scanner takes
|
||||
// them; a take for one bucket leaves the others' notices in place.
|
||||
note_mrf_repaired("bucket-a", "object-1", None);
|
||||
note_mrf_repaired("bucket-b", "object-2", None);
|
||||
note_mrf_repaired("bucket-a", "object-3", None);
|
||||
|
||||
let taken_a = take_mrf_repaired_events_for("bucket-a");
|
||||
assert_eq!(taken_a.len(), 2);
|
||||
assert_eq!(taken_a[0].object.as_ref(), "object-1");
|
||||
assert_eq!(taken_a[1].object.as_ref(), "object-3");
|
||||
assert!(take_mrf_repaired_events_for("bucket-a").is_empty(), "take is destructive per bucket");
|
||||
|
||||
let taken_b = take_mrf_repaired_events_for("bucket-b");
|
||||
assert_eq!(taken_b.len(), 1);
|
||||
assert_eq!(taken_b[0].object.as_ref(), "object-2");
|
||||
|
||||
// Cap bound: flooding the ring drops the oldest notices rather than
|
||||
// growing unbounded.
|
||||
for i in 0..=(MRF_REPAIRED_EVENT_CAP + 8) {
|
||||
note_mrf_repaired("flood-bucket", &format!("object-{i}"), None);
|
||||
}
|
||||
let flooded = take_mrf_repaired_events_for("flood-bucket");
|
||||
assert_eq!(flooded.len(), MRF_REPAIRED_EVENT_CAP);
|
||||
assert_eq!(flooded[0].object.as_ref(), "object-9", "the oldest notices past the cap are dropped");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,15 +115,6 @@ Current guidance:
|
||||
- enables KMS readiness enforcement for `/health/ready`.
|
||||
- default is `false`.
|
||||
|
||||
## Object lock admission environment variables
|
||||
|
||||
- `RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS`
|
||||
- experimental same-object PUT commit namespace-lock admission budget.
|
||||
- default is `0`, which disables this override and keeps `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT` behavior.
|
||||
- when set, only `put_object_commit` write-lock acquisition is bounded by this millisecond budget; other namespace lock users keep the global object-lock timeout.
|
||||
- timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
|
||||
- this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
|
||||
|
||||
## Drive timeout environment variables
|
||||
|
||||
- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
|
||||
|
||||
@@ -427,19 +427,6 @@ pub const ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT: &str = "RUSTFS_OBJECT_LOCK_ACQUIRE_TI
|
||||
/// Default lock acquisition timeout: 5 seconds.
|
||||
pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5;
|
||||
|
||||
/// Environment variable for the experimental PUT commit namespace lock acquire timeout in milliseconds.
|
||||
///
|
||||
/// A value of `0` disables the experiment and keeps
|
||||
/// `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT` as the timeout. This only bounds the
|
||||
/// `put_object_commit` namespace write-lock wait and is intended for #925
|
||||
/// tail-drain admission experiments.
|
||||
///
|
||||
/// Default: 0 milliseconds (disabled).
|
||||
pub const ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS: &str = "RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS";
|
||||
|
||||
/// Default: PUT commit namespace lock acquire timeout override is disabled.
|
||||
pub const DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS: u64 = 0;
|
||||
|
||||
/// Environment variable for remote namespace lock RPC transport timeout in milliseconds.
|
||||
///
|
||||
/// This timeout bounds the internode RPC call itself. It is intentionally
|
||||
|
||||
@@ -228,6 +228,15 @@ pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 4;
|
||||
/// Default object interval for cooperative scanner yields.
|
||||
pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128;
|
||||
|
||||
/// Compatibility flag kept for Patch 3 rollback windows.
|
||||
///
|
||||
/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue.
|
||||
/// When this flag is enabled, RustFS logs a warning and continues to use enqueue-based heal.
|
||||
pub const ENV_SCANNER_INLINE_HEAL_ENABLE: &str = "RUSTFS_SCANNER_INLINE_HEAL_ENABLE";
|
||||
|
||||
/// Default inline scanner heal compatibility mode.
|
||||
pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
|
||||
|
||||
/// Scanner speed preset controlling throttling behavior.
|
||||
///
|
||||
/// Each preset defines three parameters:
|
||||
|
||||
@@ -92,11 +92,15 @@ pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
|
||||
pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
|
||||
pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
|
||||
pub const NOTIFY_MYSQL_SUB_SYS: &str = "notify_mysql";
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_NATS_SUB_SYS: &str = "notify_nats";
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq";
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch";
|
||||
pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
|
||||
pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
|
||||
pub const NOTIFY_REDIS_DEFAULT_CHANNEL: &str = "rustfs_notify_channel";
|
||||
pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
|
||||
|
||||
+21
-26
@@ -48,14 +48,16 @@ cargo nextest run --profile e2e-smoke -p e2e_test
|
||||
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
|
||||
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
|
||||
|
||||
# Protocols suite — fixed ports, MUST be single-threaded, gated by build features
|
||||
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
|
||||
cargo test -p e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
|
||||
```
|
||||
|
||||
The protocols suite has its own contract (fixed bind ports 9022–9301,
|
||||
single-worker execution, feature-gated scheduling) documented in
|
||||
`--test-threads=1`, feature-gated scheduling) documented in
|
||||
[`src/protocols/README.md`](src/protocols/README.md). `RUSTFS_BUILD_FEATURES`
|
||||
selects which features the spawned binary is built with; leave it unset to run
|
||||
every protocol entry. Use the exact profile command under
|
||||
[Troubleshooting](#troubleshooting) for CI-equivalent execution.
|
||||
every protocol entry.
|
||||
|
||||
### `#[ignore]` semantics
|
||||
|
||||
@@ -157,26 +159,27 @@ construction (random port + isolated temp dir) and need no serialization.
|
||||
## CI map
|
||||
|
||||
`e2e_test` is **excluded** from the main `cargo nextest run --profile ci --all`
|
||||
pass (`--exclude e2e_test`) — the whole crate is too slow to gate every PR.
|
||||
Subsets join CI through nextest profiles; the fixed-port protocol suite uses
|
||||
the same profile for membership and execution with one nightly worker.
|
||||
pass ([`.github/workflows/ci.yml`](../../.github/workflows/ci.yml) line 158,
|
||||
`--exclude e2e_test`) — the whole crate is too slow to gate every PR. Subsets
|
||||
join CI through the nextest profile system only (never as ad-hoc jobs):
|
||||
|
||||
| Suite | Runs where | Status |
|
||||
| --- | --- | --- |
|
||||
| Smoke subset (`e2e-smoke` profile) | `e2e-tests` job, every PR | **Active** (backlog#1149 ci-4) |
|
||||
| Full single-node suite (`e2e-full` profile) | `e2e-full` job, merge queue + main | **Active** (backlog#1149 ci-5) |
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| KMS suite | — | Not in CI yet (backlog#1149 ci-5) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | — | Not in CI yet (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
| `reliant/*` | 19 tests in PR smoke; remaining default tests in `e2e-full` | **Active** except `#[ignore]` |
|
||||
| Replication (slow + dual-node) | `e2e-repl-nightly` profile, scheduled workflow | **Active** (backlog#1147 repl-1) |
|
||||
| `reliant/*` (pre-started server) | — | Manual only |
|
||||
|
||||
The profile filters in [`.config/nextest.toml`](../../.config/nextest.toml) are
|
||||
the wiring source of truth. Committed test-ID digests under
|
||||
`.config/e2e-*-selection.txt` make every membership change explicit.
|
||||
Links: [`ci.yml`](../../.github/workflows/ci.yml) `e2e-tests` (line 347),
|
||||
`test-ilm-integration-serial` (line 196). The `e2e-smoke` `default-filter` in
|
||||
[`.config/nextest.toml`](../../.config/nextest.toml) is the **single wiring
|
||||
mechanism** — extend that filter (or add a sibling profile) to admit more
|
||||
tests; do not add e2e jobs to `ci.yml`. repl-1 / ilm-3 are landing in parallel
|
||||
and may add lanes; keep the table above easy to extend.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
@@ -185,15 +188,9 @@ the wiring source of truth. Committed test-ID digests under
|
||||
```bash
|
||||
# Smoke (e2e-tests job) — includes the 20 fast replication tests
|
||||
cargo nextest run --profile e2e-smoke -p e2e_test
|
||||
# Full single-node merge/main lane
|
||||
cargo nextest run --profile e2e-full -p e2e_test
|
||||
# Cluster fault nightly lane
|
||||
cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
# Replication nightly lane; install awscurl so STS paths do not skip
|
||||
# Replication nightly lane (16 slow + dual-node tests; install awscurl for the
|
||||
# STS dual-node test, else it skips gracefully)
|
||||
cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
# Fixed-port protocol nightly lane
|
||||
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
|
||||
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
|
||||
# ILM serial lane
|
||||
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
|
||||
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
|
||||
@@ -276,6 +273,4 @@ current subset is.
|
||||
`docs/testing/e2e-suite-inventory.md` records the per-module test counts as
|
||||
listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or
|
||||
moving e2e tests so acceptance numbers in the test-strategy issues
|
||||
(backlog#1147–#1155) stay auditable. When a profile membership change is
|
||||
intentional, review its JSON listing before updating the matching
|
||||
`.config/e2e-*-selection.txt` test-ID digest.
|
||||
(backlog#1147–#1155) stay auditable.
|
||||
|
||||
@@ -53,8 +53,7 @@ pub(crate) const FAST_DATA_USAGE_SCANNER_ENV: &[(&str, &str)] =
|
||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
const TEST_PORT_MIN: u16 = 20_000;
|
||||
// Keep allocator ports below the ephemeral range used by bind(..., 0) test helpers.
|
||||
const TEST_PORT_RANGE: u16 = 10_000;
|
||||
const TEST_PORT_RANGE: u16 = 40_000;
|
||||
const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
|
||||
const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
|
||||
const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
|
||||
|
||||
@@ -6,6 +6,9 @@ use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::time::sleep;
|
||||
use tracing::info;
|
||||
|
||||
const COMPRESSION_TEST_BUCKET: &str = "compression-test-bucket";
|
||||
@@ -84,7 +87,17 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
||||
env.process = Some(process);
|
||||
|
||||
info!("Waiting for RustFS server with compression enabled on {}", env.address);
|
||||
env.wait_for_server_ready().await
|
||||
for i in 0..30 {
|
||||
if TcpStream::connect(&env.address).await.is_ok() {
|
||||
info!("RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
if i == 29 {
|
||||
return Err("RustFS server failed to become ready".into());
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -653,7 +666,17 @@ async fn start_rustfs_with_compression_and_sse(
|
||||
env.process = Some(process);
|
||||
|
||||
info!("Waiting for RustFS server with compression + SSE-S3 enabled on {}", env.address);
|
||||
env.wait_for_server_ready().await
|
||||
for i in 0..30 {
|
||||
if TcpStream::connect(&env.address).await.is_ok() {
|
||||
info!("RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
if i == 29 {
|
||||
return Err("RustFS server failed to become ready".into());
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// SSE-S3 + disk compression multipart: each part is compressed and then encrypted, and every GET
|
||||
|
||||
@@ -1,250 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! E2E proof that a mid-stream GET failure is *reportable* — rustfs#4784.
|
||||
//!
|
||||
//! The functional invariant (a beyond-quorum read must fail rather than return
|
||||
//! a clean short body) is already covered by
|
||||
//! `degraded_read_eof_regression_test`. This suite covers the half that issue
|
||||
//! #4784 got stuck on for a month: whether an operator can tell, from the
|
||||
//! source server's log alone, that a GET failed mid-body and **which object**
|
||||
//! it failed on.
|
||||
//!
|
||||
//! The reporter saw only downstream symptoms — `rclone` reporting
|
||||
//! `unexpected EOF` on its PUT, and the receiving RustFS logging
|
||||
//! `Io error: error reading a body from connection` with a 500. In a cross-remote
|
||||
//! `rclone sync`, the source GET body *is* the destination PUT body, so a source
|
||||
//! read that ends short of its committed `Content-Length` surfaces as a PUT
|
||||
//! failure on the far side. Built-in replication and site replication have the
|
||||
//! same shape (read locally, PUT remotely), which is why every transport in that
|
||||
//! report failed the same way.
|
||||
//!
|
||||
//! The source side, meanwhile, said nothing:
|
||||
//! * `GetObjectReaderStream`'s short-read and read-error arms only incremented
|
||||
//! a metric; their log lines sat behind the `tracing-chunk-debug` cargo
|
||||
//! feature, which is not in the default feature set and therefore is not
|
||||
//! compiled into any released binary.
|
||||
//! * `GetObjectStreamingReader` did log mid-stream failures, but only under a
|
||||
//! `request_id` — with no bucket or object name, a failure could not be
|
||||
//! traced back to the object that caused it.
|
||||
//! * Those lines were `warn!`, while `DEFAULT_LOG_LEVEL` is `error`, so a
|
||||
//! default deployment filtered them out anyway.
|
||||
//!
|
||||
//! This test reproduces the source-side fault against a real server over the S3
|
||||
//! API and asserts the operator-visible evidence, at the **default** log level.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::DiskFaultHarness;
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, timeout};
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
const MIB: usize = 1024 * 1024;
|
||||
const OP_TIMEOUT: Duration = Duration::from_secs(90);
|
||||
|
||||
/// The structured event name every GET body failure is tagged with.
|
||||
const STREAM_BODY_EVENT: &str = "get_object_stream_body";
|
||||
|
||||
fn payload(len: usize, seed: u8) -> Vec<u8> {
|
||||
(0..len)
|
||||
.map(|i| (i as u64).wrapping_mul(2654435761).wrapping_add(seed as u64) as u8)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Upload a multipart object so the data lands in real `part.*` shard files
|
||||
/// rather than being inlined into `xl.meta` (inlined objects cannot be
|
||||
/// corrupted shard-wise, and never exercise the streaming read path).
|
||||
async fn put_multipart(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
parts: Vec<Vec<u8>>,
|
||||
) -> Result<usize, Box<dyn Error + Send + Sync>> {
|
||||
let total_len = parts.iter().map(Vec::len).sum();
|
||||
|
||||
let create = client.create_multipart_upload().bucket(bucket).key(key).send().await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let mut completed = Vec::with_capacity(parts.len());
|
||||
for (index, part_body) in parts.into_iter().enumerate() {
|
||||
let part_number = (index + 1) as i32;
|
||||
let uploaded = timeout(
|
||||
OP_TIMEOUT,
|
||||
client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(part_body))
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| format!("upload_part {part_number} timed out"))??;
|
||||
completed.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(uploaded.e_tag().ok_or("missing part etag")?)
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
|
||||
timeout(
|
||||
OP_TIMEOUT,
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build())
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| "complete_multipart_upload timed out")??;
|
||||
|
||||
Ok(total_len)
|
||||
}
|
||||
|
||||
/// rustfs#4784: reproduce the source-side fault the reporter kept hitting —
|
||||
/// a GET that commits `200` + a full `Content-Length` and then cannot finish
|
||||
/// the body — and assert the server log names the object, at the log level a
|
||||
/// default deployment actually runs with.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn midstream_get_failure_is_logged_with_the_object_at_default_log_level() -> TestResult {
|
||||
init_logging();
|
||||
info!("rustfs#4784: a mid-stream GET failure must name its object in the source log");
|
||||
|
||||
let mut harness = DiskFaultHarness::new(4).await?;
|
||||
|
||||
// Capture the child's stdout so the test can read what an operator would.
|
||||
let log_path = format!("{}/server.log", harness.env.temp_dir);
|
||||
harness.env.capture_log_path = Some(log_path.clone());
|
||||
|
||||
// Reproduce a DEFAULT deployment's logging, not the e2e harness's
|
||||
// permissive `rustfs=info`: `DEFAULT_LOG_LEVEL` is `error`. Before the
|
||||
// #4784 fix these failures were `warn!`, so a default deployment
|
||||
// filtered them out entirely — which is why the reporter's source logs
|
||||
// were empty. extra_env is applied after the harness's own RUST_LOG, so
|
||||
// this wins.
|
||||
harness.set_env("RUST_LOG", "error");
|
||||
harness.set_env("RUSTFS_OBS_LOGGER_LEVEL", "error");
|
||||
|
||||
harness.start_server().await?;
|
||||
let client = harness.env.create_s3_client();
|
||||
|
||||
let bucket = "issue4784-source-read";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
// Named after the reporter's restic index objects, which is where they
|
||||
// saw the failures.
|
||||
let key = "index/3b18542ab3af4c3d03f804c7a24173e7836ef7fa447b5d1e9d634f975cc51611";
|
||||
let expected_len = put_multipart(
|
||||
&client,
|
||||
bucket,
|
||||
key,
|
||||
vec![payload(5 * MIB, 71), payload(5 * MIB, 72), payload(5 * MIB, 73)],
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Baseline: the object reads back completely before any corruption.
|
||||
let baseline = timeout(OP_TIMEOUT, client.get_object().bucket(bucket).key(key).send())
|
||||
.await
|
||||
.map_err(|_| "baseline GET timed out")??
|
||||
.body
|
||||
.collect()
|
||||
.await?;
|
||||
assert_eq!(baseline.into_bytes().len(), expected_len, "baseline GET must return the whole object");
|
||||
|
||||
// Corrupt three of four shards in a 2+2 set: below the 2-shard read
|
||||
// quorum. The corruption sits mid-file, so block 0 still reads clean —
|
||||
// the server commits 200 + the full Content-Length and only then cannot
|
||||
// reconstruct. That is the mid-stream window the reporter's downstream
|
||||
// saw as `unexpected EOF`.
|
||||
harness.corrupt_object_shard(0, bucket, key)?;
|
||||
harness.corrupt_object_shard(1, bucket, key)?;
|
||||
harness.corrupt_object_shard(2, bucket, key)?;
|
||||
|
||||
let response = timeout(OP_TIMEOUT, client.get_object().bucket(bucket).key(key).send())
|
||||
.await
|
||||
.map_err(|_| "degraded GET timed out")?;
|
||||
|
||||
// Either outcome is functionally correct (that invariant belongs to
|
||||
// degraded_read_eof_regression_test); this suite only needs the read to
|
||||
// have failed so there is something to report.
|
||||
let delivered = match response {
|
||||
Err(err) => {
|
||||
info!("degraded GET failed before the body: {err}");
|
||||
None
|
||||
}
|
||||
Ok(response) => match response.body.collect().await {
|
||||
Ok(aggregated) => Some(aggregated.into_bytes().len()),
|
||||
Err(err) => {
|
||||
info!("degraded GET failed mid-body as expected: {err}");
|
||||
None
|
||||
}
|
||||
},
|
||||
};
|
||||
assert_ne!(
|
||||
delivered,
|
||||
Some(expected_len),
|
||||
"the beyond-quorum read unexpectedly succeeded; this suite needs a failed read to have something to report"
|
||||
);
|
||||
|
||||
// Give the child a moment to flush its stdout.
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
let logged = std::fs::read_to_string(&log_path)?;
|
||||
|
||||
let failure_lines: Vec<&str> = logged.lines().filter(|line| line.contains(STREAM_BODY_EVENT)).collect();
|
||||
|
||||
assert!(
|
||||
!failure_lines.is_empty(),
|
||||
"a mid-stream GET failure produced no `{STREAM_BODY_EVENT}` line at the default log level. \
|
||||
This is the #4784 blind spot: the failure was only counted in a metric, or logged below \
|
||||
`error` and filtered out. Captured log:\n{logged}"
|
||||
);
|
||||
|
||||
// The identity is the whole point: a request_id alone cannot be resolved
|
||||
// back to an object once the request is over.
|
||||
assert!(
|
||||
failure_lines.iter().any(|line| line.contains(key)),
|
||||
"no `{STREAM_BODY_EVENT}` line named the failing object `{key}`, so the report is still \
|
||||
unactionable. Lines seen:\n{}",
|
||||
failure_lines.join("\n")
|
||||
);
|
||||
assert!(
|
||||
failure_lines.iter().any(|line| line.contains(bucket)),
|
||||
"no `{STREAM_BODY_EVENT}` line named the failing bucket `{bucket}`. Lines seen:\n{}",
|
||||
failure_lines.join("\n")
|
||||
);
|
||||
|
||||
info!(
|
||||
"source-side evidence now present: {} stream-body failure line(s) naming the object",
|
||||
failure_lines.len()
|
||||
);
|
||||
for line in &failure_lines {
|
||||
info!("operator-visible evidence: {line}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1028,6 +1028,20 @@ impl<'a> ReaderPathExpectation<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
fn with_size_bucket(
|
||||
object: ReaderObject<'a>,
|
||||
expected_path: &'a str,
|
||||
object_class: &'a str,
|
||||
expected_size_bucket: &'a str,
|
||||
) -> Self {
|
||||
Self {
|
||||
object,
|
||||
expected_path,
|
||||
object_class,
|
||||
expected_size_bucket: Some(expected_size_bucket),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_any_size_bucket(object: ReaderObject<'a>, expected_path: &'a str, object_class: &'a str) -> Self {
|
||||
Self {
|
||||
object,
|
||||
@@ -1895,7 +1909,12 @@ async fn four_node_compressed_inline_fallback() -> TestResult {
|
||||
assert_reader_path(
|
||||
&collector,
|
||||
&client,
|
||||
ReaderPathExpectation::for_class(ReaderObject::new(bucket, key, &body, put.e_tag(), None), LEGACY_DUPLEX, COMPRESSED),
|
||||
ReaderPathExpectation::with_size_bucket(
|
||||
ReaderObject::new(bucket, key, &body, put.e_tag(), None),
|
||||
LEGACY_DUPLEX,
|
||||
COMPRESSED,
|
||||
size_bucket(4 * KIB),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -2255,7 +2274,6 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
|
||||
hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "1");
|
||||
hot.set_env("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1");
|
||||
hot.set_env("RUSTFS_TRANSITION_QUEUE_SEND_TIMEOUT_MS", "1");
|
||||
hot.start().await?;
|
||||
|
||||
let hot_client = hot.create_s3_client(0)?;
|
||||
@@ -2272,7 +2290,7 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(1024 * KIB, index)))
|
||||
.body(ByteStream::from(payload(64 * KIB, index)))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
|
||||
@@ -39,7 +39,6 @@ use std::time::Duration;
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
type S3OperationResult<T> = Result<T, Box<aws_sdk_s3::Error>>;
|
||||
|
||||
const ALLOWED_KEY: &str = "kms-matrix-allowed-key";
|
||||
const OTHER_KEY: &str = "kms-matrix-other-key";
|
||||
@@ -131,7 +130,7 @@ fn policy_document(statements: Vec<serde_json::Value>) -> String {
|
||||
serde_json::json!({ "Version": "2012-10-17", "Statement": statements }).to_string()
|
||||
}
|
||||
|
||||
async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> S3OperationResult<()> {
|
||||
async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> Result<(), aws_sdk_s3::Error> {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
@@ -142,23 +141,16 @@ async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> S3Operatio
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|error| Box::new(aws_sdk_s3::Error::from(error)))
|
||||
.map_err(aws_sdk_s3::Error::from)
|
||||
}
|
||||
|
||||
/// Assert the operation failed with `AccessDenied` rather than any other error.
|
||||
///
|
||||
/// A bare `is_err` would also accept `KMSKeyDisabled` or an internal error, which
|
||||
/// would hide both a leak of key state and an outage masquerading as a denial.
|
||||
fn assert_access_denied<T: std::fmt::Debug, E: std::fmt::Debug + std::borrow::Borrow<aws_sdk_s3::Error>>(
|
||||
result: Result<T, E>,
|
||||
what: &str,
|
||||
) {
|
||||
fn assert_access_denied<T: std::fmt::Debug>(result: Result<T, aws_sdk_s3::Error>, what: &str) {
|
||||
let error = result.expect_err(&format!("{what} must be denied"));
|
||||
assert_eq!(
|
||||
error.borrow().code(),
|
||||
Some("AccessDenied"),
|
||||
"{what} must fail with AccessDenied: {error:?}"
|
||||
);
|
||||
assert_eq!(error.code(), Some("AccessDenied"), "{what} must fail with AccessDenied: {error:?}");
|
||||
}
|
||||
|
||||
/// Retry an SSE-KMS write until the identity's policy has reached the request path.
|
||||
@@ -304,7 +296,7 @@ async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|err| Box::new(aws_sdk_s3::Error::from(err))),
|
||||
.map_err(aws_sdk_s3::Error::from),
|
||||
"SSE-KMS read by an identity holding no kms grant",
|
||||
);
|
||||
|
||||
@@ -318,7 +310,7 @@ async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(|err| Box::new(aws_sdk_s3::Error::from(err))),
|
||||
.map_err(aws_sdk_s3::Error::from),
|
||||
"SSE-KMS read by an identity holding kms:GenerateDataKey but not kms:Decrypt",
|
||||
);
|
||||
|
||||
|
||||
@@ -647,10 +647,7 @@ async fn test_multipart_upload_with_sse_c(
|
||||
}
|
||||
|
||||
/// Test large multipart upload to verify streaming encryption works correctly
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "parked behind the TODO in test_local_kms_multipart_upload until streaming encryption is fixed for large files (backlog#1823)"
|
||||
)]
|
||||
#[allow(dead_code)]
|
||||
async fn test_large_multipart_upload(
|
||||
s3_client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
|
||||
@@ -48,11 +48,6 @@ mod replacement_privileged_e2e_test;
|
||||
#[cfg(test)]
|
||||
mod degraded_read_eof_regression_test;
|
||||
|
||||
// rustfs#4784: a mid-stream GET failure must be reportable from the source
|
||||
// server's log alone — naming the object, at the default log level.
|
||||
#[cfg(test)]
|
||||
mod get_stream_failure_observability_test;
|
||||
|
||||
// backlog#1183: GET codec-streaming fast path must be byte/header identical to
|
||||
// the legacy duplex path before its rollout gates can be flipped on by default.
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -19,17 +19,32 @@ use std::time::Instant;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::{error, info};
|
||||
|
||||
/// Core test categories
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TestCategory {
|
||||
SingleValue,
|
||||
MultiValue,
|
||||
Concatenation,
|
||||
Nested,
|
||||
DenyScenarios,
|
||||
}
|
||||
|
||||
impl TestCategory {}
|
||||
|
||||
/// Test case definition
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TestDefinition {
|
||||
pub name: String,
|
||||
#[allow(dead_code)]
|
||||
pub category: TestCategory,
|
||||
pub is_critical: bool,
|
||||
}
|
||||
|
||||
impl TestDefinition {
|
||||
pub fn new(name: impl Into<String>, is_critical: bool) -> Self {
|
||||
pub fn new(name: impl Into<String>, category: TestCategory, is_critical: bool) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
category,
|
||||
is_critical,
|
||||
}
|
||||
}
|
||||
@@ -77,12 +92,12 @@ impl PolicyTestSuite {
|
||||
/// Create default test suite
|
||||
pub fn new() -> Self {
|
||||
let tests = vec![
|
||||
TestDefinition::new("test_aws_policy_variables_single_value", true),
|
||||
TestDefinition::new("test_aws_policy_variables_multi_value", true),
|
||||
TestDefinition::new("test_aws_policy_variables_concatenation", true),
|
||||
TestDefinition::new("test_aws_policy_variables_nested", true),
|
||||
TestDefinition::new("test_aws_policy_variables_deny", true),
|
||||
TestDefinition::new("test_aws_policy_variables_sts", true),
|
||||
TestDefinition::new("test_aws_policy_variables_single_value", TestCategory::SingleValue, true),
|
||||
TestDefinition::new("test_aws_policy_variables_multi_value", TestCategory::MultiValue, true),
|
||||
TestDefinition::new("test_aws_policy_variables_concatenation", TestCategory::Concatenation, true),
|
||||
TestDefinition::new("test_aws_policy_variables_nested", TestCategory::Nested, true),
|
||||
TestDefinition::new("test_aws_policy_variables_deny", TestCategory::DenyScenarios, true),
|
||||
TestDefinition::new("test_aws_policy_variables_sts", TestCategory::SingleValue, true),
|
||||
];
|
||||
|
||||
Self {
|
||||
|
||||
@@ -11,17 +11,10 @@ test process directly.
|
||||
|
||||
## Running Tests
|
||||
|
||||
Use the canonical CI-equivalent protocol command in the parent
|
||||
[`e2e_test` README](../../README.md#troubleshooting).
|
||||
|
||||
For targeted debugging of the core suite only:
|
||||
|
||||
```bash
|
||||
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
|
||||
```
|
||||
|
||||
This targeted command does not cover the full `e2e-protocols` profile.
|
||||
|
||||
`RUSTFS_BUILD_FEATURES` controls which features the test rustfs binary is
|
||||
built with. When this variable is set, the protocol test runner schedules
|
||||
only entries whose protocol is present in the requested feature list. Leave
|
||||
@@ -140,3 +133,4 @@ property without consulting any external doc.
|
||||
Bind ports 9023 (SFTP) and 9100 (S3). Spawns rustfs with
|
||||
`RUSTFS_SFTP_IDLE_TIMEOUT=5`, sleeps 10 s past the timeout, then issues an
|
||||
SFTP request and asserts the server has closed the session.
|
||||
|
||||
|
||||
@@ -233,111 +233,6 @@ pub async fn test_webdav_core_operations() -> Result<()> {
|
||||
);
|
||||
info!("PASS: PUT file '{}' successful", filename);
|
||||
|
||||
// Regression for #6260: a bucket-scoped policy must be able to discover its bucket at the
|
||||
// WebDAV root without the unrelated global ListAllMyBuckets permission.
|
||||
let scoped_bucket = "webdav-scoped-bucket";
|
||||
let scoped_file = "visible.txt";
|
||||
let scoped_user = "webdav-scoped-user";
|
||||
let scoped_secret = "webdav-scoped-secret";
|
||||
let scoped_policy_name = "webdav-scoped-policy";
|
||||
|
||||
let resp = client
|
||||
.request(reqwest::Method::from_bytes(b"MKCOL").unwrap(), format!("{}/{}", base_url, scoped_bucket))
|
||||
.header("Authorization", &auth_header)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(resp.status().as_u16(), 201, "scoped test bucket should be created");
|
||||
|
||||
let resp = client
|
||||
.put(format!("{}/{}/{}", base_url, scoped_bucket, scoped_file))
|
||||
.header("Authorization", &auth_header)
|
||||
.body("visible to the scoped principal")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(resp.status().as_u16(), 201, "scoped test object should be created");
|
||||
|
||||
admin_create_user(&admin_base_url, scoped_user, scoped_secret).await?;
|
||||
admin_add_canned_policy(
|
||||
&admin_base_url,
|
||||
scoped_policy_name,
|
||||
&serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{}", scoped_bucket),
|
||||
format!("arn:aws:s3:::{}/*", scoped_bucket)
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{}", scoped_bucket),
|
||||
format!("arn:aws:s3:::{}/*", scoped_bucket)
|
||||
],
|
||||
"Condition": { "Bool": { "aws:SecureTransport": "true" } }
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{}", scoped_bucket),
|
||||
format!("arn:aws:s3:::{}/*", scoped_bucket)
|
||||
],
|
||||
"Condition": { "StringEquals": { "s3:signatureversion": "AWS4-HMAC-SHA256" } }
|
||||
}
|
||||
]
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
admin_attach_policy_to_user(&admin_base_url, scoped_policy_name, scoped_user).await?;
|
||||
|
||||
let scoped_auth = basic_auth_header_for(scoped_user, scoped_secret);
|
||||
let resp = client
|
||||
.request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), &base_url)
|
||||
.header("Authorization", &scoped_auth)
|
||||
.header("Depth", "1")
|
||||
.header("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(resp.status().as_u16(), 207, "bucket-scoped root PROPFIND should succeed");
|
||||
let root_listing = resp.text().await?;
|
||||
assert!(root_listing.contains(scoped_bucket), "the authorized bucket should be listed");
|
||||
assert!(!root_listing.contains(bucket_name), "an unauthorized bucket must not be listed");
|
||||
|
||||
let resp = client
|
||||
.request(
|
||||
reqwest::Method::from_bytes(b"PROPFIND").unwrap(),
|
||||
format!("{}/{}", base_url, scoped_bucket),
|
||||
)
|
||||
.header("Authorization", &scoped_auth)
|
||||
.header("Depth", "1")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(resp.status().as_u16(), 207, "authorized bucket PROPFIND should succeed");
|
||||
assert!(resp.text().await?.contains(scoped_file), "the authorized object should be listed");
|
||||
|
||||
let denied_user = "webdav-no-buckets-user";
|
||||
let denied_secret = "webdav-no-buckets-secret";
|
||||
admin_create_user(&admin_base_url, denied_user, denied_secret).await?;
|
||||
let resp = client
|
||||
.request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), &base_url)
|
||||
.header("Authorization", basic_auth_header_for(denied_user, denied_secret))
|
||||
.header("Depth", "1")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
resp.status().as_u16(),
|
||||
207,
|
||||
"PROPFIND keeps the root resource visible when the directory listing is forbidden"
|
||||
);
|
||||
let denied_body = resp.text().await?;
|
||||
assert!(!denied_body.contains(scoped_bucket), "a denied response must not leak the scoped bucket");
|
||||
assert!(!denied_body.contains(bucket_name), "a denied response must not leak the admin bucket");
|
||||
|
||||
// Test GET (download file)
|
||||
info!("Testing WebDAV: GET (download file '{}')", filename);
|
||||
let resp = client
|
||||
|
||||
@@ -169,42 +169,6 @@ impl QuotaTestEnv {
|
||||
bucket: &str,
|
||||
quota_bytes: u64,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_quota_usage_for(bucket).await?;
|
||||
|
||||
let quota_path = format!("/rustfs/admin/v3/quota/{bucket}");
|
||||
let quota_config = serde_json::json!({
|
||||
"quota": quota_bytes,
|
||||
"quota_type": "HARD"
|
||||
})
|
||||
.to_string();
|
||||
let readiness = async {
|
||||
loop {
|
||||
let (status, response) = admin_request(
|
||||
&self.env.url,
|
||||
Method::PUT,
|
||||
"a_path,
|
||||
Some(quota_config.clone()),
|
||||
&self.env.access_key,
|
||||
&self.env.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if status.is_success() {
|
||||
return Ok::<(), Box<dyn std::error::Error + Send + Sync>>(());
|
||||
}
|
||||
if status != StatusCode::SERVICE_UNAVAILABLE {
|
||||
return Err(format!("failed to set quota for {bucket}: {status} {response}").into());
|
||||
}
|
||||
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
};
|
||||
match timeout(Duration::from_secs(30), readiness).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(format!("quota readiness did not converge for {bucket} within 30 seconds").into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn wait_for_quota_usage_for(&self, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let stats_path = format!("/rustfs/admin/v3/quota-stats/{bucket}");
|
||||
let readiness = async {
|
||||
loop {
|
||||
@@ -217,12 +181,28 @@ impl QuotaTestEnv {
|
||||
if status != StatusCode::SERVICE_UNAVAILABLE {
|
||||
return Err(format!("quota usage readiness failed for {bucket}: {status} {response}").into());
|
||||
}
|
||||
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
};
|
||||
match timeout(Duration::from_secs(30), readiness).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into()),
|
||||
Ok(result) => result?,
|
||||
Err(_) => {
|
||||
return Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into());
|
||||
}
|
||||
}
|
||||
|
||||
let url = format!("{}/rustfs/admin/v3/quota/{}", self.env.url, bucket);
|
||||
let quota_config = serde_json::json!({
|
||||
"quota": quota_bytes,
|
||||
"quota_type": "HARD"
|
||||
});
|
||||
|
||||
let response = awscurl_put(&url, "a_config.to_string(), &self.env.access_key, &self.env.secret_key).await?;
|
||||
if response.contains("error") {
|
||||
Err(format!("Failed to set quota: {}", response).into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -634,7 +614,6 @@ mod integration_tests {
|
||||
let env = QuotaTestEnv::new().await?;
|
||||
|
||||
env.create_bucket().await?;
|
||||
env.wait_for_quota_usage_for(&env.bucket_name).await?;
|
||||
|
||||
// Test 1: GET quota for bucket without quota config
|
||||
let url = format!("{}/rustfs/admin/v3/quota/{}", env.env.url, env.bucket_name);
|
||||
@@ -642,7 +621,12 @@ mod integration_tests {
|
||||
assert!(response.contains("quota") && response.contains("null"));
|
||||
|
||||
// Test 2: PUT quota - valid config
|
||||
env.set_bucket_quota(1048576).await?;
|
||||
let quota_config = serde_json::json!({
|
||||
"quota": 1048576,
|
||||
"quota_type": "HARD"
|
||||
});
|
||||
let response = awscurl_put(&url, "a_config.to_string(), &env.env.access_key, &env.env.secret_key).await?;
|
||||
assert!(response.contains("success") || !response.contains("error"));
|
||||
|
||||
// Test 3: GET quota after setting
|
||||
let response = awscurl_get(&url, &env.env.access_key, &env.env.secret_key).await?;
|
||||
|
||||
@@ -168,24 +168,6 @@ async fn wait_for_version_expired(
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_key_versions_empty(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
|
||||
let start = std::time::Instant::now();
|
||||
loop {
|
||||
let listing = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
|
||||
if listing.versions().is_empty() && listing.delete_markers().is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if start.elapsed() >= deadline {
|
||||
return Err(format!(
|
||||
"object {bucket}/{key} still had versions or delete markers after {}s: {listing:?}",
|
||||
deadline.as_secs()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a prefix-scoped `Days`-based expiration rule.
|
||||
fn expiration_rule(id: &str, prefix: &str, days: i32) -> Result<LifecycleRule, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let rule = LifecycleRule::builder()
|
||||
@@ -211,21 +193,6 @@ fn noncurrent_expiration_rule(
|
||||
Ok(rule)
|
||||
}
|
||||
|
||||
fn noncurrent_expiration_with_delete_marker_cleanup_rule(
|
||||
id: &str,
|
||||
prefix: &str,
|
||||
days: i32,
|
||||
) -> Result<LifecycleRule, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let rule = LifecycleRule::builder()
|
||||
.id(id)
|
||||
.filter(LifecycleRuleFilter::builder().prefix(prefix).build())
|
||||
.expiration(LifecycleExpiration::builder().expired_object_delete_marker(true).build())
|
||||
.noncurrent_version_expiration(NoncurrentVersionExpiration::builder().noncurrent_days(days).build())
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.build()?;
|
||||
Ok(rule)
|
||||
}
|
||||
|
||||
async fn put_expiration_config(client: &Client, bucket: &str, rule: LifecycleRule) -> TestResult {
|
||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||
client
|
||||
@@ -445,143 +412,6 @@ async fn test_lifecycle_noncurrent_version_expiry_removes_only_old_version() ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A combined `NoncurrentDays=1` and `ExpiredObjectDeleteMarker=true` rule
|
||||
/// must remove a noncurrent data version and then its sole latest delete
|
||||
/// marker, without expiring current-only objects. A second prefix with only
|
||||
/// noncurrent expiry proves that marker cleanup comes from EODM.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn test_lifecycle_noncurrent_expiry_then_cleans_expired_delete_marker() -> TestResult {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut extra_env = fast_lifecycle_env();
|
||||
extra_env.push(("RUSTFS_ILM_DEBUG_DAY_SECS", "2"));
|
||||
env.start_rustfs_server_with_env(vec![], &extra_env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "ilm-expired-delete-marker";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let cascade_key = "cascade/deleted.txt";
|
||||
let cascade_put = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(cascade_key)
|
||||
.body(ByteStream::from_static(b"cascade payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let cascade_data_version = cascade_put
|
||||
.version_id()
|
||||
.map(str::to_string)
|
||||
.expect("cascade PUT returns a version id");
|
||||
let cascade_delete = client.delete_object().bucket(bucket).key(cascade_key).send().await?;
|
||||
let cascade_marker_version = cascade_delete
|
||||
.version_id()
|
||||
.map(str::to_string)
|
||||
.expect("cascade DELETE returns a marker version id");
|
||||
assert_eq!(cascade_delete.delete_marker(), Some(true));
|
||||
|
||||
let survivor_key = "cascade/current-only.txt";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(survivor_key)
|
||||
.body(ByteStream::from_static(b"current payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let survivor_before = client.get_object().bucket(bucket).key(survivor_key).send().await?;
|
||||
assert_eq!(survivor_before.body.collect().await?.into_bytes().as_ref(), b"current payload");
|
||||
|
||||
let control_key = "nve-only/deleted.txt";
|
||||
let control_put = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(control_key)
|
||||
.body(ByteStream::from_static(b"control payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let control_data_version = control_put
|
||||
.version_id()
|
||||
.map(str::to_string)
|
||||
.expect("control PUT returns a version id");
|
||||
let control_delete = client.delete_object().bucket(bucket).key(control_key).send().await?;
|
||||
let control_marker_version = control_delete
|
||||
.version_id()
|
||||
.map(str::to_string)
|
||||
.expect("control DELETE returns a marker version id");
|
||||
assert_eq!(control_delete.delete_marker(), Some(true));
|
||||
|
||||
let cascade_before = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(cascade_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
cascade_before
|
||||
.versions()
|
||||
.iter()
|
||||
.any(|version| version.version_id() == Some(cascade_data_version.as_str())),
|
||||
"cascade data version must exist before lifecycle is installed: {cascade_before:?}"
|
||||
);
|
||||
assert!(
|
||||
cascade_before
|
||||
.delete_markers()
|
||||
.iter()
|
||||
.any(|marker| { marker.version_id() == Some(cascade_marker_version.as_str()) && marker.is_latest() == Some(true) }),
|
||||
"cascade latest delete marker must exist before lifecycle is installed: {cascade_before:?}"
|
||||
);
|
||||
|
||||
let lifecycle = BucketLifecycleConfiguration::builder()
|
||||
.rules(noncurrent_expiration_with_delete_marker_cleanup_rule(
|
||||
"expire-and-clean-marker",
|
||||
"cascade/",
|
||||
1,
|
||||
)?)
|
||||
.rules(noncurrent_expiration_rule("expire-only", "nve-only/", 1)?)
|
||||
.build()?;
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.lifecycle_configuration(lifecycle)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
wait_for_key_versions_empty(&client, bucket, cascade_key, StdDuration::from_secs(90)).await?;
|
||||
wait_for_version_expired(&client, bucket, control_key, &control_data_version, StdDuration::from_secs(90)).await?;
|
||||
|
||||
let survivor = client.get_object().bucket(bucket).key(survivor_key).send().await?;
|
||||
assert_eq!(survivor.body.collect().await?.into_bytes().as_ref(), b"current payload");
|
||||
|
||||
let control_after = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(control_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
control_after.versions().is_empty(),
|
||||
"NVE-only control must remove its data version: {control_after:?}"
|
||||
);
|
||||
assert!(
|
||||
control_after
|
||||
.delete_markers()
|
||||
.iter()
|
||||
.any(|marker| { marker.version_id() == Some(control_marker_version.as_str()) && marker.is_latest() == Some(true) }),
|
||||
"NVE-only control must preserve its latest delete marker: {control_after:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `Days=0` expiration is invalid per S3 semantics (`Days` must be a positive
|
||||
/// integer >= 1). A `PutBucketLifecycleConfiguration` carrying a zero-day rule
|
||||
/// must be rejected with `InvalidArgument` (HTTP 400) - see crates/lifecycle
|
||||
|
||||
@@ -110,7 +110,6 @@ const USER_META_KEY: &str = "ilm7-origin";
|
||||
const USER_META_VAL: &str = "hermetic-transition";
|
||||
const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
|
||||
const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
|
||||
const TIER_MUTATION_RECOVERY_CHANGED: &str = "Remote tier mutation recovery changed before publish";
|
||||
|
||||
/// 5 MiB — the S3 minimum size for a non-final multipart part; the object's only
|
||||
/// internal part boundary sits at this offset.
|
||||
@@ -184,58 +183,28 @@ async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironme
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let verify_path = format!("/rustfs/admin/v3/tier/{TIER_NAME}");
|
||||
let deadline = Instant::now() + StdDuration::from_secs(30);
|
||||
let mut recovery_changed = false;
|
||||
loop {
|
||||
if recovery_changed {
|
||||
let (status, _) =
|
||||
signed_admin_request(&hot.url, Method::GET, &verify_path, None, &hot.access_key, &hot.secret_key).await?;
|
||||
if status.is_success() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let (status, resp) = signed_admin_request(
|
||||
&hot.url,
|
||||
Method::PUT,
|
||||
"/rustfs/admin/v3/tier",
|
||||
Some(&body),
|
||||
&hot.access_key,
|
||||
&hot.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if status.is_success() {
|
||||
return Ok(());
|
||||
}
|
||||
if resp.contains(TIER_MUTATION_RECOVERY_CHANGED) {
|
||||
recovery_changed = true;
|
||||
} else if !recovery_changed || !resp.contains("TierNameAlreadyExist") {
|
||||
return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(100)).await;
|
||||
let (status, resp) = signed_admin_request(
|
||||
&hot.url,
|
||||
Method::PUT,
|
||||
"/rustfs/admin/v3/tier",
|
||||
Some(&body),
|
||||
&hot.access_key,
|
||||
&hot.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn remove_rustfs_tier_force(hot: &RustFSTestEnvironment) -> TestResult {
|
||||
let path = format!("/rustfs/admin/v3/tier/{TIER_NAME}?force=true");
|
||||
let deadline = Instant::now() + StdDuration::from_secs(30);
|
||||
loop {
|
||||
let (status, resp) =
|
||||
signed_admin_request(&hot.url, Method::DELETE, &path, None, &hot.access_key, &hot.secret_key).await?;
|
||||
if status.is_success() {
|
||||
return Ok(());
|
||||
}
|
||||
if (!resp.contains("TierNameBackendInUse") && !resp.contains(TIER_MUTATION_RECOVERY_CHANGED))
|
||||
|| Instant::now() >= deadline
|
||||
{
|
||||
return Err(format!("RemoveTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
}
|
||||
// Tier mutation cleanup and startup recovery are asynchronous.
|
||||
tokio::time::sleep(StdDuration::from_millis(100)).await;
|
||||
let (status, resp) = signed_admin_request(&hot.url, Method::DELETE, &path, None, &hot.access_key, &hot.secret_key).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("RemoveTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A current-version `Transition Days=0` rule scoped to the object's prefix.
|
||||
@@ -1508,6 +1477,15 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() ->
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
hot_client.create_bucket().bucket(MANUAL_TIER_FAILURE_BUCKET).send().await?;
|
||||
let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25);
|
||||
put_backdated_single_part_object(
|
||||
&hot_client,
|
||||
MANUAL_TIER_FAILURE_BUCKET,
|
||||
MANUAL_TIER_FAILURE_KEY,
|
||||
b"manual tier failure object",
|
||||
due_mtime,
|
||||
)
|
||||
.await?;
|
||||
put_lifecycle_transition_rule(
|
||||
&hot_client,
|
||||
MANUAL_TIER_FAILURE_BUCKET,
|
||||
@@ -1518,15 +1496,6 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() ->
|
||||
.await?;
|
||||
remove_rustfs_tier_force(&hot).await?;
|
||||
|
||||
let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25);
|
||||
put_backdated_single_part_object(
|
||||
&hot_client,
|
||||
MANUAL_TIER_FAILURE_BUCKET,
|
||||
MANUAL_TIER_FAILURE_KEY,
|
||||
b"manual tier failure object",
|
||||
due_mtime,
|
||||
)
|
||||
.await?;
|
||||
let before_remote_count = cold_tier_object_count(&cold_client).await?;
|
||||
let accepted = manual_transition_async_run(&hot, MANUAL_TIER_FAILURE_BUCKET, MANUAL_TIER_FAILURE_PREFIX, false, 10).await?;
|
||||
assert_eq!(accepted.state, "accepted");
|
||||
|
||||
@@ -229,6 +229,7 @@ base64-simd.workspace = true
|
||||
serde_urlencoded.workspace = true
|
||||
google-cloud-storage = { workspace = true }
|
||||
google-cloud-auth = { workspace = true }
|
||||
aws-config = { workspace = true }
|
||||
faster-hex = { workspace = true }
|
||||
ratelimit = { workspace = true }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
@@ -267,7 +268,6 @@ tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time
|
||||
# dispatcher to keep tracing's process-global callsite-interest cache honest.
|
||||
tracing-core = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
metrics-util = { workspace = true, features = ["debugging"] }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
proptest = "1"
|
||||
rcgen.workspace = true
|
||||
|
||||
@@ -90,12 +90,6 @@ use uuid::Uuid;
|
||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
|
||||
pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
|
||||
pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
|
||||
pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
|
||||
pub type PutObjectTaggingSdkError = Box<SdkError<PutObjectTaggingError>>;
|
||||
pub type DeleteObjectTaggingSdkError = Box<SdkError<DeleteObjectTaggingError>>;
|
||||
|
||||
pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();
|
||||
|
||||
fn replication_target_versioning_enabled(versioning: Option<&BucketVersioningStatus>) -> bool {
|
||||
@@ -1974,7 +1968,7 @@ impl TargetClient {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<HeadObjectOutput, HeadObjectSdkError> {
|
||||
) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
|
||||
// Announce the replication check so a RustFS target returns SSE-C
|
||||
// object metadata (etag/size) without the customer key the replication
|
||||
// worker cannot hold; otherwise SSE-C replicas never converge on HEAD.
|
||||
@@ -1987,7 +1981,8 @@ impl TargetClient {
|
||||
// object with an identical ETag, and the worker concludes the object
|
||||
// already converged — so it never actually replicates it.
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_PROXY_REQUEST, "false");
|
||||
self.client
|
||||
match self
|
||||
.client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
@@ -2004,7 +1999,10 @@ impl TargetClient {
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
{
|
||||
Ok(res) => Ok(res),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// HEAD used by the read-proxy path (GET/HEAD of an object not yet
|
||||
@@ -2025,7 +2023,7 @@ impl TargetClient {
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
extra_headers: HeaderMap,
|
||||
) -> Result<HeadObjectOutput, HeadObjectSdkError> {
|
||||
) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
|
||||
let headers = proxy_outbound_headers(extra_headers);
|
||||
self.client
|
||||
.head_object()
|
||||
@@ -2038,7 +2036,6 @@ impl TargetClient {
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// GET used by the read-proxy path (MinIO `proxyGetToReplicationTarget`).
|
||||
@@ -2054,7 +2051,7 @@ impl TargetClient {
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
extra_headers: HeaderMap,
|
||||
) -> Result<GetObjectOutput, GetObjectSdkError> {
|
||||
) -> Result<GetObjectOutput, SdkError<GetObjectError>> {
|
||||
let headers = proxy_outbound_headers(extra_headers);
|
||||
self.client
|
||||
.get_object()
|
||||
@@ -2067,7 +2064,6 @@ impl TargetClient {
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// GetObjectTagging for the tagging read-proxy path
|
||||
@@ -2077,7 +2073,7 @@ impl TargetClient {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<GetObjectTaggingOutput, GetObjectTaggingSdkError> {
|
||||
) -> Result<GetObjectTaggingOutput, SdkError<GetObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.get_object_tagging()
|
||||
@@ -2088,7 +2084,6 @@ impl TargetClient {
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// PutObjectTagging for the tagging proxy path
|
||||
@@ -2099,7 +2094,7 @@ impl TargetClient {
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
tagging: SdkTagging,
|
||||
) -> Result<PutObjectTaggingOutput, PutObjectTaggingSdkError> {
|
||||
) -> Result<PutObjectTaggingOutput, SdkError<PutObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.put_object_tagging()
|
||||
@@ -2111,7 +2106,6 @@ impl TargetClient {
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// DeleteObjectTagging for the tagging proxy path
|
||||
@@ -2121,7 +2115,7 @@ impl TargetClient {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<DeleteObjectTaggingOutput, DeleteObjectTaggingSdkError> {
|
||||
) -> Result<DeleteObjectTaggingOutput, SdkError<DeleteObjectTaggingError>> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.delete_object_tagging()
|
||||
@@ -2132,7 +2126,6 @@ impl TargetClient {
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// On success returns the version id the target assigned (from
|
||||
|
||||
@@ -2180,7 +2180,7 @@ pub async fn recover_manual_transition_jobs_once(
|
||||
if limit == 0 {
|
||||
return Err(Error::other("manual transition job recovery limit must be greater than zero"));
|
||||
}
|
||||
let list_limit = i32::try_from(limit).unwrap_or(i32::MAX);
|
||||
let list_limit = i32::try_from(limit).map_or(i32::MAX, |value| value);
|
||||
let page = api
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
@@ -2386,7 +2386,7 @@ async fn replay_manual_transition_pending_tasks(
|
||||
version_id: task.version_id,
|
||||
etag: task.etag,
|
||||
mod_time,
|
||||
size: task.size.unwrap_or(0),
|
||||
size: task.size.map_or(0, |size| size),
|
||||
is_latest: task.is_latest.unwrap_or(false),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -4994,9 +4994,6 @@ pub async fn apply_expiry_on_transitioned_object(
|
||||
src: &LcEventSrc,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> bool {
|
||||
if lc_event.action.delete_all() {
|
||||
return apply_expiry_on_non_transitioned_objects(api, oi, lc_event, src, bucket_incarnation_id).await;
|
||||
}
|
||||
let time_ilm = Metrics::time_ilm(lc_event.action);
|
||||
if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src, bucket_incarnation_id).await {
|
||||
return false;
|
||||
@@ -5050,24 +5047,13 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
if lc_event.action.delete_all() {
|
||||
opts.delete_prefix = true;
|
||||
opts.delete_prefix_object = true;
|
||||
opts.lifecycle_delete_all = Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||
version_id: oi.version_id.filter(|version_id| !version_id.is_nil()),
|
||||
delete_marker: oi.delete_marker,
|
||||
action: lc_event.action,
|
||||
rule_id: lc_event.rule_id.clone(),
|
||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||
});
|
||||
opts.ensure_lifecycle_delete_all_journal();
|
||||
}
|
||||
|
||||
let time_ilm = Metrics::time_ilm(lc_event.action);
|
||||
|
||||
//debug!("lc_event.action: {:?}", lc_event.action);
|
||||
debug!("expiry_on_non_transitioned_objects opts: {:?}", opts);
|
||||
let mut dobj = match api
|
||||
.delete_object_with_tier_delete_journal(&oi.bucket, &encode_dir_object(&oi.name), opts)
|
||||
.await
|
||||
{
|
||||
let mut dobj = match api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await {
|
||||
Ok(dobj) => dobj,
|
||||
Err(e) => {
|
||||
error!(
|
||||
@@ -5297,7 +5283,7 @@ mod tests {
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG};
|
||||
use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
use crate::bucket::metadata_sys;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::client::transition_api::ReaderImpl;
|
||||
@@ -5318,7 +5304,6 @@ mod tests {
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
lifecycle::ExpirationOptions,
|
||||
list::ListOperations as _,
|
||||
multipart::MultipartOperations as _,
|
||||
object::{ObjectIO as _, ObjectOperations as _},
|
||||
};
|
||||
@@ -10932,199 +10917,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_delete_all_rechecks_a_same_id_rule_moved_into_the_future() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("stale-delete-all-rule-{}", Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
metadata_sys::update(
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("bucket versioning should be enabled");
|
||||
let lifecycle_xml = |days| {
|
||||
format!(
|
||||
r#"<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>delete-marker-history</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<DelMarkerExpiration><Days>{days}</Days></DelMarkerExpiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
)
|
||||
};
|
||||
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(1).into_bytes())
|
||||
.await
|
||||
.expect("initial lifecycle rule should be stored");
|
||||
|
||||
let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
|
||||
let mut reader = PutObjReader::from_vec(b"old version".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time - time::Duration::hours(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("old version should be stored");
|
||||
let marker = ecstore
|
||||
.delete_object(
|
||||
&bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete marker should be created");
|
||||
|
||||
let queued_event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::DelMarkerDeleteAllVersionsAction,
|
||||
rule_id: "delete-marker-history".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(30).into_bytes())
|
||||
.await
|
||||
.expect("updated lifecycle rule should be stored");
|
||||
let incarnation = ecstore
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("bucket incarnation should be available");
|
||||
|
||||
let deleted = super::apply_expiry_on_non_transitioned_objects(
|
||||
ecstore.clone(),
|
||||
&marker,
|
||||
&queued_event,
|
||||
&LcEventSrc::Scanner,
|
||||
incarnation,
|
||||
)
|
||||
.await;
|
||||
assert!(!deleted, "the stale queued rule must be rejected");
|
||||
let versions = ecstore
|
||||
.clone()
|
||||
.list_object_versions(&bucket, object, None, None, None, 10)
|
||||
.await
|
||||
.expect("remaining versions should be listable");
|
||||
assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 2);
|
||||
|
||||
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(1).into_bytes())
|
||||
.await
|
||||
.expect("due lifecycle rule should be restored");
|
||||
let deleted = super::apply_expiry_on_non_transitioned_objects(
|
||||
ecstore.clone(),
|
||||
&marker,
|
||||
&queued_event,
|
||||
&LcEventSrc::Scanner,
|
||||
incarnation,
|
||||
)
|
||||
.await;
|
||||
assert!(deleted, "the current due rule should purge marker and history");
|
||||
let versions = ecstore
|
||||
.clone()
|
||||
.list_object_versions(&bucket, object, None, None, None, 10)
|
||||
.await
|
||||
.expect("purged versions should be listable");
|
||||
assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_expired_object_all_versions_purges_history_through_transitioned_dispatch() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("expired-all-versions-{}", Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
metadata_sys::update(
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("bucket versioning should be enabled");
|
||||
metadata_sys::update(
|
||||
&bucket,
|
||||
BUCKET_LIFECYCLE_CONFIG,
|
||||
br#"<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>delete-all-versions</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("delete-all lifecycle rule should be stored");
|
||||
|
||||
let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
|
||||
let mut old_reader = PutObjReader::from_vec(b"old version".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut old_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time - time::Duration::hours(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("old version should be stored");
|
||||
let mut current_reader = PutObjReader::from_vec(b"current version".to_vec());
|
||||
let mut current = ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut current_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("current version should be stored");
|
||||
current.transitioned_object.status = crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string();
|
||||
|
||||
let incarnation = ecstore
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("bucket incarnation should be available");
|
||||
let deleted = super::apply_expiry_on_transitioned_object(
|
||||
ecstore.clone(),
|
||||
¤t,
|
||||
&crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::DeleteAllVersionsAction,
|
||||
rule_id: "delete-all-versions".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
&LcEventSrc::Scanner,
|
||||
incarnation,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(deleted, "delete-all must not degrade to transitioned single-version expiry");
|
||||
let versions = ecstore
|
||||
.list_object_versions(&bucket, object, None, None, None, 10)
|
||||
.await
|
||||
.expect("purged versions should be listable");
|
||||
assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_object_lifecycle_skips_current_expiration_for_explicit_legal_hold() {
|
||||
let lc = latest_expiration_lifecycle();
|
||||
|
||||
@@ -435,88 +435,12 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
fn object_info_references_tier_delete(info: &ObjectInfo, je: &Jentry) -> std::io::Result<bool> {
|
||||
if info.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE
|
||||
|| info.transitioned_object.name != je.obj_name
|
||||
|| info.transitioned_object.tier != je.tier_name
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let source_backend_identity = tier_destination_id_from_metadata(&info.user_defined)?;
|
||||
if source_backend_identity.is_some() && source_backend_identity != je.backend_identity {
|
||||
return Ok(false);
|
||||
}
|
||||
if !je.version_id_exact {
|
||||
return Ok(true);
|
||||
}
|
||||
Ok(match info.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => true,
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled => false,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact => {
|
||||
info.transitioned_object.version_id == je.version_id
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn prepared_tier_delete_has_live_source(
|
||||
api: &ECStore,
|
||||
source: &TierDeleteSourceIdentity,
|
||||
je: &Jentry,
|
||||
) -> std::io::Result<(bool, Vec<crate::store::ObjectLockDiagGuard>)> {
|
||||
let lock_object = rustfs_utils::path::encode_dir_object(&source.object);
|
||||
let mut lock_opts = ObjectOptions::default();
|
||||
let read_guards = api
|
||||
.acquire_all_object_read_locks("tier_delete_journal_recovery", &source.bucket, &lock_object, &mut lock_opts)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
if api.ctx.lock_manager().is_disabled() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery requires namespace locking",
|
||||
));
|
||||
}
|
||||
let mut has_live_source = false;
|
||||
for pool in &api.pools {
|
||||
let set = pool.get_disks_by_key(&lock_object);
|
||||
let Some(versions) = set
|
||||
.load_file_info_versions_exact(&source.bucket, &source.object)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
for version in versions.versions.iter().filter(|version| !version.tier_free_version()) {
|
||||
let info = ObjectInfo::from_file_info(version, &source.bucket, &source.object, source.versioned);
|
||||
if object_info_references_tier_delete(&info, je)? {
|
||||
has_live_source = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if has_live_source {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery object read lock was lost",
|
||||
));
|
||||
}
|
||||
Ok((has_live_source, read_guards))
|
||||
}
|
||||
|
||||
async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
let journal_name = tier_delete_journal_object_name(je);
|
||||
let (data, metadata) = config_boundary::read_config_with_metadata(api.clone(), &journal_name, &ObjectOptions::default())
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let (data, metadata) =
|
||||
config_boundary::read_config_with_metadata(api.clone(), &tier_delete_journal_object_name(je), &ObjectOptions::default())
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?;
|
||||
if tier_delete_journal_object_name(¤t) != journal_name {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"prepared tier delete journal content does not match its object name",
|
||||
));
|
||||
}
|
||||
if current.state != TierDeleteJournalState::Prepared {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
@@ -529,21 +453,16 @@ async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Je
|
||||
"prepared tier delete journal has no entity tag",
|
||||
));
|
||||
};
|
||||
let source = current
|
||||
let source = je
|
||||
.source
|
||||
.as_ref()
|
||||
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "prepared tier delete journal has no source"))?;
|
||||
match prepared_tier_delete_has_live_source(&api, source, ¤t).await {
|
||||
Ok((true, read_guards)) => {
|
||||
if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery object read lock was lost before abort",
|
||||
));
|
||||
}
|
||||
let result = config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await;
|
||||
drop(read_guards);
|
||||
match result {
|
||||
match api
|
||||
.get_object_info(&source.bucket, &source.object, &source.lookup_options())
|
||||
.await
|
||||
{
|
||||
Ok(info) if source.matches(&info) => {
|
||||
match config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
@@ -552,32 +471,17 @@ async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Je
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
Ok((false, read_guards)) if source.has_stable_identity() => {
|
||||
if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery object read lock was lost before commit",
|
||||
));
|
||||
}
|
||||
let mut commit_opts = ObjectOptions::default();
|
||||
for signal in read_guards
|
||||
.iter()
|
||||
.filter_map(crate::store::ObjectLockDiagGuard::lock_lost_signal)
|
||||
{
|
||||
commit_opts.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
let committed =
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api.clone(), current, etag, &commit_opts).await?;
|
||||
// Keep namespace locks only through the journal CAS. Remote-tier IO
|
||||
// must not block writers for the object during recovery.
|
||||
drop(read_guards);
|
||||
process_committed_tier_delete_journal_entry(api, &committed).await
|
||||
Ok(_info) if source.has_stable_identity() => {
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
|
||||
}
|
||||
Ok((false, _read_guards)) => Err(std::io::Error::new(
|
||||
Ok(_) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal source identity is not sufficient to confirm deletion",
|
||||
)),
|
||||
Err(err) => Err(err),
|
||||
Err(Error::ObjectNotFound(_, _)) | Err(Error::FileNotFound) | Err(Error::FileVersionNotFound) => {
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
|
||||
}
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -585,8 +489,7 @@ async fn commit_prepared_tier_delete_journal_entry_if_current(
|
||||
api: Arc<ECStore>,
|
||||
mut committed: Jentry,
|
||||
etag: String,
|
||||
lock_opts: &ObjectOptions,
|
||||
) -> std::io::Result<Jentry> {
|
||||
) -> std::io::Result<()> {
|
||||
committed.state = TierDeleteJournalState::Committed;
|
||||
let data = encode_tier_delete_journal_entry(&committed).map_err(std::io::Error::other)?;
|
||||
match config_boundary::save_config_with_opts(
|
||||
@@ -599,13 +502,12 @@ async fn commit_prepared_tier_delete_journal_entry_if_current(
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
}),
|
||||
namespace_lock_fence: lock_opts.namespace_lock_fence.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(committed),
|
||||
Ok(()) => process_committed_tier_delete_journal_entry(api, &committed).await,
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before commit",
|
||||
@@ -680,18 +582,6 @@ pub async fn recover_tier_delete_journal_entries(
|
||||
}
|
||||
};
|
||||
|
||||
if tier_delete_journal_object_name(&je) != object.name {
|
||||
stats.failed += 1;
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
journal_object = %object.name,
|
||||
"Tier delete journal content does not match its object name and will be retained"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if je.backend_identity.is_none() {
|
||||
stats.failed += 1;
|
||||
warn!(
|
||||
@@ -809,14 +699,16 @@ where
|
||||
mod tests {
|
||||
use super::{
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, await_tier_delete_journal_recovery,
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, object_info_references_tier_delete,
|
||||
record_tier_delete_journal_backend_identity, tier_delete_journal_object_name,
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
tier_delete_journal_object_name,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, TierDeleteJournalState, TierDeleteSourceIdentity};
|
||||
use crate::error::Result;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn journal_entry() -> Jentry {
|
||||
Jentry {
|
||||
@@ -846,16 +738,6 @@ mod tests {
|
||||
assert_eq!(decoded.version_state, je.version_state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_object_name_binds_persisted_content() {
|
||||
let original = journal_entry();
|
||||
let original_name = tier_delete_journal_object_name(&original);
|
||||
let mut replaced = original;
|
||||
replaced.obj_name = "remote/replaced".to_string();
|
||||
|
||||
assert_ne!(tier_delete_journal_object_name(&replaced), original_name);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_transaction_roundtrips_prepared_source_identity() {
|
||||
let mut je = journal_entry();
|
||||
@@ -883,35 +765,26 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepared_recovery_blocks_any_live_reference_to_the_remote_version() {
|
||||
let je = journal_entry();
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(je.backend_identity.expect("test journal should bind a backend")),
|
||||
);
|
||||
let mut info = ObjectInfo {
|
||||
user_defined: std::sync::Arc::new(metadata),
|
||||
transitioned_object: crate::storage_api_contracts::lifecycle::TransitionedObject {
|
||||
name: je.obj_name.clone(),
|
||||
version_id: je.version_id.clone(),
|
||||
tier: je.tier_name.clone(),
|
||||
status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
fn tier_delete_source_identity_rejects_recreated_object() {
|
||||
let version_id = Uuid::from_u128(1);
|
||||
let data_dir = Uuid::from_u128(2);
|
||||
let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1);
|
||||
let info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(version_id),
|
||||
data_dir: Some(data_dir),
|
||||
mod_time: Some(mod_time),
|
||||
..Default::default()
|
||||
};
|
||||
let source = TierDeleteSourceIdentity::from_object_info("bucket", "object", &info, true, false);
|
||||
assert!(source.matches(&info));
|
||||
|
||||
assert!(object_info_references_tier_delete(&info, &je).expect("matching reference should be valid"));
|
||||
info.transitioned_object.version_id = "other-version".to_string();
|
||||
assert!(!object_info_references_tier_delete(&info, &je).expect("different exact version should be valid"));
|
||||
info.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
assert!(
|
||||
object_info_references_tier_delete(&info, &je).expect("legacy unknown reference should fail closed"),
|
||||
"an unknown live source may still reference the journaled remote version"
|
||||
);
|
||||
let recreated = ObjectInfo {
|
||||
data_dir: Some(Uuid::from_u128(3)),
|
||||
..info
|
||||
};
|
||||
assert!(!source.matches(&recreated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -334,6 +334,32 @@ impl TierDeleteSourceIdentity {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn lookup_options(&self) -> crate::object_api::ObjectOptions {
|
||||
crate::object_api::ObjectOptions {
|
||||
version_id: self.version_id.clone(),
|
||||
versioned: self.versioned,
|
||||
version_suspended: self.version_suspended,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, info: &ObjectInfo) -> bool {
|
||||
if self.bucket != info.bucket {
|
||||
return false;
|
||||
}
|
||||
if let Some(version_id) = &self.version_id {
|
||||
return info.version_id.map(|id| id.to_string()).as_deref() == Some(version_id.as_str())
|
||||
&& self.data_dir == info.data_dir.map(|id| id.to_string());
|
||||
}
|
||||
if self.data_dir.is_some() {
|
||||
return self.data_dir == info.data_dir.map(|id| id.to_string());
|
||||
}
|
||||
self.etag.is_some()
|
||||
&& self.etag == info.etag
|
||||
&& self.mod_time.is_some()
|
||||
&& self.mod_time == info.mod_time.map(|time| time.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn has_stable_identity(&self) -> bool {
|
||||
self.version_id.is_some() || self.data_dir.is_some() || (self.etag.is_some() && self.mod_time.is_some())
|
||||
}
|
||||
|
||||
@@ -1016,7 +1016,7 @@ pub async fn recover_transition_transaction_records(
|
||||
return Err(Error::other("transition transaction recovery limit must be greater than zero"));
|
||||
}
|
||||
|
||||
let list_limit = i32::try_from(limit).unwrap_or(i32::MAX);
|
||||
let list_limit = i32::try_from(limit).map_or(i32::MAX, |value| value);
|
||||
let list = api
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
|
||||
@@ -41,7 +41,6 @@ const IAM_FORMAT_FILE_PATH: &str = "config/iam/format.json";
|
||||
const IAM_USERS_PREFIX: &str = "config/iam/users/";
|
||||
const IAM_SERVICE_ACCOUNTS_PREFIX: &str = "config/iam/service-accounts/";
|
||||
const IAM_STS_PREFIX: &str = "config/iam/sts/";
|
||||
const MINIO_GO_ZERO_TIME: OffsetDateTime = time::macros::datetime!(0001-01-01 00:00 UTC);
|
||||
const IAM_GROUPS_PREFIX: &str = "config/iam/groups/";
|
||||
const IAM_POLICIES_PREFIX: &str = "config/iam/policies/";
|
||||
const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
|
||||
@@ -121,15 +120,6 @@ fn normalize_iam_config_blob(path: &str, data: &[u8]) -> std::result::Result<Opt
|
||||
if is_identity_path(path) {
|
||||
let mut identity: UserIdentity =
|
||||
serde_json::from_slice(data).map_err(|err| format!("parse IAM identity failed: {err}"))?;
|
||||
if (path.starts_with(IAM_USERS_PREFIX) || path.starts_with(IAM_SERVICE_ACCOUNTS_PREFIX))
|
||||
&& identity
|
||||
.credentials
|
||||
.expiration
|
||||
.as_ref()
|
||||
.is_some_and(|expiration| *expiration == MINIO_GO_ZERO_TIME || *expiration == OffsetDateTime::UNIX_EPOCH)
|
||||
{
|
||||
identity.credentials.expiration = None;
|
||||
}
|
||||
if identity.update_at.is_none() {
|
||||
identity.update_at = Some(OffsetDateTime::now_utc());
|
||||
}
|
||||
@@ -451,10 +441,7 @@ mod tests {
|
||||
use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, ReplicationMigrationBridge, ResyncStatusType, TargetReplicationResyncStatus,
|
||||
};
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
#[test]
|
||||
fn test_normalize_policy_mapping_legacy_timestamp_and_fields() {
|
||||
@@ -506,54 +493,6 @@ mod tests {
|
||||
assert!(v.get("updatedAt").is_some(), "normalize should backfill updatedAt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_minio_permanent_credential_expiration() {
|
||||
let cases = [
|
||||
("config/iam/users/alice/identity.json", "0001-01-01T00:00:00Z", true),
|
||||
("config/iam/users/alice/identity.json", "1970-01-01T00:00:00Z", true),
|
||||
("config/iam/service-accounts/svc/identity.json", "0001-01-01T00:00:00Z", true),
|
||||
("config/iam/service-accounts/svc/identity.json", "1970-01-01T00:00:00Z", true),
|
||||
("config/iam/service-accounts/svc/identity.json", "1970-01-01T00:00:00.000000001Z", false),
|
||||
("config/iam/sts/temp/identity.json", "0001-01-01T00:00:00Z", false),
|
||||
("config/iam/sts/temp/identity.json", "1970-01-01T00:00:00Z", false),
|
||||
("config/iam/users/alice/identity.json", "1969-12-31T23:59:59Z", false),
|
||||
("config/iam/users/alice/identity.json", "1970-01-01T00:00:00.000000001Z", false),
|
||||
("config/iam/users/alice/identity.json", "0001-01-01T00:00:00.000000001Z", false),
|
||||
("config/iam/users/alice/identity.json", "2030-01-01T00:00:00Z", false),
|
||||
];
|
||||
|
||||
for (path, expiration, should_clear) in cases {
|
||||
let input = serde_json::json!({
|
||||
"version": 1,
|
||||
"credentials": {
|
||||
"accessKey": "test-access",
|
||||
"secretKey": "test-secret",
|
||||
"sessionToken": "test-session-token",
|
||||
"parentUser": "test-parent",
|
||||
"expiration": expiration,
|
||||
}
|
||||
});
|
||||
let output = normalize_iam_config_blob(path, &serde_json::to_vec(&input).expect("serialize identity fixture"))
|
||||
.expect("normalize should succeed")
|
||||
.expect("identity path should be supported");
|
||||
let identity: UserIdentity = serde_json::from_slice(&output).expect("deserialize normalized identity");
|
||||
|
||||
assert_eq!(identity.credentials.access_key, "test-access");
|
||||
assert_eq!(identity.credentials.secret_key, "test-secret");
|
||||
assert_eq!(identity.credentials.session_token, "test-session-token");
|
||||
assert_eq!(identity.credentials.parent_user, "test-parent");
|
||||
if should_clear {
|
||||
assert_eq!(identity.credentials.expiration, None, "path: {path}, expiration: {expiration}");
|
||||
} else {
|
||||
assert_eq!(
|
||||
identity.credentials.expiration,
|
||||
Some(OffsetDateTime::parse(expiration, &Rfc3339).expect("parse expected expiration")),
|
||||
"path: {path}, expiration: {expiration}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_bucket_meta_blob_resync_reencode() {
|
||||
let path = ".buckets/test/.replication/resync.bin";
|
||||
|
||||
@@ -76,12 +76,7 @@ impl QuotaChecker {
|
||||
|
||||
let current_usage = self.get_real_time_usage(bucket).await?;
|
||||
|
||||
// The reporting path projects this operation; storage mutations reserve it at commit.
|
||||
let admission_size = if uses_durable_reservations && !force_usage_calculation {
|
||||
0
|
||||
} else {
|
||||
operation_size
|
||||
};
|
||||
let admission_size = if uses_durable_reservations { 0 } else { operation_size };
|
||||
let expected_usage = match operation {
|
||||
QuotaOperation::PutObject | QuotaOperation::PostObject | QuotaOperation::CopyObject => {
|
||||
current_usage.saturating_add(admission_size)
|
||||
|
||||
@@ -23,8 +23,6 @@ use super::replication_queue_boundary::DeletedObjectReplicationInfo;
|
||||
use super::replication_storage_boundary::{
|
||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -34,9 +32,6 @@ pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
||||
|
||||
pub(crate) struct ReplicationLifecycleBridge;
|
||||
|
||||
#[cfg(test)]
|
||||
static SCHEDULED_DELETE_OBJECTS: Mutex<Vec<DeletedObject>> = Mutex::new(Vec::new());
|
||||
|
||||
impl ReplicationLifecycleBridge {
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -90,13 +85,6 @@ impl ReplicationLifecycleBridge {
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
|
||||
#[cfg(test)]
|
||||
{
|
||||
SCHEDULED_DELETE_OBJECTS
|
||||
.lock()
|
||||
.expect("scheduled delete test hook lock should not poison")
|
||||
.push(delete_object.clone());
|
||||
}
|
||||
super::replication_pool::schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket,
|
||||
@@ -105,15 +93,6 @@ impl ReplicationLifecycleBridge {
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn take_scheduled_deletes_for_test() -> Vec<DeletedObject> {
|
||||
std::mem::take(
|
||||
&mut *SCHEDULED_DELETE_OBJECTS
|
||||
.lock()
|
||||
.expect("scheduled delete test hook lock should not poison"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -52,8 +52,8 @@ use super::replication_storage_boundary::{
|
||||
ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
|
||||
};
|
||||
use super::replication_target_boundary::{
|
||||
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
|
||||
ReplicationTargetStore, SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error,
|
||||
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, PutObjectOptions, PutObjectPartOptions, ReplicationTargetStore,
|
||||
SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error,
|
||||
replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options,
|
||||
replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted,
|
||||
replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version,
|
||||
@@ -214,7 +214,7 @@ async fn head_object_for_worker(
|
||||
target_bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> std::result::Result<HeadObjectOutput, HeadObjectSdkError> {
|
||||
) -> std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>> {
|
||||
target_client.head_object(target_bucket, object, version_id).await
|
||||
}
|
||||
|
||||
@@ -233,7 +233,7 @@ async fn mark_replication_target_offline_if_needed(target_client: &Arc<TargetCli
|
||||
async fn head_object_fallback(
|
||||
tgt_client: &TargetClient,
|
||||
object: &str,
|
||||
) -> std::result::Result<Option<HeadObjectOutput>, HeadObjectSdkError> {
|
||||
) -> std::result::Result<Option<HeadObjectOutput>, SdkError<HeadObjectError>> {
|
||||
match head_object_for_worker(tgt_client, &tgt_client.bucket, object, None).await {
|
||||
Ok(oi) => Ok(Some(oi)),
|
||||
Err(e) if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) => Ok(None),
|
||||
@@ -1152,11 +1152,11 @@ fn spawn_resync_walk_task<S: ReplicationStorage>(
|
||||
/// updating the per-object status counters and returning the accounted size
|
||||
/// together with any verification error.
|
||||
async fn verify_resync_head_result(
|
||||
head_result: std::result::Result<HeadObjectOutput, HeadObjectSdkError>,
|
||||
head_result: std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>>,
|
||||
roi: &ReplicateObjectInfo,
|
||||
st: &mut TargetReplicationResyncStatus,
|
||||
target_client: &Arc<TargetClient>,
|
||||
) -> (i64, Option<HeadObjectSdkError>) {
|
||||
) -> (i64, Option<SdkError<HeadObjectError>>) {
|
||||
match head_result {
|
||||
Ok(_) => {
|
||||
st.replicated_count += 1;
|
||||
@@ -1275,7 +1275,7 @@ async fn resync_worker_process_object<S: ReplicationStorage>(
|
||||
"Processed resync object"
|
||||
);
|
||||
}
|
||||
st.error = err.as_ref().and_then(|err| resync_target_error_detail(err.as_ref()));
|
||||
st.error = err.as_ref().and_then(resync_target_error_detail);
|
||||
|
||||
st
|
||||
}
|
||||
@@ -2467,7 +2467,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
let non_retryable = matches!(
|
||||
e.as_ref(),
|
||||
&e,
|
||||
SdkError::ServiceError(service_err)
|
||||
if is_retryable_delete_replication_head_error(
|
||||
service_err.err().is_not_found(),
|
||||
|
||||
@@ -36,8 +36,7 @@ use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
||||
resolve_read_api_version_id,
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient, resolve_read_api_version_id,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
|
||||
@@ -41,10 +41,6 @@ use bytes::Bytes;
|
||||
use futures::lock::Mutex;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
|
||||
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
|
||||
};
|
||||
use rustfs_protos::ChannelClass;
|
||||
use rustfs_protos::evict_failed_connection;
|
||||
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
|
||||
@@ -68,7 +64,7 @@ use std::{
|
||||
atomic::{AtomicBool, AtomicU32, Ordering},
|
||||
},
|
||||
task::{Context, Poll},
|
||||
time::{Duration, Instant},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::time;
|
||||
use tokio::{
|
||||
@@ -1794,16 +1790,6 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_
|
||||
}
|
||||
}
|
||||
|
||||
fn read_version_stage_timer(attribution_enabled: bool) -> Option<Instant> {
|
||||
attribution_enabled.then(Instant::now)
|
||||
}
|
||||
|
||||
fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
|
||||
if let Some(started_at) = started_at {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_stage(stage, started_at.elapsed());
|
||||
}
|
||||
}
|
||||
|
||||
/// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
|
||||
/// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
|
||||
/// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
|
||||
@@ -2719,11 +2705,8 @@ impl DiskAPI for RemoteDisk {
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
let read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let encode_started = read_version_stage_timer(read_version_attribution_enabled);
|
||||
let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin)));
|
||||
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, encode_started);
|
||||
let (opts_str, opts_bin) = encoded_opts?;
|
||||
let opts_str = compat_json(opts)?;
|
||||
let opts_bin = encode_msgpack(opts)?;
|
||||
|
||||
// Idempotent version read: eligible for the bounded transient-network retry so a single
|
||||
// reset-by-peer during the read-after-write window does not erode the metadata read
|
||||
@@ -2739,14 +2722,6 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request_payload_bytes = read_version_attribution_enabled.then(|| {
|
||||
disk.len()
|
||||
.saturating_add(volume.len())
|
||||
.saturating_add(path.len())
|
||||
.saturating_add(version_id.len())
|
||||
.saturating_add(opts_str.len())
|
||||
.saturating_add(opts_bin.len())
|
||||
});
|
||||
let request = Request::new(ReadVersionRequest {
|
||||
disk,
|
||||
volume: volume.to_string(),
|
||||
@@ -2756,47 +2731,14 @@ impl DiskAPI for RemoteDisk {
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_request();
|
||||
if let Some(request_payload_bytes) = request_payload_bytes {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_sent_bytes(request_payload_bytes);
|
||||
}
|
||||
let rpc_started = read_version_stage_timer(read_version_attribution_enabled);
|
||||
let response = match client.read_version(request).await {
|
||||
Ok(response) => {
|
||||
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
response.into_inner()
|
||||
}
|
||||
Err(err) => {
|
||||
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
let response = client.read_version(request).await?.into_inner();
|
||||
|
||||
if !response.success {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_recv_bytes(
|
||||
response.file_info.len().saturating_add(response.file_info_bin.len()),
|
||||
);
|
||||
let decode_started = read_version_stage_timer(read_version_attribution_enabled);
|
||||
let file_info = match decode_msgpack_or_json::<FileInfo>(&response.file_info_bin, &response.file_info, "FileInfo")
|
||||
.and_then(|file_info| {
|
||||
validate_decoded_file_info(&file_info)?;
|
||||
Ok(file_info)
|
||||
}) {
|
||||
Ok(file_info) => {
|
||||
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, decode_started);
|
||||
file_info
|
||||
}
|
||||
Err(err) => {
|
||||
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, decode_started);
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let file_info = decode_msgpack_or_json::<FileInfo>(&response.file_info_bin, &response.file_info, "FileInfo")?;
|
||||
validate_decoded_file_info(&file_info)?;
|
||||
|
||||
Ok(file_info)
|
||||
},
|
||||
@@ -7989,17 +7931,12 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn read_version_uses_the_metadata_timeout_on_a_stalled_peer() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let Some((base_addr, accept_task)) = spawn_stalled_grpc_peer().await else {
|
||||
return;
|
||||
};
|
||||
let remote_disk = remote_disk_for_addr(&base_addr).await;
|
||||
let metrics = rustfs_io_metrics::internode_metrics::global_internode_metrics();
|
||||
let previous_stage_metrics = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
metrics.reset_for_test();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
@@ -8023,18 +7960,6 @@ mod tests {
|
||||
)
|
||||
.await;
|
||||
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(previous_stage_metrics);
|
||||
let snapshot = metrics.snapshot();
|
||||
assert!(
|
||||
snapshot.outgoing_requests_total >= 1,
|
||||
"ReadVersion call site should record outgoing attempts when attribution is enabled"
|
||||
);
|
||||
assert!(
|
||||
snapshot.sent_bytes_total > 0,
|
||||
"ReadVersion call site should record request payload bytes when attribution is enabled"
|
||||
);
|
||||
metrics.reset_for_test();
|
||||
|
||||
remote_disk.cancel_token.cancel();
|
||||
accept_task.abort();
|
||||
}
|
||||
|
||||
@@ -14,11 +14,10 @@
|
||||
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
#[cfg(test)]
|
||||
use rustfs_io_metrics::internode_metrics::InternodeMetricsSnapshot;
|
||||
@@ -83,59 +82,6 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
|
||||
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_request() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_error() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics()
|
||||
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_sent_bytes(bytes: usize) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
bytes,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_recv_bytes(bytes: usize) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
bytes,
|
||||
);
|
||||
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_VERSION, bytes);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_stage(stage: &'static str, duration: Duration) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_stage_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
stage,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_all_recv_bytes(bytes: usize) {
|
||||
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
|
||||
@@ -94,9 +94,6 @@ const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
|
||||
const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30;
|
||||
const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
|
||||
const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
|
||||
/// Background decommission walks must tolerate slow object migrations; the
|
||||
/// stall timeout is the drive-health bound, not the total listing duration.
|
||||
const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
|
||||
pub const POOL_META_NAME: &str = "pool.bin";
|
||||
pub const POOL_META_FORMAT: u16 = 1;
|
||||
@@ -1044,13 +1041,7 @@ fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: boo
|
||||
fn is_decommission_copy_cleanup_safe_error(err: &Error) -> bool {
|
||||
// DataMovementOverwriteErr only means source and destination pool resolved to
|
||||
// the same pool. Without a target equivalence check it is not cleanup-safe.
|
||||
if is_err_object_not_found(err) || is_err_version_not_found(err) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// A not-found surfacing from inside a data-movement stage is the same
|
||||
// condition once the wrapper is unwrapped (backlog#1827 T2).
|
||||
crate::data_movement::data_movement_stage_source(err).is_some_and(is_decommission_copy_cleanup_safe_error)
|
||||
is_err_object_not_found(err) || is_err_version_not_found(err)
|
||||
}
|
||||
|
||||
fn is_decommission_target_capacity_error(err: &Error) -> bool {
|
||||
@@ -1058,13 +1049,6 @@ fn is_decommission_target_capacity_error(err: &Error) -> bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
// A stage failure keeps the error it wrapped, so classify by type rather
|
||||
// than by the rendered message (backlog#1827 T2). The substring fallback
|
||||
// stays for errors that reached here through some other wrapper.
|
||||
if let Some(source) = crate::data_movement::data_movement_stage_source(err) {
|
||||
return is_decommission_target_capacity_error(source);
|
||||
}
|
||||
|
||||
let message = err.to_string();
|
||||
let disk_full = Error::DiskFull.to_string();
|
||||
let storage_full = Error::StorageFull.to_string();
|
||||
@@ -4443,36 +4427,6 @@ mod tests {
|
||||
assert!(is_decommission_target_capacity_error(&Error::StorageFull));
|
||||
}
|
||||
|
||||
/// The decommission loop classifies errors that came back through a
|
||||
/// data-movement stage wrapper. Before backlog#1827 T2 the wrapper flattened
|
||||
/// everything into `Error::other(String)`, so these two classifiers had to
|
||||
/// match on rendered text; now the wrapped error is recoverable by type.
|
||||
#[test]
|
||||
fn decommission_classifiers_see_through_a_stage_wrapper() {
|
||||
let wrap = |inner: Error| {
|
||||
crate::data_movement::data_movement_stage_error_for_test(
|
||||
"decommission_object",
|
||||
"put_object",
|
||||
"bucket-a",
|
||||
"object-a",
|
||||
inner,
|
||||
)
|
||||
};
|
||||
|
||||
// Capacity: the target pool filling up must still stop the loop.
|
||||
assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
|
||||
assert!(is_decommission_target_capacity_error(&wrap(Error::StorageFull)));
|
||||
assert!(!is_decommission_target_capacity_error(&wrap(Error::SlowDown)));
|
||||
|
||||
// Cleanup safety: a not-found surfacing from inside a stage is the same
|
||||
// condition as one surfacing directly, so the source entry stays
|
||||
// eligible for cleanup.
|
||||
let not_found = Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string());
|
||||
assert!(is_decommission_copy_cleanup_safe_error(¬_found));
|
||||
assert!(is_decommission_copy_cleanup_safe_error(&wrap(not_found)));
|
||||
assert!(!is_decommission_copy_cleanup_safe_error(&wrap(Error::SlowDown)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
|
||||
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
|
||||
@@ -5050,8 +5004,6 @@ impl SetDisks {
|
||||
path: bucket_info.prefix.clone(),
|
||||
recursive: true,
|
||||
min_disks: listing_quorum,
|
||||
skip_walkdir_total_timeout: true,
|
||||
walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
|
||||
agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
||||
let resolver = resolver.clone();
|
||||
|
||||
@@ -471,60 +471,8 @@ fn resolve_data_movement_abort_result(
|
||||
))
|
||||
}
|
||||
|
||||
/// A data-movement stage failure that keeps the error it wrapped.
|
||||
///
|
||||
/// The rendered message is byte-identical to the `format!` this replaced, so
|
||||
/// logs and any message-matching callers are unaffected. What changes is that
|
||||
/// the original error stays reachable through `source()`, which is what lets
|
||||
/// the decommission loop classify by type instead of by substring
|
||||
/// (backlog#1827 T2).
|
||||
#[derive(Debug)]
|
||||
struct DataMovementStageError {
|
||||
rendered: String,
|
||||
source: Box<dyn std::error::Error + Send + Sync>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for DataMovementStageError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.rendered)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for DataMovementStageError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
Some(self.source.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, object: &str, err: E) -> Error
|
||||
where
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let rendered = format!("{op_label}: {stage} failed for {bucket}/{object}: {err}");
|
||||
Error::other(DataMovementStageError {
|
||||
rendered,
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn data_movement_stage_error_for_test(op_label: &str, stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
data_movement_stage_error(op_label, stage, bucket, object, err)
|
||||
}
|
||||
|
||||
/// Recover the error a [`data_movement_stage_error`] wrapped, if this is one.
|
||||
///
|
||||
/// `Error::other` boxes through `std::io::Error`, so the chain is
|
||||
/// `StorageError::Io` -> `DataMovementStageError` -> the original error.
|
||||
pub(crate) fn data_movement_stage_source(err: &Error) -> Option<&Error> {
|
||||
let Error::Io(io_err) = err else {
|
||||
return None;
|
||||
};
|
||||
io_err
|
||||
.get_ref()?
|
||||
.downcast_ref::<DataMovementStageError>()?
|
||||
.source
|
||||
.downcast_ref::<Error>()
|
||||
fn data_movement_stage_error(op_label: &str, stage: &str, bucket: &str, object: &str, err: impl std::fmt::Display) -> Error {
|
||||
Error::other(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"))
|
||||
}
|
||||
|
||||
fn schedule_data_movement_multipart_abort_cleanup(
|
||||
@@ -1917,40 +1865,6 @@ mod tests {
|
||||
assert!(message.contains(Error::SlowDown.to_string().as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage_error_renders_exactly_as_the_format_it_replaced() {
|
||||
// The wrapper gained a source; its message must not have moved, or log
|
||||
// scrapers and any message-matching caller would break (backlog#1827 T2).
|
||||
// `Error::other` renders through `StorageError::Io`, which prefixes
|
||||
// "Io error: " — that was true of the `format!` this replaced too, so
|
||||
// the full string is what must stay stable.
|
||||
let err = data_movement_stage_error("rebalance_object", "put_object", "bucket-a", "object-a", Error::SlowDown);
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
format!("Io error: rebalance_object: put_object failed for bucket-a/object-a: {}", Error::SlowDown)
|
||||
);
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
Error::other(format!("rebalance_object: put_object failed for bucket-a/object-a: {}", Error::SlowDown)).to_string()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage_error_keeps_the_wrapped_error_recoverable() {
|
||||
for original in [Error::DiskFull, Error::StorageFull, Error::FileNotFound, Error::SlowDown] {
|
||||
let wrapped =
|
||||
data_movement_stage_error("decommission_object", "put_object", "bucket-a", "object-a", original.clone());
|
||||
let recovered = data_movement_stage_source(&wrapped).expect("the wrapped error must be recoverable");
|
||||
assert_eq!(recovered.to_string(), original.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage_source_ignores_errors_it_did_not_wrap() {
|
||||
assert!(data_movement_stage_source(&Error::DiskFull).is_none());
|
||||
assert!(data_movement_stage_source(&Error::other("plain io error")).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_movement_part_stage_error_includes_stage_object_and_part() {
|
||||
let err =
|
||||
|
||||
@@ -23,7 +23,6 @@ use std::{
|
||||
io,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio::sync::{
|
||||
@@ -326,11 +325,6 @@ pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
|
||||
const ENV_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE";
|
||||
const DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: bool = false;
|
||||
const ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE";
|
||||
const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: bool = false;
|
||||
const ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS";
|
||||
const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 0;
|
||||
const MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS: u64 = 1_000;
|
||||
#[cfg(not(test))]
|
||||
const MAX_DST_DIR_FSYNC_GROUPS: usize = 1024;
|
||||
#[cfg(test)]
|
||||
@@ -339,34 +333,9 @@ const MAX_DST_DIR_FSYNC_GROUPS: usize = 4;
|
||||
const MAX_DST_DIR_FSYNC_WAITERS: usize = 8192;
|
||||
#[cfg(test)]
|
||||
const MAX_DST_DIR_FSYNC_WAITERS: usize = 8;
|
||||
#[cfg(not(test))]
|
||||
const MAX_FILE_FDATASYNC_GROUPS: usize = 1024;
|
||||
#[cfg(test)]
|
||||
const MAX_FILE_FDATASYNC_GROUPS: usize = 4;
|
||||
#[cfg(not(test))]
|
||||
const MAX_FILE_FDATASYNC_WAITERS: usize = 8192;
|
||||
#[cfg(test)]
|
||||
const MAX_FILE_FDATASYNC_WAITERS: usize = 8;
|
||||
#[cfg(not(test))]
|
||||
const MAX_FILE_FDATASYNC_BATCH_FILES: usize = 1024;
|
||||
#[cfg(test)]
|
||||
const MAX_FILE_FDATASYNC_BATCH_FILES: usize = 8;
|
||||
static DST_DIR_FSYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_bool(ENV_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE, DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE)
|
||||
});
|
||||
static FILE_FDATASYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_bool(ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE, DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE)
|
||||
});
|
||||
fn file_fdatasync_group_commit_wait_duration(wait_micros: u64) -> Duration {
|
||||
Duration::from_micros(wait_micros.min(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS))
|
||||
}
|
||||
|
||||
static FILE_FDATASYNC_GROUP_COMMIT_WAIT: LazyLock<Duration> = LazyLock::new(|| {
|
||||
file_fdatasync_group_commit_wait_duration(rustfs_utils::get_env_u64(
|
||||
ENV_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
|
||||
DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS,
|
||||
))
|
||||
});
|
||||
|
||||
#[cfg(test)]
|
||||
mod dst_dir_fsync_group_commit_override {
|
||||
@@ -410,72 +379,6 @@ fn dst_dir_fsync_group_commit_enabled() -> bool {
|
||||
*DST_DIR_FSYNC_GROUP_COMMIT_ENABLED
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod file_fdatasync_group_commit_override {
|
||||
use std::sync::{Mutex, MutexGuard, PoisonError, RwLock};
|
||||
|
||||
static OVERRIDE: RwLock<Option<bool>> = RwLock::new(None);
|
||||
static WAIT_OVERRIDE_MICROS: RwLock<Option<u64>> = RwLock::new(None);
|
||||
static SERIAL: Mutex<()> = Mutex::new(());
|
||||
|
||||
pub(crate) fn get() -> Option<bool> {
|
||||
*OVERRIDE.read().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
pub(crate) struct OverrideGuard {
|
||||
_serial: MutexGuard<'static, ()>,
|
||||
}
|
||||
|
||||
impl Drop for OverrideGuard {
|
||||
fn drop(&mut self) {
|
||||
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = None;
|
||||
*WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = None;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn set(enabled: bool) -> OverrideGuard {
|
||||
let serial = SERIAL.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = Some(enabled);
|
||||
OverrideGuard { _serial: serial }
|
||||
}
|
||||
|
||||
pub(crate) fn set_wait_micros(wait_micros: u64) {
|
||||
*WAIT_OVERRIDE_MICROS.write().unwrap_or_else(PoisonError::into_inner) = Some(wait_micros);
|
||||
}
|
||||
|
||||
pub(crate) fn wait_micros() -> Option<u64> {
|
||||
*WAIT_OVERRIDE_MICROS.read().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_file_fdatasync_group_commit_for_test(enabled: bool) -> file_fdatasync_group_commit_override::OverrideGuard {
|
||||
file_fdatasync_group_commit_override::set(enabled)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn set_file_fdatasync_group_commit_wait_for_test(wait_micros: u64) {
|
||||
file_fdatasync_group_commit_override::set_wait_micros(wait_micros);
|
||||
}
|
||||
|
||||
fn file_fdatasync_group_commit_enabled() -> bool {
|
||||
#[cfg(test)]
|
||||
if let Some(enabled) = file_fdatasync_group_commit_override::get() {
|
||||
return enabled;
|
||||
}
|
||||
|
||||
*FILE_FDATASYNC_GROUP_COMMIT_ENABLED
|
||||
}
|
||||
|
||||
fn file_fdatasync_group_commit_wait() -> Duration {
|
||||
#[cfg(test)]
|
||||
if let Some(wait_micros) = file_fdatasync_group_commit_override::wait_micros() {
|
||||
return file_fdatasync_group_commit_wait_duration(wait_micros);
|
||||
}
|
||||
|
||||
*FILE_FDATASYNC_GROUP_COMMIT_WAIT
|
||||
}
|
||||
|
||||
#[derive(Clone, Eq, Hash, PartialEq)]
|
||||
struct DstDirFsyncGroupKey {
|
||||
canonical_path: PathBuf,
|
||||
@@ -800,273 +703,6 @@ fn clear_dst_dir_fsync_group_commit_for_test() {
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.clear_for_test();
|
||||
}
|
||||
|
||||
type FileFdatasyncGroupKey = usize;
|
||||
|
||||
struct FileFdatasyncWaiter {
|
||||
files: Vec<PathBuf>,
|
||||
enqueued_at: Option<Instant>,
|
||||
wait_role: &'static str,
|
||||
result_tx: oneshot::Sender<SharedFileFdatasyncResult>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SharedFileFdatasyncError {
|
||||
kind: io::ErrorKind,
|
||||
message: Arc<str>,
|
||||
}
|
||||
|
||||
impl SharedFileFdatasyncError {
|
||||
fn from_error(err: io::Error) -> Self {
|
||||
Self {
|
||||
kind: err.kind(),
|
||||
message: Arc::from(err.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_error(self) -> io::Error {
|
||||
io::Error::new(self.kind, self.message.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
type SharedFileFdatasyncResult = std::result::Result<(), SharedFileFdatasyncError>;
|
||||
|
||||
struct FileFdatasyncGroup {
|
||||
key: FileFdatasyncGroupKey,
|
||||
disk_permits: Weak<Semaphore>,
|
||||
inner: Mutex<FileFdatasyncGroupInner>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FileFdatasyncGroupInner {
|
||||
worker_running: bool,
|
||||
pending_files: usize,
|
||||
pending: VecDeque<FileFdatasyncWaiter>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FileFdatasyncGroupCommit {
|
||||
inner: Mutex<FileFdatasyncGroupCommitInner>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FileFdatasyncGroupCommitInner {
|
||||
groups: HashMap<FileFdatasyncGroupKey, Arc<FileFdatasyncGroup>>,
|
||||
total_waiters: usize,
|
||||
total_files: usize,
|
||||
}
|
||||
|
||||
static FILE_FDATASYNC_GROUP_COMMIT: LazyLock<FileFdatasyncGroupCommit> = LazyLock::new(FileFdatasyncGroupCommit::default);
|
||||
|
||||
impl FileFdatasyncGroupCommit {
|
||||
// Lock order: registry first, then per-group state. No path may hold a
|
||||
// group lock while acquiring the registry lock.
|
||||
fn enqueue(
|
||||
&self,
|
||||
disk_permits: Arc<Semaphore>,
|
||||
files: Vec<PathBuf>,
|
||||
) -> io::Result<(oneshot::Receiver<SharedFileFdatasyncResult>, Option<Arc<FileFdatasyncGroup>>)> {
|
||||
if files.is_empty() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"file fdatasync group commit needs at least one file",
|
||||
));
|
||||
}
|
||||
let (result_tx, result_rx) = oneshot::channel();
|
||||
let key = Arc::as_ptr(&disk_permits) as FileFdatasyncGroupKey;
|
||||
let mut registry = self.inner.lock();
|
||||
registry.groups.retain(|_, group| group.disk_permits.strong_count() > 0);
|
||||
if registry.total_waiters >= MAX_FILE_FDATASYNC_WAITERS {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
"file fdatasync group commit waiter limit reached",
|
||||
));
|
||||
}
|
||||
if registry.total_files.saturating_add(files.len()) > MAX_FILE_FDATASYNC_BATCH_FILES {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
"file fdatasync group commit file limit reached",
|
||||
));
|
||||
}
|
||||
let group = if let Some(group) = registry.groups.get(&key) {
|
||||
group.clone()
|
||||
} else {
|
||||
if registry.groups.len() >= MAX_FILE_FDATASYNC_GROUPS {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
"file fdatasync group commit active group limit reached",
|
||||
));
|
||||
}
|
||||
let group = Arc::new(FileFdatasyncGroup {
|
||||
key,
|
||||
disk_permits: Arc::downgrade(&disk_permits),
|
||||
inner: Mutex::new(FileFdatasyncGroupInner::default()),
|
||||
});
|
||||
registry.groups.insert(key, group.clone());
|
||||
group
|
||||
};
|
||||
let file_count = files.len();
|
||||
let mut group_state = group.inner.lock();
|
||||
let start_worker = !group_state.worker_running;
|
||||
let wait_role = if start_worker {
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER
|
||||
} else {
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER
|
||||
};
|
||||
group_state.pending.push_back(FileFdatasyncWaiter {
|
||||
files,
|
||||
enqueued_at: rustfs_io_metrics::put_stage_timer(),
|
||||
wait_role,
|
||||
result_tx,
|
||||
});
|
||||
group_state.pending_files += file_count;
|
||||
if start_worker {
|
||||
group_state.worker_running = true;
|
||||
}
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_group_outstanding(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS,
|
||||
group_state.pending.len(),
|
||||
);
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_group_outstanding(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_FILES,
|
||||
group_state.pending_files,
|
||||
);
|
||||
registry.total_waiters += 1;
|
||||
registry.total_files += file_count;
|
||||
drop(group_state);
|
||||
drop(registry);
|
||||
Ok((result_rx, start_worker.then_some(group)))
|
||||
}
|
||||
|
||||
fn complete_batch(&self, waiters: usize, files: usize) {
|
||||
let mut registry = self.inner.lock();
|
||||
registry.total_waiters = registry.total_waiters.saturating_sub(waiters);
|
||||
registry.total_files = registry.total_files.saturating_sub(files);
|
||||
}
|
||||
|
||||
fn remove_idle_group(&self, group: &Arc<FileFdatasyncGroup>) {
|
||||
let mut registry = self.inner.lock();
|
||||
let group_state = group.inner.lock();
|
||||
if !group_state.worker_running && group_state.pending.is_empty() {
|
||||
registry.groups.remove(&group.key);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn counts_for_test(&self) -> (usize, usize, usize) {
|
||||
let registry = self.inner.lock();
|
||||
(registry.groups.len(), registry.total_waiters, registry.total_files)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn clear_for_test(&self) {
|
||||
let mut registry = self.inner.lock();
|
||||
registry.groups.clear();
|
||||
registry.total_waiters = 0;
|
||||
registry.total_files = 0;
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_file_fdatasync_group_worker(group: Arc<FileFdatasyncGroup>) {
|
||||
loop {
|
||||
#[cfg(test)]
|
||||
file_sync_probe::run_before_group_batch();
|
||||
tokio::task::yield_now().await;
|
||||
let wait = file_fdatasync_group_commit_wait();
|
||||
if !wait.is_zero() {
|
||||
tokio::time::sleep(wait).await;
|
||||
}
|
||||
let (batch, batch_file_count): (Vec<FileFdatasyncWaiter>, usize) = {
|
||||
let mut group_state = group.inner.lock();
|
||||
let batch_file_count = group_state.pending_files;
|
||||
group_state.pending_files = 0;
|
||||
(group_state.pending.drain(..).collect(), batch_file_count)
|
||||
};
|
||||
if batch.is_empty() {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.worker_running = false;
|
||||
drop(group_state);
|
||||
FILE_FDATASYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
return;
|
||||
}
|
||||
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_group_outstanding(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_WAITERS,
|
||||
batch.len(),
|
||||
);
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_group_outstanding(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_FILES,
|
||||
batch_file_count,
|
||||
);
|
||||
for waiter in &batch {
|
||||
if let Some(enqueued_at) = waiter.enqueued_at {
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_group_wait(
|
||||
waiter.wait_role,
|
||||
enqueued_at.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
let batch_files: Vec<PathBuf> = batch.iter().flat_map(|waiter| waiter.files.iter().cloned()).collect();
|
||||
#[cfg(test)]
|
||||
file_sync_probe::record_group_batch(batch_file_count);
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_batch(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL,
|
||||
batch_file_count,
|
||||
);
|
||||
let result = if let Some(disk_permits) = group.disk_permits.upgrade() {
|
||||
run_file_sync_blocking(disk_permits, move || sync_files(&batch_files))
|
||||
.await
|
||||
.map_err(SharedFileFdatasyncError::from_error)
|
||||
} else {
|
||||
Err(SharedFileFdatasyncError::from_error(io::Error::other(
|
||||
"file fdatasync group commit limiter dropped",
|
||||
)))
|
||||
};
|
||||
FILE_FDATASYNC_GROUP_COMMIT.complete_batch(batch.len(), batch_file_count);
|
||||
|
||||
let should_stop = {
|
||||
let mut group_state = group.inner.lock();
|
||||
if group_state.pending.is_empty() {
|
||||
group_state.worker_running = false;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_stop {
|
||||
FILE_FDATASYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
}
|
||||
for waiter in batch {
|
||||
let _ = waiter.result_tx.send(result.clone());
|
||||
}
|
||||
if should_stop {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn sync_files_group_commit(files: Vec<PathBuf>, disk_permits: Arc<Semaphore>) -> io::Result<()> {
|
||||
let (result_rx, worker) = FILE_FDATASYNC_GROUP_COMMIT.enqueue(disk_permits, files)?;
|
||||
if let Some(group) = worker {
|
||||
tokio::spawn(run_file_fdatasync_group_worker(group));
|
||||
}
|
||||
|
||||
match result_rx.await {
|
||||
Ok(Ok(())) => Ok(()),
|
||||
Ok(Err(err)) => Err(err.into_error()),
|
||||
Err(_) => Err(io::Error::other("file fdatasync group worker dropped the waiter")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn file_fdatasync_group_commit_counts_for_test() -> (usize, usize, usize) {
|
||||
FILE_FDATASYNC_GROUP_COMMIT.counts_for_test()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn clear_file_fdatasync_group_commit_for_test() {
|
||||
FILE_FDATASYNC_GROUP_COMMIT.clear_for_test();
|
||||
}
|
||||
|
||||
// Small object directories are cheaper to flush in one blocking task. Multipart
|
||||
// directories fan out only once enough files can amortize per-task scheduling.
|
||||
const PARALLEL_FILE_SYNC_THRESHOLD: usize = 16;
|
||||
@@ -1244,8 +880,6 @@ pub(crate) mod file_sync_probe {
|
||||
static ATTEMPTS: AtomicUsize = AtomicUsize::new(0);
|
||||
static FAIL_ON_ATTEMPT: AtomicUsize = AtomicUsize::new(usize::MAX);
|
||||
static BLOCK: AtomicBool = AtomicBool::new(false);
|
||||
static GROUP_BATCHES: Mutex<Vec<usize>> = Mutex::new(Vec::new());
|
||||
static BEFORE_GROUP_BATCH: Mutex<Option<Box<dyn FnOnce() + Send>>> = Mutex::new(None);
|
||||
const WAIT_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
pub(crate) struct ProbeGuard;
|
||||
@@ -1271,8 +905,6 @@ pub(crate) mod file_sync_probe {
|
||||
fn drop(&mut self) {
|
||||
release();
|
||||
FAIL_ON_ATTEMPT.store(usize::MAX, Ordering::SeqCst);
|
||||
GROUP_BATCHES.lock().expect("file sync group batch recorder poisoned").clear();
|
||||
BEFORE_GROUP_BATCH.lock().expect("file sync group batch hook poisoned").take();
|
||||
ROOTS.write().expect("file sync probe lock poisoned").clear();
|
||||
}
|
||||
}
|
||||
@@ -1282,8 +914,6 @@ pub(crate) mod file_sync_probe {
|
||||
PEAK.store(0, Ordering::SeqCst);
|
||||
ATTEMPTS.store(0, Ordering::SeqCst);
|
||||
FAIL_ON_ATTEMPT.store(fail_on_attempt.unwrap_or(usize::MAX), Ordering::SeqCst);
|
||||
GROUP_BATCHES.lock().expect("file sync group batch recorder poisoned").clear();
|
||||
BEFORE_GROUP_BATCH.lock().expect("file sync group batch hook poisoned").take();
|
||||
{
|
||||
let _guard = BLOCK_MUTEX.lock().expect("file sync probe blocker poisoned");
|
||||
BLOCK.store(block, Ordering::SeqCst);
|
||||
@@ -1395,27 +1025,6 @@ pub(crate) mod file_sync_probe {
|
||||
BLOCK.store(false, Ordering::SeqCst);
|
||||
BLOCK_CONDVAR.notify_all();
|
||||
}
|
||||
|
||||
pub(super) fn record_group_batch(batch_len: usize) {
|
||||
GROUP_BATCHES
|
||||
.lock()
|
||||
.expect("file sync group batch recorder poisoned")
|
||||
.push(batch_len);
|
||||
}
|
||||
|
||||
pub(crate) fn group_batches() -> Vec<usize> {
|
||||
GROUP_BATCHES.lock().expect("file sync group batch recorder poisoned").clone()
|
||||
}
|
||||
|
||||
pub(crate) fn set_before_group_batch(hook: impl FnOnce() + Send + 'static) {
|
||||
*BEFORE_GROUP_BATCH.lock().expect("file sync group batch hook poisoned") = Some(Box::new(hook));
|
||||
}
|
||||
|
||||
pub(super) fn run_before_group_batch() {
|
||||
if let Some(hook) = BEFORE_GROUP_BATCH.lock().expect("file sync group batch hook poisoned").take() {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn sync_file(path: &Path) -> io::Result<()> {
|
||||
@@ -1486,17 +1095,9 @@ pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef<Path>, disk_permits: Arc<Semaphore>) -> io::Result<()> {
|
||||
let dir = dir.as_ref().to_path_buf();
|
||||
let scan_dir = dir.clone();
|
||||
let group_file_fdatasync = file_fdatasync_group_commit_enabled();
|
||||
let files = run_file_sync_blocking(disk_permits.clone(), move || {
|
||||
let files = regular_files(&scan_dir)?;
|
||||
if files.len() < PARALLEL_FILE_SYNC_THRESHOLD {
|
||||
if group_file_fdatasync && !files.is_empty() {
|
||||
return Ok(Some(files));
|
||||
}
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_batch(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL,
|
||||
files.len(),
|
||||
);
|
||||
sync_files(&files)?;
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = fsync_dir_std(scan_dir);
|
||||
@@ -1514,23 +1115,6 @@ pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef<Path>, disk_perm
|
||||
let Some(files) = files else {
|
||||
return Ok(());
|
||||
};
|
||||
if group_file_fdatasync && files.len() < PARALLEL_FILE_SYNC_THRESHOLD {
|
||||
sync_files_group_commit(files, disk_permits.clone()).await?;
|
||||
return run_file_sync_blocking(disk_permits, move || {
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = fsync_dir_std(dir);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
result
|
||||
})
|
||||
.await;
|
||||
}
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_batch(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL,
|
||||
files.len(),
|
||||
);
|
||||
futures::stream::iter(files.into_iter().map(Ok::<_, io::Error>))
|
||||
.try_for_each_concurrent(MAX_PARALLEL_FILE_SYNCS, |path| {
|
||||
let disk_permits = disk_permits.clone();
|
||||
@@ -6083,341 +5667,6 @@ mod tests {
|
||||
.expect("sequential file sync must succeed");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn file_fdatasync_group_commit_default_off_keeps_small_directory_serial() {
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(false);
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
std::fs::write(temp_dir.path().join("part.1"), b"shard").expect("write part");
|
||||
let _probe = file_sync_probe::set_blocking(temp_dir.path());
|
||||
let path = temp_dir.path().to_path_buf();
|
||||
let task = tokio::spawn(async move { sync_dir_files_with_limiter(path, file_sync_limiter()).await });
|
||||
file_sync_probe::wait_for_active(1).await;
|
||||
|
||||
assert_eq!(
|
||||
file_sync_probe::group_batches(),
|
||||
Vec::<usize>::new(),
|
||||
"default-off small directory sync must not enter the file fdatasync group coordinator"
|
||||
);
|
||||
file_sync_probe::release();
|
||||
task.await
|
||||
.expect("join default-off file sync")
|
||||
.expect("default-off file sync must succeed");
|
||||
assert!(
|
||||
fsync_dir_recorder::was_fsynced(temp_dir.path()),
|
||||
"default-off successful sync must fsync the source directory"
|
||||
);
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn file_fdatasync_group_commit_batches_same_disk_small_directories() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
set_file_fdatasync_group_commit_wait_for_test(0);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first_dir = temp_dir.path().join("first");
|
||||
let second_dir = temp_dir.path().join("second");
|
||||
std::fs::create_dir(&first_dir).expect("create first dir");
|
||||
std::fs::create_dir(&second_dir).expect("create second dir");
|
||||
std::fs::write(first_dir.join("part.1"), b"first").expect("write first part");
|
||||
std::fs::write(second_dir.join("part.1"), b"second").expect("write second part");
|
||||
let _probe = file_sync_probe::set_blocking(temp_dir.path());
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_batch_tx, release_batch_rx) = mpsc::channel();
|
||||
file_sync_probe::set_before_group_batch(move || {
|
||||
entered_tx.send(()).expect("signal first file fdatasync group worker");
|
||||
release_batch_rx.recv().expect("wait until second waiter is queued");
|
||||
});
|
||||
|
||||
let limiter = file_sync_limiter();
|
||||
let first_limiter = limiter.clone();
|
||||
let first_path = first_dir.clone();
|
||||
let first = tokio::spawn(async move { sync_dir_files_with_limiter(first_path, first_limiter).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("group worker hook waiter should run")
|
||||
.expect("first file fdatasync group worker should start");
|
||||
|
||||
let second_limiter = limiter.clone();
|
||||
let second_path = second_dir.clone();
|
||||
let second = tokio::spawn(async move { sync_dir_files_with_limiter(second_path, second_limiter).await });
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
if file_fdatasync_group_commit_counts_for_test().1 == 2 {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second waiter should enqueue before releasing the grouped batch");
|
||||
release_batch_tx.send(()).expect("release group batch hook");
|
||||
file_sync_probe::wait_for_active(1).await;
|
||||
|
||||
assert_eq!(
|
||||
file_sync_probe::group_batches(),
|
||||
vec![2],
|
||||
"same-disk small directory fdatasync waiters must share one observable batch"
|
||||
);
|
||||
file_sync_probe::release();
|
||||
first
|
||||
.await
|
||||
.expect("join first grouped file sync")
|
||||
.expect("first grouped file sync must succeed");
|
||||
second
|
||||
.await
|
||||
.expect("join second grouped file sync")
|
||||
.expect("second grouped file sync must succeed");
|
||||
assert!(
|
||||
fsync_dir_recorder::was_fsynced(&first_dir),
|
||||
"first source directory must still be fsynced"
|
||||
);
|
||||
assert!(
|
||||
fsync_dir_recorder::was_fsynced(&second_dir),
|
||||
"second source directory must still be fsynced"
|
||||
);
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_fdatasync_group_commit_wait_duration_uses_default_and_cap() {
|
||||
assert_eq!(DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS, 0);
|
||||
assert_eq!(
|
||||
file_fdatasync_group_commit_wait_duration(DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS),
|
||||
Duration::ZERO
|
||||
);
|
||||
assert_eq!(file_fdatasync_group_commit_wait_duration(250), Duration::from_micros(250));
|
||||
assert_eq!(
|
||||
file_fdatasync_group_commit_wait_duration(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS),
|
||||
Duration::from_micros(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS)
|
||||
);
|
||||
assert_eq!(
|
||||
file_fdatasync_group_commit_wait_duration(u64::MAX),
|
||||
Duration::from_micros(MAX_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread", start_paused = true)]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn file_fdatasync_group_commit_wait_budget_batches_late_follower() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
let wait_budget_micros = 1_000;
|
||||
let wait_budget = file_fdatasync_group_commit_wait_duration(wait_budget_micros);
|
||||
set_file_fdatasync_group_commit_wait_for_test(wait_budget_micros);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first_dir = temp_dir.path().join("first");
|
||||
let second_dir = temp_dir.path().join("second");
|
||||
std::fs::create_dir(&first_dir).expect("create first dir");
|
||||
std::fs::create_dir(&second_dir).expect("create second dir");
|
||||
std::fs::write(first_dir.join("part.1"), b"first").expect("write first part");
|
||||
std::fs::write(second_dir.join("part.1"), b"second").expect("write second part");
|
||||
let _probe = file_sync_probe::set_blocking(temp_dir.path());
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
file_sync_probe::set_before_group_batch(move || {
|
||||
entered_tx.send(()).expect("signal first file fdatasync group worker");
|
||||
});
|
||||
|
||||
let limiter = file_sync_limiter();
|
||||
let first_limiter = limiter.clone();
|
||||
let first_path = first_dir.clone();
|
||||
let first = tokio::spawn(async move { sync_dir_files_with_limiter(first_path, first_limiter).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("group worker hook waiter should run")
|
||||
.expect("first file fdatasync group worker should start");
|
||||
|
||||
let second_limiter = limiter.clone();
|
||||
let second_path = second_dir.clone();
|
||||
let second = tokio::spawn(async move { sync_dir_files_with_limiter(second_path, second_limiter).await });
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
if file_fdatasync_group_commit_counts_for_test().1 == 2 {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second waiter should enqueue during the configured wait budget");
|
||||
tokio::time::advance(wait_budget).await;
|
||||
tokio::task::yield_now().await;
|
||||
file_sync_probe::wait_for_active(1).await;
|
||||
|
||||
assert_eq!(
|
||||
file_sync_probe::group_batches(),
|
||||
vec![2],
|
||||
"configured wait budget should let a follower join the leader's batch"
|
||||
);
|
||||
file_sync_probe::release();
|
||||
first
|
||||
.await
|
||||
.expect("join first wait-budget file sync")
|
||||
.expect("first wait-budget file sync must succeed");
|
||||
second
|
||||
.await
|
||||
.expect("join second wait-budget file sync")
|
||||
.expect("second wait-budget file sync must succeed");
|
||||
assert!(
|
||||
fsync_dir_recorder::was_fsynced(&first_dir),
|
||||
"first source directory must still be fsynced"
|
||||
);
|
||||
assert!(
|
||||
fsync_dir_recorder::was_fsynced(&second_dir),
|
||||
"second source directory must still be fsynced"
|
||||
);
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn file_fdatasync_group_commit_failure_fails_all_waiters_before_dir_fsync() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
set_file_fdatasync_group_commit_wait_for_test(0);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first_dir = temp_dir.path().join("first");
|
||||
let second_dir = temp_dir.path().join("second");
|
||||
std::fs::create_dir(&first_dir).expect("create first dir");
|
||||
std::fs::create_dir(&second_dir).expect("create second dir");
|
||||
std::fs::write(first_dir.join("part.1"), b"first").expect("write first part");
|
||||
std::fs::write(second_dir.join("part.1"), b"second").expect("write second part");
|
||||
let _probe = file_sync_probe::set_failing_blocking(temp_dir.path());
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_batch_tx, release_batch_rx) = mpsc::channel();
|
||||
file_sync_probe::set_before_group_batch(move || {
|
||||
entered_tx.send(()).expect("signal first file fdatasync group worker");
|
||||
release_batch_rx.recv().expect("wait until second waiter is queued");
|
||||
});
|
||||
|
||||
let limiter = file_sync_limiter();
|
||||
let first_limiter = limiter.clone();
|
||||
let first_path = first_dir.clone();
|
||||
let first = tokio::spawn(async move { sync_dir_files_with_limiter(first_path, first_limiter).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("group worker hook waiter should run")
|
||||
.expect("first file fdatasync group worker should start");
|
||||
|
||||
let second_limiter = limiter.clone();
|
||||
let second_path = second_dir.clone();
|
||||
let second = tokio::spawn(async move { sync_dir_files_with_limiter(second_path, second_limiter).await });
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
if file_fdatasync_group_commit_counts_for_test().1 == 2 {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second waiter should enqueue before releasing the grouped batch");
|
||||
release_batch_tx.send(()).expect("release group batch hook");
|
||||
|
||||
let first_err = first
|
||||
.await
|
||||
.expect("join first grouped file sync")
|
||||
.expect_err("first grouped waiter must fail closed");
|
||||
let second_err = second
|
||||
.await
|
||||
.expect("join second grouped file sync")
|
||||
.expect_err("second grouped waiter must fail closed");
|
||||
|
||||
assert_eq!(first_err.kind(), io::ErrorKind::Other);
|
||||
assert_eq!(second_err.kind(), io::ErrorKind::Other);
|
||||
assert_eq!(file_sync_probe::group_batches(), vec![2]);
|
||||
assert!(
|
||||
!fsync_dir_recorder::was_fsynced(&first_dir) && !fsync_dir_recorder::was_fsynced(&second_dir),
|
||||
"source directories must not be fsynced after grouped file fdatasync failure"
|
||||
);
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
file_sync_probe::release();
|
||||
file_sync_probe::wait_for_idle().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
fn file_fdatasync_group_commit_rejects_active_group_overflow() {
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let mut receivers = Vec::new();
|
||||
let mut limiters = Vec::new();
|
||||
for index in 0..MAX_FILE_FDATASYNC_GROUPS {
|
||||
let limiter = Arc::new(Semaphore::new(1));
|
||||
let (result_rx, _worker) = FILE_FDATASYNC_GROUP_COMMIT
|
||||
.enqueue(limiter.clone(), vec![PathBuf::from(format!("part-{index}"))])
|
||||
.expect("group below cap should enqueue");
|
||||
limiters.push(limiter);
|
||||
receivers.push(result_rx);
|
||||
}
|
||||
|
||||
let overflow_limiter = Arc::new(Semaphore::new(1));
|
||||
let err = match FILE_FDATASYNC_GROUP_COMMIT.enqueue(overflow_limiter, vec![PathBuf::from("overflow")]) {
|
||||
Ok(_) => panic!("active group max+1 must fail closed"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::WouldBlock);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
drop(receivers);
|
||||
drop(limiters);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
fn file_fdatasync_group_commit_rejects_waiter_and_file_overflow() {
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let limiter = Arc::new(Semaphore::new(1));
|
||||
let mut receivers = Vec::new();
|
||||
for index in 0..MAX_FILE_FDATASYNC_WAITERS {
|
||||
let (result_rx, _worker) = FILE_FDATASYNC_GROUP_COMMIT
|
||||
.enqueue(limiter.clone(), vec![PathBuf::from(format!("part-{index}"))])
|
||||
.expect("waiter below cap should enqueue");
|
||||
receivers.push(result_rx);
|
||||
}
|
||||
|
||||
let waiter_err = match FILE_FDATASYNC_GROUP_COMMIT.enqueue(limiter, vec![PathBuf::from("overflow-waiter")]) {
|
||||
Ok(_) => panic!("waiter max+1 must fail closed"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(waiter_err.kind(), io::ErrorKind::WouldBlock);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
drop(receivers);
|
||||
|
||||
let mut receivers = Vec::new();
|
||||
let file_limit_limiter = Arc::new(Semaphore::new(1));
|
||||
let (result_rx, _worker) = FILE_FDATASYNC_GROUP_COMMIT
|
||||
.enqueue(
|
||||
file_limit_limiter.clone(),
|
||||
(0..MAX_FILE_FDATASYNC_BATCH_FILES)
|
||||
.map(|index| PathBuf::from(format!("part-{index}")))
|
||||
.collect(),
|
||||
)
|
||||
.expect("file count up to cap should enqueue");
|
||||
receivers.push(result_rx);
|
||||
|
||||
let file_err = match FILE_FDATASYNC_GROUP_COMMIT.enqueue(file_limit_limiter, vec![PathBuf::from("overflow-file")]) {
|
||||
Ok(_) => panic!("file max+1 must fail closed"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(file_err.kind(), io::ErrorKind::WouldBlock);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
drop(receivers);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn sync_dir_files_bounds_concurrency_across_directories() {
|
||||
|
||||
@@ -218,63 +218,6 @@ pub struct QuotaAdmission {
|
||||
quota_limit: u64,
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct LifecycleDeleteAllRequest {
|
||||
pub(crate) version_id: Option<Uuid>,
|
||||
pub(crate) delete_marker: bool,
|
||||
pub(crate) action: rustfs_common::metrics::IlmAction,
|
||||
pub(crate) rule_id: String,
|
||||
pub(crate) phase: LifecycleDeleteAllPhase,
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LifecycleDeleteAllPhase {
|
||||
Preflight,
|
||||
History,
|
||||
FinalPreflight,
|
||||
Trigger,
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
#[derive(Default)]
|
||||
pub struct LifecycleDeleteAllJournalState {
|
||||
prepared: HashMap<String, crate::bucket::lifecycle::tier_sweeper::Jentry>,
|
||||
mutation_started: bool,
|
||||
}
|
||||
|
||||
impl Debug for LifecycleDeleteAllJournalState {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("LifecycleDeleteAllJournalState")
|
||||
.field("prepared_count", &self.prepared.len())
|
||||
.field("mutation_started", &self.mutation_started)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl LifecycleDeleteAllJournalState {
|
||||
pub(crate) fn contains(&self, name: &str) -> bool {
|
||||
self.prepared.contains_key(name)
|
||||
}
|
||||
|
||||
pub(crate) fn insert(&mut self, name: String, entry: crate::bucket::lifecycle::tier_sweeper::Jentry) {
|
||||
self.prepared.insert(name, entry);
|
||||
}
|
||||
|
||||
pub(crate) fn prepared_entries(&self) -> Vec<crate::bucket::lifecycle::tier_sweeper::Jentry> {
|
||||
self.prepared.values().cloned().collect()
|
||||
}
|
||||
|
||||
pub(crate) fn mark_mutation_started(&mut self) {
|
||||
self.mutation_started = true;
|
||||
}
|
||||
|
||||
pub(crate) fn mutation_started(&self) -> bool {
|
||||
self.mutation_started
|
||||
}
|
||||
}
|
||||
|
||||
impl QuotaAdmission {
|
||||
pub(crate) fn current_usage(self) -> u64 {
|
||||
self.current_usage
|
||||
@@ -299,11 +242,6 @@ pub struct ObjectOptions {
|
||||
pub delete_prefix: bool,
|
||||
pub delete_prefix_object: bool,
|
||||
pub version_id: Option<String>,
|
||||
/// Lifecycle-only staged purge request checked under the object write lock.
|
||||
#[doc(hidden)]
|
||||
pub lifecycle_delete_all: Option<LifecycleDeleteAllRequest>,
|
||||
#[doc(hidden)]
|
||||
pub lifecycle_delete_all_journal: Option<Arc<parking_lot::Mutex<LifecycleDeleteAllJournalState>>>,
|
||||
/// RustFS-only compare-and-set condition checked under the object write lock.
|
||||
pub expected_current_version_id: Option<String>,
|
||||
/// Persisted bucket incarnation observed before authorization.
|
||||
@@ -411,15 +349,6 @@ impl ObjectOptions {
|
||||
self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new);
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) {
|
||||
self.lifecycle_delete_all_journal
|
||||
.get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default())));
|
||||
}
|
||||
|
||||
pub(crate) fn lifecycle_delete_all_journal(&self) -> Option<&Arc<parking_lot::Mutex<LifecycleDeleteAllJournalState>>> {
|
||||
self.lifecycle_delete_all_journal.as_ref()
|
||||
}
|
||||
|
||||
pub fn add_namespace_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
|
||||
if let Some(signal) = guard.lock_lost_signal() {
|
||||
self.add_namespace_lock_lost_signal(signal);
|
||||
@@ -719,23 +648,14 @@ impl ObjectInfo {
|
||||
}
|
||||
|
||||
pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
|
||||
let name = decode_dir_object(object);
|
||||
|
||||
let mut version_id = fi.version_id;
|
||||
|
||||
if versioned && version_id.is_none() {
|
||||
version_id = Some(Uuid::nil())
|
||||
}
|
||||
|
||||
Self::from_file_info_with_version_id(fi, bucket, object, version_id)
|
||||
}
|
||||
|
||||
pub(crate) fn from_file_info_with_version_id(
|
||||
fi: &FileInfo,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<Uuid>,
|
||||
) -> ObjectInfo {
|
||||
let name = decode_dir_object(object);
|
||||
|
||||
// etag
|
||||
let (content_type, content_encoding, etag) = {
|
||||
let content_type = fi.metadata.get("content-type").cloned();
|
||||
@@ -1649,18 +1569,6 @@ mod tests {
|
||||
assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_file_info_with_version_id_keeps_normalized_absent_version() {
|
||||
let fi = FileInfo {
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let info = ObjectInfo::from_file_info_with_version_id(&fi, "bucket", "object", None);
|
||||
|
||||
assert_eq!(info.version_id, None, "a normalized absent version must not be rewritten to nil");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_file_info_reports_effective_storage_class_for_legacy_metadata() {
|
||||
for legacy_label in [
|
||||
@@ -1982,13 +1890,4 @@ mod tests {
|
||||
assert!(default_cloned.user_tags.is_empty());
|
||||
assert!(default_cloned.parts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_options_default_does_not_allocate_lifecycle_delete_all_journal() {
|
||||
let mut opts = ObjectOptions::default();
|
||||
|
||||
assert!(opts.lifecycle_delete_all_journal().is_none());
|
||||
opts.ensure_lifecycle_delete_all_journal();
|
||||
assert!(opts.lifecycle_delete_all_journal().is_some());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -573,7 +573,9 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
pub(crate) async fn init_tier_config_mgr(store: Arc<ECStore>) -> Result<()> {
|
||||
let handle = get_global_tier_config_mgr();
|
||||
TierConfigMgr::reload_handle(&handle, store.clone()).await?;
|
||||
tokio::spawn(TierConfigMgr::refresh_tier_config_handle(handle, store));
|
||||
if setup_is_dist_erasure().await {
|
||||
tokio::spawn(TierConfigMgr::refresh_tier_config_handle(handle, store));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -657,7 +657,7 @@ where
|
||||
prefix,
|
||||
marker,
|
||||
None,
|
||||
i32::try_from(limit).unwrap_or(i32::MAX),
|
||||
i32::try_from(limit).map_or(i32::MAX, |value| value),
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
|
||||
@@ -17,7 +17,7 @@ use std::sync::Arc;
|
||||
use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::tier::{TierConfigMgr, tier_config_abort_matches, tier_config_commit_matches, tier_config_etag_matches};
|
||||
use super::tier::{TierConfigMgr, tier_config_etag_matches};
|
||||
use super::tier_mutation_intent::{
|
||||
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
|
||||
load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
|
||||
@@ -53,10 +53,6 @@ pub enum TierMutationPeerError {
|
||||
InvalidPayload(String),
|
||||
#[error("tier mutation peer intent conflicts with existing record")]
|
||||
ConflictingIntent,
|
||||
#[error("tier mutation peer commit proof does not match the persisted tier configuration")]
|
||||
CommitProofMismatch,
|
||||
#[error("tier mutation peer abort proof does not match the persisted tier configuration")]
|
||||
AbortProofMismatch,
|
||||
#[error("tier mutation peer runtime error: {0}")]
|
||||
Runtime(#[source] AdminError),
|
||||
#[error("tier mutation peer store error: {0}")]
|
||||
@@ -124,13 +120,8 @@ async fn handle_prepare(
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
}
|
||||
TierMutationIntentState::Committed => {
|
||||
TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &existing)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
}
|
||||
TierMutationIntentState::Aborted => {
|
||||
TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
|
||||
TierMutationIntentState::Committed | TierMutationIntentState::Aborted => {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
|
||||
}
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
@@ -149,18 +140,6 @@ async fn handle_commit(
|
||||
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
|
||||
let committed_config_etag = parse_commit_etag(canonical_payload)?;
|
||||
let tier_config_mgr = api.tier_config_mgr();
|
||||
match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
|
||||
Ok(intent) if intent.state == TierMutationIntentState::Prepared => {
|
||||
let proof_matches = tier_config_commit_matches(api.clone(), &committed_config_etag, intent.candidate_digest)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
if !proof_matches {
|
||||
return Err(TierMutationPeerError::CommitProofMismatch);
|
||||
}
|
||||
}
|
||||
Ok(_) | Err(Error::ConfigNotFound) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
let (intent, applied) = match advance_tier_mutation_intent_record_idempotent(
|
||||
api.clone(),
|
||||
mutation_id,
|
||||
@@ -175,9 +154,6 @@ async fn handle_commit(
|
||||
.await
|
||||
.map_err(Error::other)? =>
|
||||
{
|
||||
TierConfigMgr::promote_prepared_mutation_intent_block(&tier_config_mgr, mutation_id)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
return Ok(TierMutationPeerOutcome {
|
||||
state: TierMutationPeerState::Committed,
|
||||
applied: false,
|
||||
@@ -186,9 +162,7 @@ async fn handle_commit(
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
if intent.state == TierMutationIntentState::Committed {
|
||||
TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &intent)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(intent.state),
|
||||
@@ -204,18 +178,11 @@ async fn handle_abort(
|
||||
if !canonical_payload.is_empty() {
|
||||
return Err(TierMutationPeerError::InvalidPayload("abort payload must be empty".to_string()));
|
||||
}
|
||||
let existing = load_tier_mutation_intent_record(api.clone(), mutation_id).await?;
|
||||
if existing.state == TierMutationIntentState::Prepared
|
||||
&& !tier_config_abort_matches(api.clone(), &existing)
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
return Err(TierMutationPeerError::AbortProofMismatch);
|
||||
}
|
||||
let tier_config_mgr = api.tier_config_mgr();
|
||||
let (intent, applied) =
|
||||
advance_tier_mutation_intent_record_idempotent(api.clone(), mutation_id, TierMutationIntentState::Aborted, None).await?;
|
||||
advance_tier_mutation_intent_record_idempotent(api, mutation_id, TierMutationIntentState::Aborted, None).await?;
|
||||
if intent.state == TierMutationIntentState::Aborted {
|
||||
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(intent.state),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -272,7 +272,6 @@ const MULTIPART_WRITE_QUORUM_RENAME_PART: &str = "rename_part";
|
||||
const EVENT_SET_DISK_WRITE: &str = "set_disk_write";
|
||||
const EVENT_SET_DISK_HEAL: &str = "set_disk_heal";
|
||||
const EVENT_SET_DISK_COMMIT_TAIL_SLOW: &str = "set_disk_commit_tail_slow";
|
||||
const EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED: &str = "set_disk_rename_tail_drain_failed";
|
||||
const EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY: &str = "set_disk_put_object_stage_summary";
|
||||
const SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS: u128 = 5_000;
|
||||
const ENV_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES";
|
||||
@@ -922,10 +921,14 @@ mod prepared_get_object_metadata_tests {
|
||||
.expect("test should find an object whose initial fanout covers both data shards")
|
||||
}
|
||||
|
||||
fn bounded_initial_parity_disk_index(bucket: &str, object: &str) -> usize {
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "test fixture no assertion in this module uses today; the live namesake lives in io_primitives tests (backlog#1823)"
|
||||
)]
|
||||
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
|
||||
*bounded_metadata_fanout_order(bucket, object, 4, 2)
|
||||
.get(2)
|
||||
.expect("4-disk test geometry should schedule one parity disk initially")
|
||||
.get(3)
|
||||
.expect("4-disk test geometry should leave one bounded spare disk")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1083,7 +1086,7 @@ mod prepared_get_object_metadata_tests {
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
||||
],
|
||||
async {
|
||||
let slow_parity_disk = bounded_initial_parity_disk_index(bucket, &object);
|
||||
let slow_parity_disk = bounded_spare_disk_index(bucket, &object);
|
||||
let barrier =
|
||||
rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
@@ -1497,102 +1500,6 @@ pub fn get_lock_acquire_timeout() -> Duration {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_put_object_commit_lock_acquire_timeout_override_ms() -> u64 {
|
||||
#[cfg(test)]
|
||||
{
|
||||
rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
|
||||
rustfs_config::DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
|
||||
)
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
static CACHED: OnceLock<u64> = OnceLock::new();
|
||||
*CACHED.get_or_init(|| {
|
||||
rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
|
||||
rustfs_config::DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn get_put_object_commit_lock_acquire_timeout(op: &'static str) -> Duration {
|
||||
let default_timeout = get_lock_acquire_timeout();
|
||||
if op != "put_object_commit" {
|
||||
return default_timeout;
|
||||
}
|
||||
|
||||
let timeout_ms = get_put_object_commit_lock_acquire_timeout_override_ms();
|
||||
if timeout_ms == 0 {
|
||||
default_timeout
|
||||
} else {
|
||||
Duration::from_millis(timeout_ms)
|
||||
}
|
||||
}
|
||||
|
||||
fn put_object_commit_lock_timeout_override_enabled(op: &'static str) -> bool {
|
||||
op == "put_object_commit" && get_put_object_commit_lock_acquire_timeout_override_ms() != 0
|
||||
}
|
||||
|
||||
fn put_object_commit_lock_admission_budget_label() -> &'static str {
|
||||
match get_put_object_commit_lock_acquire_timeout_override_ms() {
|
||||
0 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
|
||||
1..=250 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
251..=500 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
|
||||
501..=1000 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS,
|
||||
_ => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_put_object_commit_lock_admission(op: &'static str, outcome: &'static str) {
|
||||
if op != "put_object_commit" || !rustfs_io_metrics::put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_commit_lock_admission(put_object_commit_lock_admission_budget_label(), outcome);
|
||||
}
|
||||
|
||||
fn put_object_commit_lock_acquire_error_outcome(op: &'static str, err: &rustfs_lock::error::LockError) -> &'static str {
|
||||
if put_object_commit_lock_timeout_override_enabled(op) && matches!(err, rustfs_lock::error::LockError::Timeout { .. }) {
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN
|
||||
} else {
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_put_object_commit_lock_acquire_result(
|
||||
set: &SetDisks,
|
||||
op: &'static str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
result: std::result::Result<rustfs_lock::namespace::NamespaceLockGuard, rustfs_lock::error::LockError>,
|
||||
) -> Result<rustfs_lock::namespace::NamespaceLockGuard> {
|
||||
match result {
|
||||
Ok(guard) => {
|
||||
record_put_object_commit_lock_admission(op, rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED);
|
||||
Ok(guard)
|
||||
}
|
||||
Err(err) => {
|
||||
record_put_object_commit_lock_admission(op, put_object_commit_lock_acquire_error_outcome(op, &err));
|
||||
Err(map_put_object_commit_lock_acquire_error(set, op, bucket, object, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_put_object_commit_lock_acquire_error(
|
||||
set: &SetDisks,
|
||||
op: &'static str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
err: rustfs_lock::error::LockError,
|
||||
) -> StorageError {
|
||||
if put_object_commit_lock_timeout_override_enabled(op) && matches!(err, rustfs_lock::error::LockError::Timeout { .. }) {
|
||||
StorageError::SlowDown
|
||||
} else {
|
||||
set.map_namespace_lock_error(bucket, object, "write", err)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_object_lock_diag_enabled() -> bool {
|
||||
*OBJECT_LOCK_DIAG_ENABLED.get_or_init(|| {
|
||||
let enabled = rustfs_utils::get_env_bool(
|
||||
@@ -3356,17 +3263,6 @@ impl SetDisks {
|
||||
self.get_object_metadata_cache.invalidate_all();
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn record_put_object_commit_namespace_lock_wait(op: &'static str, acquire_start: Instant) {
|
||||
if op != "put_object_commit" || !rustfs_io_metrics::put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT,
|
||||
Some(acquire_start),
|
||||
);
|
||||
}
|
||||
|
||||
async fn acquire_read_lock_diag(&self, op: &'static str, bucket: &str, object: &str) -> Result<ObjectLockDiagGuard> {
|
||||
crate::hp_guard!("SetDisks::acquire_read_lock");
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
@@ -3394,15 +3290,10 @@ impl SetDisks {
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||
let acquire_start = Instant::now();
|
||||
let acquire_timeout = get_put_object_commit_lock_acquire_timeout(op);
|
||||
let guard = resolve_put_object_commit_lock_acquire_result(
|
||||
self,
|
||||
op,
|
||||
bucket,
|
||||
object,
|
||||
ns_lock.get_write_lock(acquire_timeout).await,
|
||||
)?;
|
||||
Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
|
||||
let guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?;
|
||||
let owner = diag_enabled.then(|| ns_lock.owner().to_string());
|
||||
self.log_object_lock_acquire_if_slow(
|
||||
op,
|
||||
@@ -3436,27 +3327,20 @@ impl SetDisks {
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||
let acquire_start = Instant::now();
|
||||
let acquire_timeout = get_put_object_commit_lock_acquire_timeout(op);
|
||||
let acquire = ns_lock.get_write_lock(acquire_timeout);
|
||||
let acquire = ns_lock.get_write_lock(get_lock_acquire_timeout());
|
||||
tokio::pin!(acquire);
|
||||
let mut on_pending = Some(on_pending);
|
||||
let guard = resolve_put_object_commit_lock_acquire_result(
|
||||
self,
|
||||
op,
|
||||
bucket,
|
||||
object,
|
||||
futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
|
||||
std::task::Poll::Pending => {
|
||||
if let Some(on_pending) = on_pending.take() {
|
||||
on_pending();
|
||||
}
|
||||
std::task::Poll::Pending
|
||||
let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
|
||||
std::task::Poll::Pending => {
|
||||
if let Some(on_pending) = on_pending.take() {
|
||||
on_pending();
|
||||
}
|
||||
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
|
||||
})
|
||||
.await,
|
||||
)?;
|
||||
Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
|
||||
std::task::Poll::Pending
|
||||
}
|
||||
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
|
||||
})
|
||||
.await
|
||||
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?;
|
||||
let owner = diag_enabled.then(|| ns_lock.owner().to_string());
|
||||
self.log_object_lock_acquire_if_slow(
|
||||
op,
|
||||
@@ -4694,15 +4578,10 @@ impl SetDisks {
|
||||
)?;
|
||||
let fi = build_tiered_decommission_file_info(bucket, object, fi, layout);
|
||||
let write_quorum = layout.write_quorum;
|
||||
if _lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
if opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
@@ -5575,7 +5454,6 @@ mod tests {
|
||||
};
|
||||
use crate::store::init_format::save_format_file;
|
||||
use crate::store::list_objects::ListPathOptions;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_filemeta::ErasureInfo;
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_filemeta::MetaCacheEntry;
|
||||
@@ -5811,448 +5689,6 @@ mod tests {
|
||||
assert_eq!(Arc::strong_count(&set.set_lock_namespace), before);
|
||||
}
|
||||
|
||||
fn put_object_commit_namespace_lock_wait_sample_count(snapshotter: &metrics_util::debugging::Snapshotter) -> usize {
|
||||
snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.key().name() == "rustfs_s3_put_object_stage_duration_ms"
|
||||
&& composite.key().labels().any(|label| {
|
||||
label.key() == "stage"
|
||||
&& label.value() == rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT
|
||||
})
|
||||
})
|
||||
.map(|(_, _, _, value)| match value {
|
||||
DebugValue::Histogram(samples) => samples.len(),
|
||||
_ => 0,
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn put_object_commit_lock_admission_count(
|
||||
rows: &[(
|
||||
metrics_util::CompositeKey,
|
||||
Option<metrics::Unit>,
|
||||
Option<metrics::SharedString>,
|
||||
DebugValue,
|
||||
)],
|
||||
budget: &'static str,
|
||||
outcome: &'static str,
|
||||
) -> u64 {
|
||||
rows.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
|
||||
&& composite
|
||||
.key()
|
||||
.labels()
|
||||
.any(|label| label.key() == "budget" && label.value() == budget)
|
||||
&& composite
|
||||
.key()
|
||||
.labels()
|
||||
.any(|label| label.key() == "outcome" && label.value() == outcome)
|
||||
})
|
||||
.map(|(_, _, _, value)| match value {
|
||||
DebugValue::Counter(count) => *count,
|
||||
_ => 0,
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_budget_labels_are_bounded() {
|
||||
let cases = [
|
||||
("0", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED),
|
||||
("250", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS),
|
||||
("251", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS),
|
||||
("500", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS),
|
||||
("501", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS),
|
||||
("1000", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS),
|
||||
("1001", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS),
|
||||
];
|
||||
for (timeout_ms, expected) in cases {
|
||||
temp_env::with_vars(
|
||||
[(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some(timeout_ms))],
|
||||
|| {
|
||||
assert_eq!(put_object_commit_lock_admission_budget_label(), expected);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_error_outcomes_are_bounded() {
|
||||
let timeout = LockError::timeout("bucket/object", Duration::from_millis(1));
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
|
||||
assert_eq!(
|
||||
put_object_commit_lock_acquire_error_outcome("put_object_commit", &timeout),
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN
|
||||
);
|
||||
assert_eq!(
|
||||
put_object_commit_lock_acquire_error_outcome("complete_multipart_upload_commit", &timeout),
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
|
||||
);
|
||||
});
|
||||
|
||||
let internal = LockError::internal("simulated lock manager error");
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
|
||||
assert_eq!(
|
||||
put_object_commit_lock_acquire_error_outcome("put_object_commit", &internal),
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_namespace_lock_wait_metric_is_wired_to_both_write_lock_paths() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let bucket = "bucket";
|
||||
let object = "object";
|
||||
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
let guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", bucket, object)
|
||||
.await
|
||||
.expect("disabled metrics acquire should succeed");
|
||||
drop(guard);
|
||||
assert_eq!(put_object_commit_namespace_lock_wait_sample_count(&snapshotter), 0);
|
||||
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
let guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", bucket, object)
|
||||
.await
|
||||
.expect("normal PUT commit acquire should succeed");
|
||||
drop(guard);
|
||||
assert_eq!(put_object_commit_namespace_lock_wait_sample_count(&snapshotter), 1);
|
||||
|
||||
let guard = set
|
||||
.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
|
||||
.await
|
||||
.expect("non-PUT commit acquire should succeed");
|
||||
drop(guard);
|
||||
assert_eq!(put_object_commit_namespace_lock_wait_sample_count(&snapshotter), 0);
|
||||
|
||||
let held_guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", bucket, object)
|
||||
.await
|
||||
.expect("holder acquire should succeed");
|
||||
assert_eq!(put_object_commit_namespace_lock_wait_sample_count(&snapshotter), 1);
|
||||
|
||||
let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
|
||||
let pending_acquire =
|
||||
set.acquire_write_lock_diag_with_pending_hook("put_object_commit", bucket, object, move || {
|
||||
let _ = pending_tx.send(());
|
||||
});
|
||||
let release_holder = async {
|
||||
pending_rx.await.expect("pending hook should fire");
|
||||
drop(held_guard);
|
||||
};
|
||||
let (pending_guard, ()) = tokio::join!(pending_acquire, release_holder);
|
||||
drop(pending_guard.expect("pending-hook PUT commit acquire should succeed"));
|
||||
assert_eq!(put_object_commit_namespace_lock_wait_sample_count(&snapshotter), 1);
|
||||
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_timeout_override_only_applies_to_put_commit() {
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("17"))], || {
|
||||
assert_eq!(get_put_object_commit_lock_acquire_timeout("put_object_commit"), Duration::from_millis(17));
|
||||
assert_eq!(
|
||||
get_put_object_commit_lock_acquire_timeout("complete_multipart_upload_commit"),
|
||||
get_lock_acquire_timeout()
|
||||
);
|
||||
});
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("0"))], || {
|
||||
assert_eq!(
|
||||
get_put_object_commit_lock_acquire_timeout("put_object_commit"),
|
||||
get_lock_acquire_timeout()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_timeout_override_bounds_contention_wait() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let bucket = "bucket";
|
||||
let object = "object";
|
||||
|
||||
let held_guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", bucket, object)
|
||||
.await
|
||||
.expect("holder acquire should succeed");
|
||||
let started = Instant::now();
|
||||
let err = match set.acquire_write_lock_diag("put_object_commit", bucket, object).await {
|
||||
Ok(_) => panic!("contended PUT commit lock should honor the short timeout"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(1),
|
||||
"short PUT commit lock timeout should not wait for the global timeout"
|
||||
);
|
||||
assert!(matches!(err, StorageError::SlowDown));
|
||||
|
||||
drop(held_guard);
|
||||
set.acquire_write_lock_diag("put_object_commit", bucket, object)
|
||||
.await
|
||||
.expect("permit should not leak after timeout");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_records_acquired_and_timeout() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let held_guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", "bucket", "object")
|
||||
.await
|
||||
.expect("holder acquire should succeed");
|
||||
let err = match set.acquire_write_lock_diag("put_object_commit", "bucket", "object").await {
|
||||
Ok(_) => panic!("contended PUT commit acquire should return SlowDown"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(matches!(err, StorageError::SlowDown));
|
||||
drop(held_guard);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
|
||||
),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
|
||||
),
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_records_disabled_budget_acquired() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("0"))], || {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", "bucket", "object")
|
||||
.await
|
||||
.expect("PUT commit acquire should succeed with default timeout");
|
||||
drop(guard);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
|
||||
),
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_skips_non_put_commit_ops() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("250"))], || {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let guard = set
|
||||
.acquire_write_lock_diag("complete_multipart_upload_commit", "bucket", "object")
|
||||
.await
|
||||
.expect("non-PUT commit acquire should succeed");
|
||||
drop(guard);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
rows.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
|
||||
})
|
||||
.count(),
|
||||
0
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_records_lock_error() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("250"))], || {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let healthy: Arc<dyn LockClient> =
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let failing: Arc<dyn LockClient> = Arc::new(FailingClient);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
let set = make_test_set_disks_with_ctx(vec![healthy, failing], ctx).await;
|
||||
assert!(
|
||||
set.acquire_write_lock_diag("put_object_commit", "bucket", "object")
|
||||
.await
|
||||
.is_err(),
|
||||
"one healthy locker must not satisfy the PUT commit write quorum"
|
||||
);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR,
|
||||
),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
|
||||
),
|
||||
0
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_records_pending_hook_acquired() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("500"))], || {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let held_guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", "bucket", "object")
|
||||
.await
|
||||
.expect("holder acquire should succeed");
|
||||
let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
|
||||
let pending_acquire =
|
||||
set.acquire_write_lock_diag_with_pending_hook("put_object_commit", "bucket", "object", move || {
|
||||
let _ = pending_tx.send(());
|
||||
});
|
||||
let release_holder = async {
|
||||
pending_rx.await.expect("pending hook should fire");
|
||||
drop(held_guard);
|
||||
};
|
||||
let (pending_guard, ()) = tokio::join!(pending_acquire, release_holder);
|
||||
drop(pending_guard.expect("pending-hook PUT commit acquire should succeed"));
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
|
||||
),
|
||||
2
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_ns_lock_shares_clients_without_changing_quorum() {
|
||||
let healthy: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
|
||||
@@ -124,7 +124,14 @@ impl HealWalkCollector {
|
||||
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
||||
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
|
||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
||||
Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
|
||||
let mut lifecycle_fi = fi.clone();
|
||||
lifecycle_fi.version_id = version_uuid;
|
||||
Some(ObjectInfo::from_file_info(
|
||||
&lifecycle_fi,
|
||||
&self.bucket,
|
||||
&entry.name,
|
||||
version_uuid.is_some(),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -191,7 +198,14 @@ impl HealWalkCollector {
|
||||
let vid = version_uuid.map(|u| u.to_string());
|
||||
if seen.insert(vid.clone()) {
|
||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
||||
Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
|
||||
let mut lifecycle_fi = fi.clone();
|
||||
lifecycle_fi.version_id = version_uuid;
|
||||
Some(ObjectInfo::from_file_info(
|
||||
&lifecycle_fi,
|
||||
&self.bucket,
|
||||
&entry.name,
|
||||
version_uuid.is_some(),
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
@@ -2433,8 +2433,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let commit_object_lock_guard = object_lock_guard.take();
|
||||
let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some() || quota_mutation_fence;
|
||||
let commit = async move {
|
||||
let mut _object_lock_guard = commit_object_lock_guard;
|
||||
let mut _upload_guard = upload_guard;
|
||||
let _object_lock_guard = commit_object_lock_guard;
|
||||
let _upload_guard = upload_guard;
|
||||
let mut quota_reservation = quota_reservation;
|
||||
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
@@ -2570,7 +2570,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let op_old_dir = rename_commit.data_dir;
|
||||
let cleanup_disks = rename_commit.cleanup_disks;
|
||||
let committed_file_info = rename_commit.committed_file_info;
|
||||
let rename_tail_drain = rename_commit.tail_drain;
|
||||
|
||||
// Detach admission before any post-commit await: client cancellation
|
||||
// must not couple durable convergence repair to cleanup work.
|
||||
@@ -2629,30 +2628,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.invalidate_get_object_metadata_cache(&commit_bucket, &commit_object)
|
||||
.await;
|
||||
|
||||
if let Some(rename_tail_drain) = rename_tail_drain {
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let upload_guard = _upload_guard.take();
|
||||
let tail_bucket = commit_bucket.clone();
|
||||
let tail_object = commit_object.clone();
|
||||
tokio::spawn(async move {
|
||||
let _object_lock_guard = object_lock_guard;
|
||||
let _upload_guard = upload_guard;
|
||||
if let Err(err) = rename_tail_drain.await {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
state = "failed",
|
||||
bucket = %tail_bucket,
|
||||
object = %tail_object,
|
||||
error = %err,
|
||||
"rename tail drain failed"
|
||||
);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
drop(_object_lock_guard.take()); // release the object lock before multipart cleanup tail IO.
|
||||
}
|
||||
drop(_object_lock_guard); // release the object lock before multipart cleanup tail IO.
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterObjectPublication).await;
|
||||
@@ -2709,7 +2685,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
drop(_upload_guard.take());
|
||||
drop(_upload_guard);
|
||||
|
||||
Ok(ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned))
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1077,23 +1077,23 @@ impl SetDisks {
|
||||
"Recoverable decode error triggered read repair"
|
||||
);
|
||||
let version_id = fi.version_id.as_ref().map(ToString::to_string);
|
||||
// Single-flight (backlog#1894 axis A): the durable
|
||||
// MRF intent (Urgent ECDecode across restarts, HS-01)
|
||||
// is bound to the read-repair reservation, so only the
|
||||
// first sighting within the dedup TTL books a journal
|
||||
// record instead of one per retried read.
|
||||
submit_read_repair_heal_with_submitter(
|
||||
ReadRepairHealSubmission {
|
||||
bucket,
|
||||
object,
|
||||
version_id: version_id.as_deref(),
|
||||
pool_index,
|
||||
set_index,
|
||||
part_number: Some(part_number),
|
||||
reason: "decode_error",
|
||||
mrf_intent: Some((rustfs_common::mrf_channel::MrfKind::DecodeFailure, fi.version_id)),
|
||||
},
|
||||
send_read_repair_heal_request,
|
||||
// MRF journal intent: keeps a durable Urgent ECDecode
|
||||
// request alive across restarts even when the in-memory
|
||||
// read-repair request is dropped or lost (HS-01).
|
||||
rustfs_common::mrf_channel::try_send_mrf_intent(
|
||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure,
|
||||
bucket,
|
||||
object,
|
||||
fi.version_id,
|
||||
);
|
||||
submit_read_repair_heal(
|
||||
bucket,
|
||||
object,
|
||||
version_id.as_deref(),
|
||||
pool_index,
|
||||
set_index,
|
||||
Some(part_number),
|
||||
"decode_error",
|
||||
)
|
||||
.await;
|
||||
has_err = false;
|
||||
@@ -2577,7 +2577,6 @@ mod metadata_cache_tests {
|
||||
set_index: 0,
|
||||
part_number: Some(1),
|
||||
reason: "missing_shards",
|
||||
mrf_intent: None,
|
||||
},
|
||||
slow_read_repair_submitter,
|
||||
)
|
||||
@@ -2612,7 +2611,6 @@ mod metadata_cache_tests {
|
||||
set_index: 0,
|
||||
part_number: Some(1),
|
||||
reason: "missing_shards",
|
||||
mrf_intent: None,
|
||||
},
|
||||
dropped_read_repair_submitter,
|
||||
)
|
||||
@@ -2649,7 +2647,6 @@ mod metadata_cache_tests {
|
||||
set_index: 0,
|
||||
part_number: Some(1),
|
||||
reason: "missing_shards",
|
||||
mrf_intent: None,
|
||||
},
|
||||
capture_read_repair_submitter,
|
||||
)
|
||||
|
||||
@@ -18,78 +18,6 @@ use rustfs_filemeta::RestoreStatusOps;
|
||||
use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
|
||||
use s3s::dto::{RestoreStatus, Timestamp};
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
struct RestoreFinalizeBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
static RESTORE_FINALIZE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<RestoreFinalizeBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(in crate::set_disk) struct RestoreFinalizeBarrier {
|
||||
state: Arc<RestoreFinalizeBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl RestoreFinalizeBarrier {
|
||||
pub(in crate::set_disk) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(RestoreFinalizeBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = RESTORE_FINALIZE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("restore finalize barrier mutex should not poison");
|
||||
assert!(slot.is_none(), "restore finalize barrier must be installed by one test at a time");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn wait_until_paused(&self) {
|
||||
self.state.arrived.notified().await;
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl Drop for RestoreFinalizeBarrier {
|
||||
fn drop(&mut self) {
|
||||
let mut slot = RESTORE_FINALIZE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("restore finalize barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
async fn maybe_pause_restore_finalize(bucket: &str, object: &str) {
|
||||
let barrier = RESTORE_FINALIZE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("restore finalize barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct RestoreCleanupIdentity {
|
||||
version_id: Option<Uuid>,
|
||||
@@ -152,7 +80,7 @@ impl SetDisks {
|
||||
.clone()
|
||||
.unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref()));
|
||||
let version_id = expected.version_id.map(|v| v.to_string());
|
||||
let lock_guard = if !opts.no_lock {
|
||||
let _lock_guard = if !opts.no_lock {
|
||||
Some(
|
||||
self.acquire_write_lock_diag("restore_finalize_metadata", bucket, object)
|
||||
.await?,
|
||||
@@ -171,16 +99,13 @@ impl SetDisks {
|
||||
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
|
||||
.await?
|
||||
.into_owned();
|
||||
if let Some(expected_operation_id) = expected_operation_id
|
||||
&& restore_operation_id_from_metadata(&fi.metadata)?.is_some_and(|actual| actual != expected_operation_id)
|
||||
{
|
||||
return Err(Error::other("restore operation id changed before metadata finalization"));
|
||||
if let Some(expected_operation_id) = expected_operation_id {
|
||||
require_restore_operation_id(&fi.metadata, expected_operation_id)?;
|
||||
}
|
||||
if !expected.matches_file_info(&fi, &expected_etag) {
|
||||
return Err(Error::other("restored object changed before restore metadata finalization"));
|
||||
}
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
maybe_pause_restore_finalize(bucket, object).await;
|
||||
ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?;
|
||||
let restore_expiry =
|
||||
lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1));
|
||||
fi.metadata.insert(
|
||||
@@ -192,10 +117,6 @@ impl SetDisks {
|
||||
.to_string(),
|
||||
);
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?;
|
||||
if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(Error::other("restore finalization lock lost before metadata update"));
|
||||
}
|
||||
self.update_object_meta_with_opts(
|
||||
bucket,
|
||||
object,
|
||||
|
||||
+38
-1116
File diff suppressed because it is too large
Load Diff
@@ -151,7 +151,6 @@ pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
mod object;
|
||||
pub(crate) use object::ObjectLockDiagGuard;
|
||||
pub use object::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
SnapshotConsistencyError,
|
||||
@@ -343,23 +342,6 @@ impl ECStore {
|
||||
let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
|
||||
decommission || rebalance
|
||||
}
|
||||
|
||||
/// Returns whether scanner metadata may still be hidden by a local
|
||||
/// data-movement state. Terminal failed/canceled decommission entries
|
||||
/// remain suspended until an operator clears or retries them, so they are
|
||||
/// a publication barrier even after the worker has stopped.
|
||||
pub async fn scanner_data_usage_publication_blocked(&self) -> bool {
|
||||
if self.scanner_data_movement_active().await {
|
||||
return true;
|
||||
}
|
||||
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
pool_meta.pools.iter().any(|pool| {
|
||||
pool.decommission
|
||||
.as_ref()
|
||||
.is_some_and(|info| !info.queued && (info.failed || info.canceled))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// impl Clone for ECStore {
|
||||
@@ -892,7 +874,6 @@ impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
|
||||
use crate::runtime::global::reset_local_disk_test_state;
|
||||
use crate::runtime::sources::{clear_local_disk_id_map_for_test, local_disk_path_by_id};
|
||||
@@ -929,72 +910,6 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
let cases = [
|
||||
(
|
||||
"active",
|
||||
PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
"failed",
|
||||
PoolDecommissionInfo {
|
||||
failed: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
"canceled",
|
||||
PoolDecommissionInfo {
|
||||
canceled: true,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
),
|
||||
(
|
||||
"queued_failed",
|
||||
PoolDecommissionInfo {
|
||||
failed: true,
|
||||
queued: true,
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
),
|
||||
(
|
||||
"complete",
|
||||
PoolDecommissionInfo {
|
||||
complete: true,
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
),
|
||||
("idle", PoolDecommissionInfo::default(), false),
|
||||
];
|
||||
|
||||
for (name, decommission, expected) in cases {
|
||||
*store.pool_meta.write().await = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: format!("scanner-publication-{name}"),
|
||||
last_update: OffsetDateTime::now_utc(),
|
||||
decommission: Some(decommission),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
store.scanner_data_usage_publication_blocked().await,
|
||||
expected,
|
||||
"unexpected scanner publication barrier state for {name}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// The object graph is the isolation carrier: two ECStore instances holding
|
||||
// distinct contexts report independent erasure state through their real
|
||||
// `&self` accessors — no cross-contamination.
|
||||
|
||||
@@ -14,8 +14,6 @@
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::{
|
||||
bucket_lifecycle_ops::eval_action_from_lifecycle,
|
||||
get_expiry_configs,
|
||||
tier_delete_journal::{
|
||||
abort_prepared_tier_delete_journal_entry as abort_prepared_journal_entry_if_current, commit_tier_delete_journal_entry,
|
||||
enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry,
|
||||
@@ -213,8 +211,7 @@ async fn delete_prefix_with_tier_delete_journal(
|
||||
opts: &ObjectOptions,
|
||||
tier_journal_api: Option<&Arc<ECStore>>,
|
||||
) -> Result<()> {
|
||||
let lifecycle_delete_all = opts.lifecycle_delete_all.is_some();
|
||||
let journal_entry = if !lifecycle_delete_all && let Some(api) = tier_journal_api {
|
||||
let journal_entry = if let Some(api) = tier_journal_api {
|
||||
Some(prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?)
|
||||
} else {
|
||||
None
|
||||
@@ -223,34 +220,14 @@ async fn delete_prefix_with_tier_delete_journal(
|
||||
let result = store.delete_prefix(bucket, object, opts).await;
|
||||
match result {
|
||||
Ok(()) => {
|
||||
let lifecycle_entries = if lifecycle_delete_all {
|
||||
opts.lifecycle_delete_all_journal()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.lock()
|
||||
.prepared_entries()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let entries = journal_entry.as_deref().unwrap_or(&lifecycle_entries);
|
||||
if let Some(api) = tier_journal_api {
|
||||
if let (Some(api), Some(entries)) = (tier_journal_api, journal_entry.as_ref()) {
|
||||
commit_prepared_tier_delete_journal_entries(api, entries).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => {
|
||||
if let Some(api) = tier_journal_api {
|
||||
if lifecycle_delete_all {
|
||||
let (abort, entries) = {
|
||||
let journal = opts.lifecycle_delete_all_journal().ok_or(StorageError::PreconditionFailed)?;
|
||||
let state = journal.lock();
|
||||
(!state.mutation_started(), state.prepared_entries())
|
||||
};
|
||||
if abort {
|
||||
abort_prepared_tier_delete_journal_entries(api, &entries).await;
|
||||
}
|
||||
} else if let Some(entries) = journal_entry.as_ref() {
|
||||
abort_prepared_tier_delete_journal_entries(api, entries).await;
|
||||
}
|
||||
if let (Some(api), Some(entries)) = (tier_journal_api, journal_entry.as_ref()) {
|
||||
abort_prepared_tier_delete_journal_entries(api, entries).await;
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
@@ -350,7 +327,7 @@ impl fmt::Display for ObjectLockDiagMode {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ObjectLockDiagGuard {
|
||||
struct ObjectLockDiagGuard {
|
||||
guard: rustfs_lock::NamespaceLockGuard,
|
||||
enabled: bool,
|
||||
op: &'static str,
|
||||
@@ -383,14 +360,14 @@ impl ObjectLockDiagGuard {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
|
||||
fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
|
||||
match &self.guard {
|
||||
rustfs_lock::NamespaceLockGuard::Standard(guard) => Some(guard.lock_lost()),
|
||||
rustfs_lock::NamespaceLockGuard::Fast(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
fn is_lock_lost(&self) -> bool {
|
||||
self.guard.is_lock_lost()
|
||||
}
|
||||
}
|
||||
@@ -1132,26 +1109,6 @@ fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo,
|
||||
&& source_actual_size == target_actual_size
|
||||
}
|
||||
|
||||
fn tiered_data_movement_source_matches(
|
||||
expected: &rustfs_filemeta::FileInfo,
|
||||
current: &rustfs_filemeta::FileInfo,
|
||||
) -> Result<bool> {
|
||||
let expected_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&expected.metadata)?;
|
||||
let current_backend = crate::services::tier::tier::tier_destination_id_from_metadata(¤t.metadata)?;
|
||||
Ok(expected.version_id == current.version_id
|
||||
&& expected.data_dir == current.data_dir
|
||||
&& expected.mod_time == current.mod_time
|
||||
&& expected.size == current.size
|
||||
&& expected.get_etag() == current.get_etag()
|
||||
&& expected.transition_status == current.transition_status
|
||||
&& expected.transitioned_objname == current.transitioned_objname
|
||||
&& expected.transition_tier == current.transition_tier
|
||||
&& expected.transition_version_id == current.transition_version_id
|
||||
&& expected.transition_version == current.transition_version
|
||||
&& expected.transition_version_state == current.transition_version_state
|
||||
&& expected_backend == current_backend)
|
||||
}
|
||||
|
||||
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
|
||||
target_pool_idx != src_pool_idx
|
||||
}
|
||||
@@ -1290,9 +1247,7 @@ impl ECStore {
|
||||
let mut opts = opts.clone();
|
||||
opts.no_lock = false;
|
||||
opts.metadata_cache_safe = false;
|
||||
let read_lock_guards = self
|
||||
.acquire_all_object_read_locks("select_object", bucket, &object, &mut opts)
|
||||
.await?;
|
||||
let read_lock_guards = self.acquire_select_object_read_locks(bucket, &object, &mut opts).await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(SnapshotConsistencyError::LockingDisabled.into());
|
||||
}
|
||||
@@ -1533,9 +1488,8 @@ impl ECStore {
|
||||
)))
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_all_object_read_locks(
|
||||
async fn acquire_select_object_read_locks(
|
||||
&self,
|
||||
op: &'static str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &mut ObjectOptions,
|
||||
@@ -1547,7 +1501,10 @@ impl ECStore {
|
||||
// for each object's hashed set. DELETE and same-key CopyObject use the
|
||||
// fixed domain, while PUT commits and data movement use the hashed set.
|
||||
let distributed = self.ctx.is_dist_erasure().await;
|
||||
if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
|
||||
if let Some(guard) = self
|
||||
.acquire_object_read_lock_if_needed("select_object", bucket, object, opts)
|
||||
.await?
|
||||
{
|
||||
guards.push(guard);
|
||||
}
|
||||
let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
|
||||
@@ -1570,7 +1527,7 @@ impl ECStore {
|
||||
.map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
|
||||
let owner = diag_enabled.then(|| ns_lock.owner().to_string());
|
||||
log_object_lock_acquire_if_slow(
|
||||
op,
|
||||
"select_object",
|
||||
bucket,
|
||||
object,
|
||||
owner.as_deref(),
|
||||
@@ -1581,7 +1538,7 @@ impl ECStore {
|
||||
guards.push(ObjectLockDiagGuard::new(
|
||||
guard,
|
||||
diag_enabled,
|
||||
op,
|
||||
"select_object",
|
||||
diag_enabled.then(|| bucket.to_string()),
|
||||
diag_enabled.then(|| object.to_string()),
|
||||
owner,
|
||||
@@ -1592,77 +1549,6 @@ impl ECStore {
|
||||
Ok(guards)
|
||||
}
|
||||
|
||||
async fn acquire_data_movement_object_write_locks(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
source_pool_idx: usize,
|
||||
target_pool_idx: usize,
|
||||
opts: &mut ObjectOptions,
|
||||
) -> Result<Vec<ObjectLockDiagGuard>> {
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("tiered data movement requires namespace locking"));
|
||||
}
|
||||
let distributed = self.ctx.is_dist_erasure().await;
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
let mut pool_indices = [source_pool_idx, target_pool_idx];
|
||||
pool_indices.sort_unstable();
|
||||
let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
|
||||
let mut locked_sets = vec![fixed_set];
|
||||
let mut guards = Vec::with_capacity(3);
|
||||
|
||||
// Lock order matches journal recovery: fixed store domain first, then
|
||||
// hashed domains by ascending pool index. This also serializes source
|
||||
// revalidation and target publication against ordinary object deletes.
|
||||
guards.push(self.acquire_object_write_lock("tiered_data_movement", bucket, object).await?);
|
||||
for pool_idx in pool_indices {
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("invalid tiered data movement pool {pool_idx}")))?;
|
||||
let set = pool.get_disks_by_key(object);
|
||||
let lock_domain_already_held = !distributed
|
||||
|| locked_sets.iter().any(|locked_set: &Arc<crate::set_disk::SetDisks>| {
|
||||
same_distributed_lock_domain(&locked_set.lockers, &set.lockers)
|
||||
});
|
||||
if lock_domain_already_held {
|
||||
continue;
|
||||
}
|
||||
let ns_lock = set.new_ns_lock(bucket, object).await?;
|
||||
let acquire_start = Instant::now();
|
||||
let guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
|
||||
let owner = diag_enabled.then(|| ns_lock.owner().to_string());
|
||||
log_object_lock_acquire_if_slow(
|
||||
"tiered_data_movement",
|
||||
bucket,
|
||||
object,
|
||||
owner.as_deref(),
|
||||
ObjectLockDiagMode::Write,
|
||||
acquire_start.elapsed(),
|
||||
diag_enabled,
|
||||
);
|
||||
guards.push(ObjectLockDiagGuard::new(
|
||||
guard,
|
||||
diag_enabled,
|
||||
"tiered_data_movement",
|
||||
diag_enabled.then(|| bucket.to_string()),
|
||||
diag_enabled.then(|| object.to_string()),
|
||||
owner,
|
||||
ObjectLockDiagMode::Write,
|
||||
));
|
||||
locked_sets.push(set);
|
||||
}
|
||||
opts.no_lock = true;
|
||||
for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
|
||||
opts.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
opts.ensure_namespace_lock_fence();
|
||||
Ok(guards)
|
||||
}
|
||||
|
||||
fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
|
||||
if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
|
||||
return reader;
|
||||
@@ -1800,8 +1686,13 @@ impl ECStore {
|
||||
Some(guard)
|
||||
};
|
||||
|
||||
let logical_object = object;
|
||||
let object = encode_dir_object(logical_object);
|
||||
let mut fi = fi.clone();
|
||||
if opts.data_movement {
|
||||
crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
|
||||
}
|
||||
|
||||
let object = encode_dir_object(object);
|
||||
|
||||
if self.single_pool() {
|
||||
return Self::resolve_decommission_tiered_object_result(
|
||||
Err(Error::other("single pool deployments cannot decommission tiered objects")),
|
||||
@@ -1824,33 +1715,6 @@ impl ECStore {
|
||||
&object,
|
||||
)?
|
||||
};
|
||||
let _object_guards = self
|
||||
.acquire_data_movement_object_write_locks(bucket, &object, opts.src_pool_idx, idx, &mut opts)
|
||||
.await?;
|
||||
let source_pool = self
|
||||
.pools
|
||||
.get(opts.src_pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("invalid tiered data movement source pool {}", opts.src_pool_idx)))?;
|
||||
let source_versions = source_pool
|
||||
.get_disks_by_key(&object)
|
||||
.load_file_info_versions_exact(bucket, logical_object)
|
||||
.await?;
|
||||
let current_source = source_versions
|
||||
.as_ref()
|
||||
.and_then(|versions| {
|
||||
versions
|
||||
.versions
|
||||
.iter()
|
||||
.find(|current| current.version_id == fi.version_id && !current.tier_free_version())
|
||||
})
|
||||
.ok_or_else(|| to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]))?;
|
||||
if !tiered_data_movement_source_matches(fi, current_source)? {
|
||||
return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]));
|
||||
}
|
||||
let mut fi = current_source.clone();
|
||||
if opts.data_movement {
|
||||
crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
|
||||
}
|
||||
if opts.data_movement && idx == opts.src_pool_idx {
|
||||
let resume_target_pool_idx = self
|
||||
.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
|
||||
@@ -2326,10 +2190,6 @@ impl ECStore {
|
||||
) -> Result<ObjectInfo> {
|
||||
check_del_obj_args(bucket, object)?;
|
||||
|
||||
if opts.lifecycle_delete_all.is_some() && self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("lifecycle delete-all requires namespace locking"));
|
||||
}
|
||||
|
||||
let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
|
||||
None
|
||||
} else if opts.delete_prefix {
|
||||
@@ -2344,11 +2204,6 @@ impl ECStore {
|
||||
};
|
||||
let object = object.as_str();
|
||||
let mut opts = opts;
|
||||
let delete_all_configs = if opts.lifecycle_delete_all.is_some() {
|
||||
Some(get_expiry_configs(self, bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
opts.tier_delete_journal_api = tier_journal_api.clone();
|
||||
if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
|
||||
opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
@@ -2446,34 +2301,6 @@ impl ECStore {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(trigger) = opts.lifecycle_delete_all.as_ref() {
|
||||
let configs = delete_all_configs.as_ref().ok_or(StorageError::PreconditionFailed)?;
|
||||
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
|
||||
if configs.table_bucket_enabled || configs.bucket_incarnation_id != expected_bucket_incarnation_id {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
let lifecycle = configs.lifecycle.as_ref().ok_or(StorageError::PreconditionFailed)?;
|
||||
let (mut current, _) = self
|
||||
.get_latest_object_info_with_idx(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let current_version_id = current.version_id.filter(|version_id| !version_id.is_nil());
|
||||
if current_version_id != trigger.version_id || current.delete_marker != trigger.delete_marker {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
current.name = decode_dir_object(¤t.name);
|
||||
let current_event = eval_action_from_lifecycle(lifecycle, configs.object_lock.as_deref(), ¤t).await;
|
||||
if current_event.action != trigger.action || current_event.rule_id != trigger.rule_id {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
if opts.delete_prefix {
|
||||
delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
|
||||
return Ok(ObjectInfo::default());
|
||||
@@ -4001,24 +3828,6 @@ mod tests {
|
||||
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiered_data_movement_source_match_rejects_transition_identity_changes() {
|
||||
let source = tiered_equivalence_source();
|
||||
assert!(tiered_data_movement_source_matches(&source, &source).expect("matching source metadata should parse"));
|
||||
|
||||
let mut changed_remote = source.clone();
|
||||
changed_remote.transitioned_objname = "remote/replaced".to_string();
|
||||
assert!(!tiered_data_movement_source_matches(&source, &changed_remote).expect("changed remote metadata should parse"));
|
||||
|
||||
let mut changed_backend = source.clone();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut changed_backend.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex([9; 32]),
|
||||
);
|
||||
assert!(!tiered_data_movement_source_matches(&source, &changed_backend).expect("backend metadata should parse"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equivalent_data_movement_tiered_object_uses_logical_compressed_and_encrypted_sizes() {
|
||||
let mut compressed = tiered_equivalence_source();
|
||||
|
||||
@@ -202,76 +202,10 @@ impl ECStore {
|
||||
}
|
||||
|
||||
pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
if opts.lifecycle_delete_all.is_some() {
|
||||
let mut preflight_opts = opts.clone();
|
||||
preflight_opts
|
||||
.lifecycle_delete_all
|
||||
.as_mut()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
|
||||
for pool in &self.pools {
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
|
||||
pool.delete_object(bucket, object, preflight_opts.clone()).await?;
|
||||
}
|
||||
|
||||
opts.lifecycle_delete_all_journal()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.lock()
|
||||
.mark_mutation_started();
|
||||
let mut non_trigger_opts = opts.clone();
|
||||
non_trigger_opts
|
||||
.lifecycle_delete_all
|
||||
.as_mut()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::History;
|
||||
for pool in &self.pools {
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
|
||||
let mut pool_opts = non_trigger_opts.clone();
|
||||
pool_opts.delete_prefix = true;
|
||||
pool.delete_object(bucket, object, pool_opts).await?;
|
||||
}
|
||||
|
||||
let mut final_preflight_opts = opts.clone();
|
||||
final_preflight_opts
|
||||
.lifecycle_delete_all
|
||||
.as_mut()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
|
||||
let mut trigger_pools = Vec::new();
|
||||
for (pool_index, pool) in self.pools.iter().enumerate() {
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
|
||||
let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
|
||||
if !result.name.is_empty() {
|
||||
trigger_pools.push(pool_index);
|
||||
}
|
||||
}
|
||||
if trigger_pools.is_empty() {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
|
||||
let mut trigger_opts = opts.clone();
|
||||
trigger_opts
|
||||
.lifecycle_delete_all
|
||||
.as_mut()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
|
||||
for pool_index in trigger_pools {
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
|
||||
let mut pool_opts = trigger_opts.clone();
|
||||
pool_opts.delete_prefix = true;
|
||||
self.pools[pool_index].delete_object(bucket, object, pool_opts).await?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut first_error = None;
|
||||
let mut first_volume_error = None;
|
||||
let mut has_success = false;
|
||||
for pool in &self.pools {
|
||||
for pool in self.pools.iter() {
|
||||
let mut opts = opts.clone();
|
||||
opts.delete_prefix = true;
|
||||
match pool.delete_object(bucket, object, opts).await {
|
||||
@@ -840,22 +774,6 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static LIFECYCLE_DELETE_ALL_TEST_FAILURE: std::sync::Mutex<Option<(crate::object_api::LifecycleDeleteAllPhase, usize)>> =
|
||||
std::sync::Mutex::new(None);
|
||||
|
||||
#[cfg(test)]
|
||||
fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAllPhase, pool_index: usize) -> Result<()> {
|
||||
if LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison")
|
||||
.is_some_and(|failure| failure == (phase, pool_index))
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -863,28 +781,23 @@ mod tests {
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::object_api::ObjectLockConfigSnapshot;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use arc_swap::ArcSwap;
|
||||
use rustfs_config::server_config::KVS;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use std::sync::Arc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
async fn setup_multi_pool_test_store(
|
||||
name: &str,
|
||||
drives_per_pool: &[usize],
|
||||
) -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool test directory should be created");
|
||||
let mut pools = Vec::with_capacity(drives_per_pool.len());
|
||||
for (pool_index, drives_per_set) in drives_per_pool.iter().copied().enumerate() {
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_attempts_later_pools_after_an_earlier_pool_error() {
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool delete test directory should be created");
|
||||
let mut pools = Vec::with_capacity(2);
|
||||
for (pool_index, drives_per_set) in [2, 4].into_iter().enumerate() {
|
||||
let mut endpoints = Vec::with_capacity(drives_per_set);
|
||||
for disk_index in 0..drives_per_set {
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
.expect("multi-pool test disk should be created");
|
||||
.expect("multi-pool delete test disk should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
@@ -897,7 +810,7 @@ mod tests {
|
||||
set_count: 1,
|
||||
drives_per_set,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("{name}-pool-{pool_index}"),
|
||||
cmd_line: format!("delete-prefix-pool-{pool_index}"),
|
||||
platform: "test".to_string(),
|
||||
});
|
||||
}
|
||||
@@ -917,92 +830,6 @@ mod tests {
|
||||
.await
|
||||
.expect("multi-pool store should initialize");
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(temp_dir, store, shutdown)
|
||||
}
|
||||
|
||||
struct LifecycleDeleteAllFailureGuard;
|
||||
|
||||
impl Drop for LifecycleDeleteAllFailureGuard {
|
||||
fn drop(&mut self) {
|
||||
*LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison") = None;
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_multi_pool_delete_all(store: &Arc<ECStore>, bucket: &str, object: &str) -> ObjectOptions {
|
||||
let trigger_id = Uuid::new_v4();
|
||||
for (pool_index, pool) in store.pools.iter().enumerate() {
|
||||
let mut history_reader = PutObjReader::from_vec(format!("{object}-history-{pool_index}").into_bytes());
|
||||
pool.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut history_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("history should be stored");
|
||||
let mut trigger_reader = PutObjReader::from_vec(format!("{object}-trigger-{pool_index}").into_bytes());
|
||||
pool.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut trigger_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(trigger_id.to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("shared trigger should be stored");
|
||||
}
|
||||
let mut opts = ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
versioned: true,
|
||||
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||
version_id: Some(trigger_id),
|
||||
delete_marker: false,
|
||||
action: rustfs_common::metrics::IlmAction::DeleteAllVersionsAction,
|
||||
rule_id: "rule".to_string(),
|
||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||
}),
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
delete_replication_config_snapshot: Some(Arc::new(
|
||||
crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
|
||||
)),
|
||||
..Default::default()
|
||||
};
|
||||
opts.ensure_lifecycle_delete_all_journal();
|
||||
opts
|
||||
}
|
||||
|
||||
async fn ordinary_version_count(store: &ECStore, pool_index: usize, bucket: &str, object: &str) -> usize {
|
||||
store.pools[pool_index].disk_set[0]
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("pool metadata should load")
|
||||
.map(|versions| {
|
||||
versions
|
||||
.versions
|
||||
.iter()
|
||||
.filter(|version| !version.tier_free_version())
|
||||
.count()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_attempts_later_pools_after_an_earlier_pool_error() {
|
||||
let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("delete-prefix", &[2, 4]).await;
|
||||
let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
@@ -1094,171 +921,6 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn lifecycle_delete_all_history_failure_preserves_trigger_and_retry_converges() {
|
||||
let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("lifecycle-delete-all", &[4, 4]).await;
|
||||
let bucket = format!("lifecycle-delete-all-{}", Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created in both pools");
|
||||
|
||||
for pool_index in 0..2 {
|
||||
let mut reader = PutObjReader::from_vec(format!("pool-{pool_index}-history").into_bytes());
|
||||
store.pools[pool_index]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("historical version should be stored");
|
||||
}
|
||||
let marker = store.pools[0]
|
||||
.delete_object(
|
||||
&bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("trigger marker should be stored in the first pool");
|
||||
let marker_id = marker.version_id.expect("trigger marker should have a version id");
|
||||
let mut opts = ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
versioned: true,
|
||||
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||
version_id: Some(marker_id),
|
||||
delete_marker: true,
|
||||
action: rustfs_common::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
|
||||
rule_id: "rule".to_string(),
|
||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||
}),
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
delete_replication_config_snapshot: Some(Arc::new(
|
||||
crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
|
||||
)),
|
||||
..Default::default()
|
||||
};
|
||||
opts.ensure_lifecycle_delete_all_journal();
|
||||
|
||||
let _failure_guard = LifecycleDeleteAllFailureGuard;
|
||||
*LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison") =
|
||||
Some((crate::object_api::LifecycleDeleteAllPhase::History, 1));
|
||||
let err = store
|
||||
.delete_prefix(&bucket, object, &opts)
|
||||
.await
|
||||
.expect_err("a later pool history failure must stop before trigger deletion");
|
||||
assert_eq!(err, StorageError::PreconditionFailed);
|
||||
assert!(
|
||||
opts.lifecycle_delete_all_journal()
|
||||
.expect("delete-all journal should be initialized")
|
||||
.lock()
|
||||
.mutation_started()
|
||||
);
|
||||
|
||||
let first_pool = store.pools[0].disk_set[0]
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("first pool metadata should load")
|
||||
.expect("the trigger should remain");
|
||||
let first_pool_ordinary: Vec<&FileInfo> = first_pool
|
||||
.versions
|
||||
.iter()
|
||||
.filter(|version| !version.tier_free_version())
|
||||
.collect();
|
||||
assert_eq!(first_pool_ordinary.len(), 1);
|
||||
assert_eq!(first_pool_ordinary[0].version_id, Some(marker_id));
|
||||
assert!(first_pool_ordinary[0].deleted);
|
||||
|
||||
*LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison") = None;
|
||||
store
|
||||
.delete_prefix(&bucket, object, &opts)
|
||||
.await
|
||||
.expect("retry should delete remaining history and its trigger owner");
|
||||
for pool in &store.pools {
|
||||
assert!(
|
||||
pool.disk_set[0]
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("pool metadata should load after retry")
|
||||
.is_none(),
|
||||
"all ordinary versions should be removed after retry"
|
||||
);
|
||||
}
|
||||
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn lifecycle_delete_all_phase_failures_preserve_barriers_and_retry() {
|
||||
let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("lifecycle-delete-all-phases", &[4, 4]).await;
|
||||
let bucket = format!("lifecycle-delete-all-phases-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created in both pools");
|
||||
let _failure_guard = LifecycleDeleteAllFailureGuard;
|
||||
|
||||
for (object, phase, expected_counts, mutation_started) in [
|
||||
("preflight-failure", crate::object_api::LifecycleDeleteAllPhase::Preflight, [2, 2], false),
|
||||
(
|
||||
"final-preflight-failure",
|
||||
crate::object_api::LifecycleDeleteAllPhase::FinalPreflight,
|
||||
[1, 1],
|
||||
true,
|
||||
),
|
||||
("trigger-failure", crate::object_api::LifecycleDeleteAllPhase::Trigger, [0, 1], true),
|
||||
] {
|
||||
let opts = seed_multi_pool_delete_all(&store, &bucket, object).await;
|
||||
*LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison") = Some((phase, 1));
|
||||
let err = store
|
||||
.delete_prefix(&bucket, object, &opts)
|
||||
.await
|
||||
.expect_err("injected phase failure should stop the transaction");
|
||||
assert_eq!(err, StorageError::PreconditionFailed);
|
||||
assert_eq!(
|
||||
opts.lifecycle_delete_all_journal()
|
||||
.expect("delete-all journal should be initialized")
|
||||
.lock()
|
||||
.mutation_started(),
|
||||
mutation_started
|
||||
);
|
||||
assert_eq!(ordinary_version_count(&store, 0, &bucket, object).await, expected_counts[0]);
|
||||
assert_eq!(ordinary_version_count(&store, 1, &bucket, object).await, expected_counts[1]);
|
||||
|
||||
*LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison") = None;
|
||||
store
|
||||
.delete_prefix(&bucket, object, &opts)
|
||||
.await
|
||||
.expect("retry should converge after the injected failure is removed");
|
||||
assert_eq!(ordinary_version_count(&store, 0, &bucket, object).await, 0);
|
||||
assert_eq!(ordinary_version_count(&store, 1, &bucket, object).await, 0);
|
||||
}
|
||||
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
fn assert_backend_layout_empty(info: &rustfs_madmin::BackendInfo) {
|
||||
assert!(info.standard_sc_parities.is_empty());
|
||||
assert!(info.standard_sc_data.is_empty());
|
||||
|
||||
@@ -759,7 +759,7 @@ impl HealChannelProcessor {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::DiskStore;
|
||||
use super::super::{DiskStore, Endpoint};
|
||||
use super::*;
|
||||
use crate::heal::manager::HealConfig;
|
||||
use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
|
||||
@@ -776,18 +776,45 @@ mod tests {
|
||||
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> crate::Result<Option<HealObjectInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn get_object_data(&self, _bucket: &str, _object: &str) -> crate::Result<Option<Vec<u8>>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn delete_object(&self, _bucket: &str, _object: &str) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> crate::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> crate::Result<Vec<u8>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn get_disk_status(&self, _endpoint: &Endpoint) -> crate::Result<crate::heal::storage::DiskStatus> {
|
||||
Ok(crate::heal::storage::DiskStatus::Ok)
|
||||
}
|
||||
async fn format_disk(&self, _endpoint: &Endpoint) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> crate::Result<Option<crate::heal::storage_api::status::BucketInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn heal_bucket_metadata(&self, _bucket: &str) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn list_buckets(&self) -> crate::Result<Vec<crate::heal::storage_api::status::BucketInfo>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn object_exists(&self, _bucket: &str, _object: &str) -> crate::Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
async fn get_object_size(&self, _bucket: &str, _object: &str) -> crate::Result<Option<u64>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> crate::Result<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn heal_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -810,6 +837,13 @@ mod tests {
|
||||
) -> crate::Result<(rustfs_madmin::heal_commands::HealResultItem, Option<crate::Error>)> {
|
||||
Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None))
|
||||
}
|
||||
async fn list_objects_for_heal(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
) -> crate::Result<Vec<crate::heal::storage::HealListItem>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
|
||||
@@ -1267,7 +1267,7 @@ mod resume_loop_tests {
|
||||
CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils,
|
||||
compose_key,
|
||||
};
|
||||
use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage::{DiskStatus, HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage_api::status::BucketInfo;
|
||||
use crate::heal::{
|
||||
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk,
|
||||
@@ -1448,15 +1448,36 @@ mod resume_loop_tests {
|
||||
async fn get_object_meta(&self, _b: &str, _o: &str) -> Result<Option<HealObjectInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn get_object_data(&self, _b: &str, _o: &str) -> Result<Option<Vec<u8>>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn put_object_data(&self, _b: &str, _o: &str, _d: &[u8]) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn delete_object(&self, _b: &str, _o: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn verify_object_integrity(&self, _b: &str, _o: &str) -> Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
async fn ec_decode_rebuild(&self, _b: &str, _o: &str) -> Result<Vec<u8>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn get_disk_status(&self, _e: &Endpoint) -> Result<DiskStatus> {
|
||||
Ok(DiskStatus::Ok)
|
||||
}
|
||||
async fn format_disk(&self, _e: &Endpoint) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
|
||||
Ok(Some(BucketInfo {
|
||||
name: bucket.to_string(),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
async fn heal_bucket_metadata(&self, _b: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
@@ -1464,6 +1485,12 @@ mod resume_loop_tests {
|
||||
// Must never be consulted: the resume loop always goes through heal_object.
|
||||
panic!("object_exists must not be called by the resume heal loop");
|
||||
}
|
||||
async fn get_object_size(&self, _b: &str, _o: &str) -> Result<Option<u64>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn get_object_checksum(&self, _b: &str, _o: &str) -> Result<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
Ok((!self.lifecycle_expired.lock().unwrap().is_empty()).then(HealLifecycleExpiryContext::test))
|
||||
}
|
||||
@@ -1529,6 +1556,9 @@ mod resume_loop_tests {
|
||||
ReplacementCommitEvidence::Error(message) => Err(Error::other(message)),
|
||||
}
|
||||
}
|
||||
async fn list_objects_for_heal(&self, _b: &str, _p: &str) -> Result<Vec<HealListItem>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
|
||||
@@ -0,0 +1,683 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::heal::{HealOptions, HealPriority, HealRequest, HealType};
|
||||
use crate::{Error, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use super::Endpoint;
|
||||
|
||||
/// Corruption type
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum CorruptionType {
|
||||
/// Data corruption
|
||||
DataCorruption,
|
||||
/// Metadata corruption
|
||||
MetadataCorruption,
|
||||
/// Partial corruption
|
||||
PartialCorruption,
|
||||
/// Complete corruption
|
||||
CompleteCorruption,
|
||||
}
|
||||
|
||||
/// Severity level
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
pub enum Severity {
|
||||
/// Low severity
|
||||
Low = 0,
|
||||
/// Medium severity
|
||||
Medium = 1,
|
||||
/// High severity
|
||||
High = 2,
|
||||
/// Critical severity
|
||||
Critical = 3,
|
||||
}
|
||||
|
||||
/// Heal event
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum HealEvent {
|
||||
/// Object corruption event
|
||||
ObjectCorruption {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
corruption_type: CorruptionType,
|
||||
severity: Severity,
|
||||
},
|
||||
/// Object missing event
|
||||
ObjectMissing {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
expected_locations: Vec<usize>,
|
||||
available_locations: Vec<usize>,
|
||||
},
|
||||
/// Metadata corruption event
|
||||
MetadataCorruption {
|
||||
bucket: String,
|
||||
object: String,
|
||||
corruption_type: CorruptionType,
|
||||
},
|
||||
/// Disk status change event
|
||||
DiskStatusChange {
|
||||
endpoint: Endpoint,
|
||||
old_status: String,
|
||||
new_status: String,
|
||||
},
|
||||
/// EC decode failure event
|
||||
ECDecodeFailure {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
missing_shards: Vec<usize>,
|
||||
available_shards: Vec<usize>,
|
||||
},
|
||||
/// Checksum mismatch event
|
||||
ChecksumMismatch {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
expected_checksum: String,
|
||||
actual_checksum: String,
|
||||
},
|
||||
/// Bucket metadata corruption event
|
||||
BucketMetadataCorruption {
|
||||
bucket: String,
|
||||
corruption_type: CorruptionType,
|
||||
},
|
||||
/// MRF metadata corruption event
|
||||
MRFMetadataCorruption {
|
||||
meta_path: String,
|
||||
corruption_type: CorruptionType,
|
||||
},
|
||||
}
|
||||
|
||||
impl HealEvent {
|
||||
/// Convert HealEvent to HealRequest
|
||||
pub fn to_heal_request(&self) -> Result<HealRequest> {
|
||||
match self {
|
||||
HealEvent::ObjectCorruption {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
severity,
|
||||
..
|
||||
} => Ok(HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
version_id: version_id.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
Self::severity_to_priority(severity),
|
||||
)),
|
||||
HealEvent::ObjectMissing {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
..
|
||||
} => Ok(HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
version_id: version_id.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)),
|
||||
HealEvent::MetadataCorruption { bucket, object, .. } => Ok(HealRequest::new(
|
||||
HealType::Metadata {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)),
|
||||
HealEvent::DiskStatusChange { endpoint, .. } => {
|
||||
// Convert disk status change to erasure set heal
|
||||
// Note: This requires access to storage to get bucket list, which is not available here
|
||||
// The actual bucket list will need to be provided by the caller or retrieved differently
|
||||
let set_disk_id = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx)
|
||||
.ok_or_else(|| Error::InvalidHealType {
|
||||
heal_type: format!("erasure-set(pool={}, set={})", endpoint.pool_idx, endpoint.set_idx),
|
||||
})?;
|
||||
Ok(HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec![], // Empty bucket list - caller should populate this
|
||||
set_disk_id,
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
))
|
||||
}
|
||||
HealEvent::ECDecodeFailure {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
..
|
||||
} => Ok(HealRequest::new(
|
||||
HealType::ECDecode {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
version_id: version_id.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Urgent,
|
||||
)),
|
||||
HealEvent::ChecksumMismatch {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
..
|
||||
} => Ok(HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
version_id: version_id.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)),
|
||||
HealEvent::BucketMetadataCorruption { bucket, .. } => Ok(HealRequest::new(
|
||||
HealType::Bucket { bucket: bucket.clone() },
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)),
|
||||
HealEvent::MRFMetadataCorruption { meta_path, .. } => Ok(HealRequest::new(
|
||||
HealType::MRF {
|
||||
meta_path: meta_path.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert severity to priority
|
||||
fn severity_to_priority(severity: &Severity) -> HealPriority {
|
||||
match severity {
|
||||
Severity::Low => HealPriority::Low,
|
||||
Severity::Medium => HealPriority::Normal,
|
||||
Severity::High => HealPriority::High,
|
||||
Severity::Critical => HealPriority::Urgent,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get event description
|
||||
pub fn description(&self) -> String {
|
||||
match self {
|
||||
HealEvent::ObjectCorruption {
|
||||
bucket,
|
||||
object,
|
||||
corruption_type,
|
||||
..
|
||||
} => {
|
||||
format!("Object corruption detected: {bucket}/{object} - {corruption_type:?}")
|
||||
}
|
||||
HealEvent::ObjectMissing { bucket, object, .. } => {
|
||||
format!("Object missing: {bucket}/{object}")
|
||||
}
|
||||
HealEvent::MetadataCorruption {
|
||||
bucket,
|
||||
object,
|
||||
corruption_type,
|
||||
..
|
||||
} => {
|
||||
format!("Metadata corruption: {bucket}/{object} - {corruption_type:?}")
|
||||
}
|
||||
HealEvent::DiskStatusChange {
|
||||
endpoint,
|
||||
old_status,
|
||||
new_status,
|
||||
..
|
||||
} => {
|
||||
format!("Disk status changed: {endpoint:?} {old_status} -> {new_status}")
|
||||
}
|
||||
HealEvent::ECDecodeFailure {
|
||||
bucket,
|
||||
object,
|
||||
missing_shards,
|
||||
..
|
||||
} => {
|
||||
format!("EC decode failure: {bucket}/{object} - missing shards: {missing_shards:?}")
|
||||
}
|
||||
HealEvent::ChecksumMismatch {
|
||||
bucket,
|
||||
object,
|
||||
expected_checksum,
|
||||
actual_checksum,
|
||||
..
|
||||
} => {
|
||||
format!("Checksum mismatch: {bucket}/{object} - expected: {expected_checksum}, actual: {actual_checksum}")
|
||||
}
|
||||
HealEvent::BucketMetadataCorruption {
|
||||
bucket, corruption_type, ..
|
||||
} => {
|
||||
format!("Bucket metadata corruption: {bucket} - {corruption_type:?}")
|
||||
}
|
||||
HealEvent::MRFMetadataCorruption {
|
||||
meta_path,
|
||||
corruption_type,
|
||||
..
|
||||
} => {
|
||||
format!("MRF metadata corruption: {meta_path} - {corruption_type:?}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get event severity
|
||||
pub fn severity(&self) -> Severity {
|
||||
match self {
|
||||
HealEvent::ObjectCorruption { severity, .. } => severity.clone(),
|
||||
HealEvent::ObjectMissing { .. } => Severity::High,
|
||||
HealEvent::MetadataCorruption { .. } => Severity::High,
|
||||
HealEvent::DiskStatusChange { .. } => Severity::High,
|
||||
HealEvent::ECDecodeFailure { .. } => Severity::Critical,
|
||||
HealEvent::ChecksumMismatch { .. } => Severity::High,
|
||||
HealEvent::BucketMetadataCorruption { .. } => Severity::High,
|
||||
HealEvent::MRFMetadataCorruption { .. } => Severity::High,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get event timestamp
|
||||
pub fn timestamp(&self) -> SystemTime {
|
||||
SystemTime::now()
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal event handler
|
||||
pub struct HealEventHandler {
|
||||
/// Event queue
|
||||
events: Vec<HealEvent>,
|
||||
/// Maximum number of events
|
||||
max_events: usize,
|
||||
}
|
||||
|
||||
impl HealEventHandler {
|
||||
pub fn new(max_events: usize) -> Self {
|
||||
Self {
|
||||
events: Vec::new(),
|
||||
max_events,
|
||||
}
|
||||
}
|
||||
|
||||
/// Add event
|
||||
pub fn add_event(&mut self, event: HealEvent) {
|
||||
if self.events.len() >= self.max_events {
|
||||
// Remove oldest event
|
||||
self.events.remove(0);
|
||||
}
|
||||
self.events.push(event);
|
||||
}
|
||||
|
||||
/// Get all events
|
||||
pub fn get_events(&self) -> &[HealEvent] {
|
||||
&self.events
|
||||
}
|
||||
|
||||
/// Clear events
|
||||
pub fn clear_events(&mut self) {
|
||||
self.events.clear();
|
||||
}
|
||||
|
||||
/// Get event count
|
||||
pub fn event_count(&self) -> usize {
|
||||
self.events.len()
|
||||
}
|
||||
|
||||
/// Filter events by severity
|
||||
pub fn filter_by_severity(&self, min_severity: Severity) -> Vec<&HealEvent> {
|
||||
self.events.iter().filter(|event| event.severity() >= min_severity).collect()
|
||||
}
|
||||
|
||||
/// Filter events by type
|
||||
pub fn filter_by_type(&self, event_type: &str) -> Vec<&HealEvent> {
|
||||
self.events
|
||||
.iter()
|
||||
.filter(|event| match event {
|
||||
HealEvent::ObjectCorruption { .. } => event_type == "ObjectCorruption",
|
||||
HealEvent::ObjectMissing { .. } => event_type == "ObjectMissing",
|
||||
HealEvent::MetadataCorruption { .. } => event_type == "MetadataCorruption",
|
||||
HealEvent::DiskStatusChange { .. } => event_type == "DiskStatusChange",
|
||||
HealEvent::ECDecodeFailure { .. } => event_type == "ECDecodeFailure",
|
||||
HealEvent::ChecksumMismatch { .. } => event_type == "ChecksumMismatch",
|
||||
HealEvent::BucketMetadataCorruption { .. } => event_type == "BucketMetadataCorruption",
|
||||
HealEvent::MRFMetadataCorruption { .. } => event_type == "MRFMetadataCorruption",
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for HealEventHandler {
|
||||
fn default() -> Self {
|
||||
Self::new(1000)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::heal::task::{HealPriority, HealType};
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_object_corruption_to_request() {
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Object { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_object_missing_to_request() {
|
||||
let event = HealEvent::ObjectMissing {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: Some("v1".to_string()),
|
||||
expected_locations: vec![0, 1],
|
||||
available_locations: vec![2, 3],
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Object { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_metadata_corruption_to_request() {
|
||||
let event = HealEvent::MetadataCorruption {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
corruption_type: CorruptionType::MetadataCorruption,
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Metadata { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_ec_decode_failure_to_request() {
|
||||
let event = HealEvent::ECDecodeFailure {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: None,
|
||||
missing_shards: vec![0, 1],
|
||||
available_shards: vec![2, 3, 4],
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::ECDecode { .. }));
|
||||
assert_eq!(request.priority, HealPriority::Urgent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_checksum_mismatch_to_request() {
|
||||
let event = HealEvent::ChecksumMismatch {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: None,
|
||||
expected_checksum: "abc123".to_string(),
|
||||
actual_checksum: "def456".to_string(),
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Object { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_bucket_metadata_corruption_to_request() {
|
||||
let event = HealEvent::BucketMetadataCorruption {
|
||||
bucket: "test-bucket".to_string(),
|
||||
corruption_type: CorruptionType::MetadataCorruption,
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Bucket { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_mrf_metadata_corruption_to_request() {
|
||||
let event = HealEvent::MRFMetadataCorruption {
|
||||
meta_path: "test-bucket/test-object".to_string(),
|
||||
corruption_type: CorruptionType::MetadataCorruption,
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::MRF { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_severity_to_priority() {
|
||||
let event_low = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::Low,
|
||||
};
|
||||
let request = event_low.to_heal_request().unwrap();
|
||||
assert_eq!(request.priority, HealPriority::Low);
|
||||
|
||||
let event_medium = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::Medium,
|
||||
};
|
||||
let request = event_medium.to_heal_request().unwrap();
|
||||
assert_eq!(request.priority, HealPriority::Normal);
|
||||
|
||||
let event_high = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
let request = event_high.to_heal_request().unwrap();
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
|
||||
let event_critical = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::Critical,
|
||||
};
|
||||
let request = event_critical.to_heal_request().unwrap();
|
||||
assert_eq!(request.priority, HealPriority::Urgent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_description() {
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
let desc = event.description();
|
||||
assert!(desc.contains("Object corruption detected"));
|
||||
assert!(desc.contains("test-bucket/test-object"));
|
||||
assert!(desc.contains("DataCorruption"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_severity() {
|
||||
let event = HealEvent::ECDecodeFailure {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
missing_shards: vec![],
|
||||
available_shards: vec![],
|
||||
};
|
||||
assert_eq!(event.severity(), Severity::Critical);
|
||||
|
||||
let event = HealEvent::ObjectMissing {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
expected_locations: vec![],
|
||||
available_locations: vec![],
|
||||
};
|
||||
assert_eq!(event.severity(), Severity::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_new() {
|
||||
let handler = HealEventHandler::new(10);
|
||||
assert_eq!(handler.event_count(), 0);
|
||||
assert_eq!(handler.max_events, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_default() {
|
||||
let handler = HealEventHandler::default();
|
||||
assert_eq!(handler.max_events, 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_add_event() {
|
||||
let mut handler = HealEventHandler::new(3);
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
handler.add_event(event.clone());
|
||||
assert_eq!(handler.event_count(), 1);
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event);
|
||||
assert_eq!(handler.event_count(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_max_events() {
|
||||
let mut handler = HealEventHandler::new(2);
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event); // Should remove oldest
|
||||
|
||||
assert_eq!(handler.event_count(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_get_events() {
|
||||
let mut handler = HealEventHandler::new(10);
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event);
|
||||
|
||||
let events = handler.get_events();
|
||||
assert_eq!(events.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_clear_events() {
|
||||
let mut handler = HealEventHandler::new(10);
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
handler.add_event(event);
|
||||
assert_eq!(handler.event_count(), 1);
|
||||
|
||||
handler.clear_events();
|
||||
assert_eq!(handler.event_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_filter_by_severity() {
|
||||
let mut handler = HealEventHandler::new(10);
|
||||
handler.add_event(HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::Low,
|
||||
});
|
||||
handler.add_event(HealEvent::ECDecodeFailure {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
missing_shards: vec![],
|
||||
available_shards: vec![],
|
||||
});
|
||||
|
||||
let high_severity = handler.filter_by_severity(Severity::High);
|
||||
assert_eq!(high_severity.len(), 1); // Only ECDecodeFailure is Critical >= High
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_filter_by_type() {
|
||||
let mut handler = HealEventHandler::new(10);
|
||||
handler.add_event(HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
});
|
||||
handler.add_event(HealEvent::ObjectMissing {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
expected_locations: vec![],
|
||||
available_locations: vec![],
|
||||
});
|
||||
|
||||
let corruption_events = handler.filter_by_type("ObjectCorruption");
|
||||
assert_eq!(corruption_events.len(), 1);
|
||||
|
||||
let missing_events = handler.filter_by_type("ObjectMissing");
|
||||
assert_eq!(missing_events.len(), 1);
|
||||
}
|
||||
}
|
||||
+5001
-266
File diff suppressed because it is too large
Load Diff
@@ -1,562 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// The automatic disk scanner: replacement discovery and unformatted-disk enqueue.
|
||||
use super::*;
|
||||
|
||||
impl HealManager {
|
||||
/// Start background task to auto scan local disks and enqueue erasure set heal requests
|
||||
pub(super) async fn start_auto_disk_scanner(&self) -> Result<()> {
|
||||
let config = self.config.clone();
|
||||
let heal_queue = self.heal_queue.clone();
|
||||
let active_heals = self.active_heals.clone();
|
||||
let task_aliases = self.task_aliases.clone();
|
||||
let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
|
||||
let storage = self.storage.clone();
|
||||
let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
|
||||
let replacement_recovery_blocked_sets = self.replacement_recovery_blocked_sets.clone();
|
||||
let cancel_token = self.cancel_token.clone();
|
||||
let notify = self.notify.clone();
|
||||
let mut duration = {
|
||||
let config = config.read().await;
|
||||
config.heal_interval
|
||||
};
|
||||
if duration < Duration::from_secs(10) {
|
||||
duration = Duration::from_secs(10);
|
||||
}
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "started",
|
||||
interval = ?duration,
|
||||
"Heal auto disk scanner started"
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = interval(duration);
|
||||
|
||||
loop {
|
||||
let mut candidate_count = 0usize;
|
||||
let mut skipped_duplicate_count = 0usize;
|
||||
let mut skipped_invalid_count = 0usize;
|
||||
let mut enqueued_count = 0usize;
|
||||
let mut not_enqueued_count = 0usize;
|
||||
let mut dropped_count = 0usize;
|
||||
let mut full_count = 0usize;
|
||||
tokio::select! {
|
||||
_ = cancel_token.cancelled() => {
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "shutdown",
|
||||
"Heal auto disk scanner stopped"
|
||||
);
|
||||
break;
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
// Build list of endpoints that need healing
|
||||
let mut endpoints = HashMap::<String, Vec<Endpoint>>::new();
|
||||
let mut durable_recoveries = HashMap::<String, (String, Vec<Endpoint>, Vec<String>, String)>::new();
|
||||
let mut conflicted_recovery_sets = HashSet::<String>::new();
|
||||
let mut deferred_replacement_endpoints = HashSet::<String>::new();
|
||||
let local_disks = {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
let local_endpoints = local_disks.iter().map(|disk| disk.endpoint()).collect::<Vec<_>>();
|
||||
let blocked_sets = replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.clone();
|
||||
if !blocked_sets.is_empty() {
|
||||
let mut retry_succeeded = HashSet::new();
|
||||
let mut retry_failed = HashSet::new();
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
let Some(set_disk_id) =
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !blocked_sets.contains(&set_disk_id) {
|
||||
continue;
|
||||
}
|
||||
match Self::validate_replacement_recovery_records(disk).await {
|
||||
Ok(()) => {
|
||||
retry_succeeded.insert(set_disk_id);
|
||||
}
|
||||
Err(error) => {
|
||||
retry_failed.insert(set_disk_id.clone());
|
||||
conflicted_recovery_sets.insert(set_disk_id);
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
error = %error,
|
||||
"Replacement recovery retry failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut blocked = replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned");
|
||||
unblock_replacement_recovery_sets_after_validation(&mut blocked, retry_succeeded, &retry_failed);
|
||||
}
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
let runtime_state = disk.runtime_state();
|
||||
let set_disk_id =
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
|
||||
if set_disk_id.as_ref().is_some_and(|set_disk_id| {
|
||||
replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.contains(set_disk_id)
|
||||
}) {
|
||||
skipped_invalid_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
set_disk_id = set_disk_id.as_deref().unwrap_or_default(),
|
||||
disk_state = "replacement_recovery_blocked",
|
||||
"Heal auto-scan replacement deferred because durable recovery is blocked"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// detect unformatted disk via get_disk_id()
|
||||
match disk.get_disk_id().await {
|
||||
Err(DiskError::UnformattedDisk) => {
|
||||
if !super::super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks)
|
||||
.await
|
||||
{
|
||||
deferred_replacement_endpoints.insert(endpoint.to_string());
|
||||
skipped_invalid_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
disk_state = "replacement_path_unavailable",
|
||||
"Heal auto-scan replacement deferred"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let Some(set_disk_id) = set_disk_id else {
|
||||
skipped_invalid_count += 1;
|
||||
continue;
|
||||
};
|
||||
candidate_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
disk_state = "unformatted",
|
||||
"Heal auto-scan candidate detected"
|
||||
);
|
||||
endpoints.entry(set_disk_id).or_default().push(endpoint);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
disk_state = "check_failed",
|
||||
error = ?e,
|
||||
"Heal auto-scan disk inspection failed"
|
||||
);
|
||||
}
|
||||
Ok(_) => {
|
||||
if runtime_state.as_str() == "returning" && let Some(set_disk_id) = set_disk_id {
|
||||
candidate_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
set_disk_id,
|
||||
disk_state = "returning",
|
||||
"Heal auto-scan returning disk candidate detected"
|
||||
);
|
||||
endpoints.entry(set_disk_id).or_default().push(endpoint);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Once formatting succeeds a replacement is no longer
|
||||
// discoverable as UnformattedDisk. Re-admit exactly one
|
||||
// incomplete durable generation per set after bounded
|
||||
// scheduler retries are exhausted, or re-admit its
|
||||
// verified terminal cleanup. Multiple generations are a
|
||||
// durable conflict: leave every marker/state intact and
|
||||
// require reconciliation rather than choosing one.
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
let disk_set_disk_id =
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
|
||||
let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await {
|
||||
Ok(task_ids) => task_ids,
|
||||
Err(error) => {
|
||||
let endpoint_string = endpoint.to_string();
|
||||
if replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
&error,
|
||||
&endpoint_string,
|
||||
&deferred_replacement_endpoints,
|
||||
) {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
disk_state = "replacement_path_unavailable",
|
||||
result = "recovery_records_unavailable",
|
||||
"Replacement recovery discovery skipped for deferred replacement"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
error = %error,
|
||||
"Replacement recovery discovery failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for task_id in replacement_task_ids {
|
||||
let resume_manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await {
|
||||
Ok(resume_manager) => resume_manager,
|
||||
Err(error) => {
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
task_id,
|
||||
error = %error,
|
||||
"Replacement recovery intent load failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let state = resume_manager.get_state().await;
|
||||
if !durable_replacement_recovery_is_due(&state, &task_id) {
|
||||
continue;
|
||||
}
|
||||
if !matches!(state.replacement_phase, ReplacementPhase::CleanupPending) {
|
||||
let Ok(identities) = storage.replacement_target_identities(&state.replacement_targets).await else {
|
||||
continue;
|
||||
};
|
||||
if identities != state.replacement_target_identities {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let targets = state
|
||||
.replacement_targets
|
||||
.iter()
|
||||
.filter_map(|target| {
|
||||
local_endpoints
|
||||
.iter()
|
||||
.find(|endpoint| endpoint.to_string() == *target)
|
||||
.cloned()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if targets.len() != state.replacement_targets.len() {
|
||||
continue;
|
||||
}
|
||||
let Some(set_disk_id) = crate::heal::utils::format_set_disk_id_from_i32(
|
||||
targets[0].pool_idx,
|
||||
targets[0].set_idx,
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
if targets.iter().any(|target| {
|
||||
crate::heal::utils::format_set_disk_id_from_i32(target.pool_idx, target.set_idx)
|
||||
.as_deref()
|
||||
!= Some(set_disk_id.as_str())
|
||||
}) {
|
||||
continue;
|
||||
}
|
||||
let resume_endpoint = disk.endpoint().to_string();
|
||||
match durable_recoveries.get(&set_disk_id) {
|
||||
Some((existing_task_id, _, _, existing_anchor))
|
||||
if existing_task_id != &task_id || existing_anchor != &resume_endpoint => {
|
||||
replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.insert(set_disk_id.clone());
|
||||
conflicted_recovery_sets.insert(set_disk_id);
|
||||
}
|
||||
Some(_) => {}
|
||||
None => {
|
||||
durable_recoveries.insert(
|
||||
set_disk_id,
|
||||
(task_id, targets, state.replacement_buckets, resume_endpoint),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for set_disk_id in &conflicted_recovery_sets {
|
||||
durable_recoveries.remove(set_disk_id);
|
||||
endpoints.remove(set_disk_id);
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
set_disk_id,
|
||||
result = "durable_generation_conflict",
|
||||
"Replacement recovery deferred because multiple durable generations exist"
|
||||
);
|
||||
}
|
||||
|
||||
for (set_disk_id, (_, targets, _, _)) in &durable_recoveries {
|
||||
let expected = targets.iter().map(ToString::to_string).collect::<HashSet<_>>();
|
||||
let observed = endpoints
|
||||
.get(set_disk_id)
|
||||
.map(|endpoints| endpoints.iter().map(ToString::to_string).collect::<HashSet<_>>())
|
||||
.unwrap_or_default();
|
||||
if !observed.is_subset(&expected) {
|
||||
replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.insert(set_disk_id.clone());
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
continue;
|
||||
}
|
||||
endpoints.entry(set_disk_id.clone()).or_default().extend(targets.clone());
|
||||
}
|
||||
for set_disk_id in &conflicted_recovery_sets {
|
||||
durable_recoveries.remove(set_disk_id);
|
||||
endpoints.remove(set_disk_id);
|
||||
}
|
||||
|
||||
for target_endpoints in endpoints.values_mut() {
|
||||
target_endpoints.sort_by_key(ToString::to_string);
|
||||
target_endpoints.dedup_by(|left, right| left.to_string() == right.to_string());
|
||||
}
|
||||
|
||||
if endpoints.is_empty() {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "idle",
|
||||
"Heal auto disk scanner idle"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Admit one set task with every ready replacement target. Queue deduplication is
|
||||
// set-scoped, so admitting endpoints independently would silently drop later targets.
|
||||
for (set_disk_id, endpoints) in endpoints {
|
||||
if replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.contains(&set_disk_id)
|
||||
{
|
||||
skipped_invalid_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
set_disk_id,
|
||||
result = "replacement_recovery_blocked",
|
||||
"Heal auto-scan replacement admission deferred because durable recovery is blocked"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// skip if already queued or healing
|
||||
// Use consistent lock order: queue first, then active_heals to avoid deadlock
|
||||
let mut skip = false;
|
||||
{
|
||||
let queue = heal_queue.lock().await;
|
||||
if queue.contains_erasure_set(&set_disk_id) {
|
||||
skip = true;
|
||||
}
|
||||
}
|
||||
if !skip {
|
||||
let active = active_heals.lock().await;
|
||||
if active.values().any(|task| {
|
||||
matches!(
|
||||
&task.heal_type,
|
||||
crate::heal::task::HealType::ErasureSet { set_disk_id: active_id, .. }
|
||||
if active_id == &set_disk_id
|
||||
)
|
||||
}) {
|
||||
skip = true;
|
||||
}
|
||||
}
|
||||
|
||||
if skip {
|
||||
skipped_duplicate_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint_count = endpoints.len(),
|
||||
set_disk_id,
|
||||
result = "skipped_duplicate",
|
||||
"Heal auto-scan duplicate skipped"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// enqueue erasure set heal request for all ready replacements in this set
|
||||
let recovery = durable_recoveries.remove(&set_disk_id);
|
||||
let mut req = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: recovery
|
||||
.as_ref()
|
||||
.map(|(_, _, buckets, _)| buckets.clone())
|
||||
.unwrap_or_default(),
|
||||
set_disk_id: set_disk_id.clone(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: endpoints
|
||||
.first()
|
||||
.and_then(|endpoint| usize::try_from(endpoint.pool_idx).ok()),
|
||||
set_index: endpoints
|
||||
.first()
|
||||
.and_then(|endpoint| usize::try_from(endpoint.set_idx).ok()),
|
||||
timeout: None,
|
||||
..HealOptions::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
let recovery_anchor = recovery.as_ref().map(|(_, _, _, anchor)| anchor.clone());
|
||||
if let Some((task_id, _, _, _)) = recovery {
|
||||
req.id = task_id;
|
||||
}
|
||||
req.source = HealRequestSource::AutoHeal;
|
||||
req.heal_endpoints = endpoints.iter().map(ToString::to_string).collect();
|
||||
let request_id = req.id.clone();
|
||||
let endpoint_count = req.heal_endpoints.len();
|
||||
let config = config.read().await;
|
||||
let mut queue = heal_queue.lock().await;
|
||||
let admission_decision = Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan");
|
||||
let admission = admission_decision.result;
|
||||
let should_notify =
|
||||
matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
|
||||
if matches!(admission, HealAdmissionResult::Accepted)
|
||||
&& let Some(anchor) = recovery_anchor
|
||||
{
|
||||
replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.insert(request_id, anchor);
|
||||
}
|
||||
drop(queue);
|
||||
drop(config);
|
||||
if let Some(displaced_task_id) = admission_decision.displaced_task_id {
|
||||
remove_task_aliases_for_task(&task_aliases, &displaced_task_id).await;
|
||||
lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id);
|
||||
}
|
||||
if matches!(admission, HealAdmissionResult::Accepted) {
|
||||
if should_notify {
|
||||
notify.notify_one();
|
||||
}
|
||||
enqueued_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint_count,
|
||||
set_disk_id,
|
||||
bucket_count = 0,
|
||||
result = "enqueued",
|
||||
"Heal auto-scan task enqueued"
|
||||
);
|
||||
} else {
|
||||
if matches!(admission, HealAdmissionResult::Merged) {
|
||||
skipped_duplicate_count += 1;
|
||||
} else {
|
||||
not_enqueued_count += 1;
|
||||
}
|
||||
if matches!(admission, HealAdmissionResult::Full) {
|
||||
full_count += 1;
|
||||
}
|
||||
if matches!(admission, HealAdmissionResult::Dropped(_)) {
|
||||
dropped_count += 1;
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint_count,
|
||||
set_disk_id,
|
||||
bucket_count = 0,
|
||||
admission = admission.result_label(),
|
||||
reason = admission.reason_label(),
|
||||
result = "not_enqueued",
|
||||
"Heal auto-scan task not enqueued"
|
||||
);
|
||||
}
|
||||
}
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "cycle_completed",
|
||||
candidate_count,
|
||||
enqueued_count,
|
||||
not_enqueued_count,
|
||||
dropped_count,
|
||||
full_count,
|
||||
skipped_duplicate_count,
|
||||
skipped_invalid_count,
|
||||
"Heal auto-scan cycle completed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,458 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// The priority heal queue and its per-key dedup index.
|
||||
use super::*;
|
||||
|
||||
/// Per-key bookkeeping for the queued-request dedup index: how many queued
|
||||
/// requests hold the key, and the id of the first request that opened it —
|
||||
/// the O(1) stand-in for the former heap scan when a merge receipt needs to
|
||||
/// name a queued representative.
|
||||
#[derive(Debug)]
|
||||
pub(super) struct DedupKeyEntry {
|
||||
pub(super) refcount: usize,
|
||||
pub(super) representative_request_id: String,
|
||||
}
|
||||
|
||||
/// Priority queue wrapper for heal requests
|
||||
/// Uses BinaryHeap for priority-based ordering while maintaining FIFO for same-priority items
|
||||
#[derive(Debug)]
|
||||
pub(super) struct PriorityHealQueue {
|
||||
/// Heap of (priority, sequence, request) tuples
|
||||
pub(super) heap: BinaryHeap<PriorityQueueItem>,
|
||||
/// Sequence counter for FIFO ordering within same priority
|
||||
pub(super) sequence: u64,
|
||||
/// Deduplication index for queued requests
|
||||
pub(super) dedup_keys: HashMap<String, DedupKeyEntry>,
|
||||
}
|
||||
|
||||
/// Wrapper for heap items to implement proper ordering
|
||||
#[derive(Debug)]
|
||||
pub(super) struct PriorityQueueItem {
|
||||
pub(super) priority: HealPriority,
|
||||
pub(super) sequence: u64,
|
||||
pub(super) dedup_key: String,
|
||||
pub(super) request: HealRequest,
|
||||
}
|
||||
|
||||
impl Eq for PriorityQueueItem {}
|
||||
|
||||
impl PartialEq for PriorityQueueItem {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.priority == other.priority && self.sequence == other.sequence
|
||||
}
|
||||
}
|
||||
|
||||
impl Ord for PriorityQueueItem {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
// First compare by priority (higher priority first)
|
||||
match self.priority.cmp(&other.priority) {
|
||||
std::cmp::Ordering::Equal => {
|
||||
// If priorities are equal, use sequence for FIFO (lower sequence first)
|
||||
other.sequence.cmp(&self.sequence)
|
||||
}
|
||||
ordering => ordering,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for PriorityQueueItem {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum QueuePushOutcome {
|
||||
Accepted,
|
||||
Merged,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) struct ForegroundPressure {
|
||||
pub(super) class: WorkloadClass,
|
||||
pub(super) usage_pct: usize,
|
||||
pub(super) threshold_pct: usize,
|
||||
}
|
||||
|
||||
impl ForegroundPressure {
|
||||
pub(super) const fn reason(self) -> &'static str {
|
||||
match self.class {
|
||||
WorkloadClass::ForegroundRead => "foreground_read_pressure",
|
||||
WorkloadClass::ForegroundWrite => "foreground_write_pressure",
|
||||
_ => "foreground_pressure",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct CompletedHealStatus {
|
||||
pub(super) heal_type: HealType,
|
||||
pub(super) status: HealTaskStatus,
|
||||
pub(super) result_items_truncated: bool,
|
||||
pub(super) completed_at: SystemTime,
|
||||
/// Sequence-stamped retained window, archived with the completion so
|
||||
/// incremental consumers keep their cursor across the transition (HS-06).
|
||||
/// The un-stamped legacy view is derived from it on demand.
|
||||
pub(super) seqed_items: Vec<(u64, HealResultItem)>,
|
||||
pub(super) next_seq: u64,
|
||||
pub(super) min_seq: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct HealTaskAlias {
|
||||
pub(super) task_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct RetryingHeal {
|
||||
pub(super) request: HealRequest,
|
||||
pub(super) error: String,
|
||||
pub(super) cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl PriorityHealQueue {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
heap: BinaryHeap::new(),
|
||||
sequence: 0,
|
||||
dedup_keys: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn len(&self) -> usize {
|
||||
self.heap.len()
|
||||
}
|
||||
|
||||
pub(super) fn pop_next(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn is_empty(&self) -> bool {
|
||||
self.heap.is_empty()
|
||||
}
|
||||
|
||||
pub(super) fn push(&mut self, request: HealRequest) -> QueuePushOutcome {
|
||||
let key = Self::make_dedup_key(&request);
|
||||
|
||||
// Check for duplicates unless the caller explicitly forces admission.
|
||||
if self.dedup_keys.contains_key(&key) && !request.force_start {
|
||||
return QueuePushOutcome::Merged;
|
||||
}
|
||||
// Track dedup keys for both normal and forced requests so queued forced work
|
||||
// also reserves the dedup key for later non-forced duplicates. The first
|
||||
// request that opens the key becomes the named representative for merge
|
||||
// receipts (taken before `request` moves into the heap).
|
||||
self.dedup_keys
|
||||
.entry(key.clone())
|
||||
.or_insert_with(|| DedupKeyEntry {
|
||||
refcount: 0,
|
||||
representative_request_id: request.id.clone(),
|
||||
})
|
||||
.refcount += 1;
|
||||
self.sequence += 1;
|
||||
self.heap.push(PriorityQueueItem {
|
||||
priority: request.priority,
|
||||
sequence: self.sequence,
|
||||
dedup_key: key,
|
||||
request,
|
||||
});
|
||||
QueuePushOutcome::Accepted
|
||||
}
|
||||
|
||||
pub(super) fn can_displace_lower_priority(&self, priority: HealPriority) -> bool {
|
||||
self.heap.iter().any(|item| item.priority < priority)
|
||||
}
|
||||
|
||||
pub(super) fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option<HealRequest> {
|
||||
let mut retained = BinaryHeap::new();
|
||||
let mut displaced: Option<PriorityQueueItem> = None;
|
||||
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if item.priority < request.priority {
|
||||
let should_displace = displaced
|
||||
.as_ref()
|
||||
.map(|current| {
|
||||
item.priority < current.priority
|
||||
|| (item.priority == current.priority && item.sequence > current.sequence)
|
||||
})
|
||||
.unwrap_or(true);
|
||||
if should_displace {
|
||||
if let Some(current) = displaced.replace(item) {
|
||||
retained.push(current);
|
||||
}
|
||||
} else {
|
||||
retained.push(item);
|
||||
}
|
||||
} else {
|
||||
retained.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
self.heap = retained;
|
||||
|
||||
let displaced = displaced.map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.refresh_dedup_representative(&item.dedup_key);
|
||||
item.request
|
||||
});
|
||||
|
||||
if displaced.is_some() {
|
||||
// The enqueue side effect must run in ALL builds. Do NOT fold `self.push(request)`
|
||||
// into `debug_assert_eq!` — in release builds (`debug_assertions` off) the whole
|
||||
// macro, including its argument expression, is compiled out, which would silently
|
||||
// drop the new high-priority request after having already evicted a queued item.
|
||||
let outcome = self.push(request);
|
||||
debug_assert_eq!(outcome, QueuePushOutcome::Accepted);
|
||||
}
|
||||
|
||||
displaced
|
||||
}
|
||||
|
||||
/// Get statistics about queue contents by priority
|
||||
pub(super) fn get_priority_stats(&self) -> HashMap<HealPriority, usize> {
|
||||
let mut stats = HashMap::new();
|
||||
for item in &self.heap {
|
||||
*stats.entry(item.priority).or_insert(0) += 1;
|
||||
}
|
||||
stats
|
||||
}
|
||||
|
||||
pub(super) fn operation_counts(&self) -> (HealPriorityCounts, HealSourceCounts) {
|
||||
let mut priority = HealPriorityCounts::default();
|
||||
let mut source = HealSourceCounts::default();
|
||||
for item in &self.heap {
|
||||
priority.increment(item.request.priority);
|
||||
source.increment(item.request.source);
|
||||
}
|
||||
(priority, source)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn pop(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn pop_runnable_with_skips<F, G>(&mut self, can_run: F, skip_label: G) -> (Option<HealRequest>, Vec<String>)
|
||||
where
|
||||
F: Fn(&HealRequest) -> bool,
|
||||
G: Fn(&HealRequest) -> Option<String>,
|
||||
{
|
||||
let mut deferred = Vec::new();
|
||||
let mut selected = None;
|
||||
let mut skipped = Vec::new();
|
||||
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if can_run(&item.request) {
|
||||
selected = Some(item);
|
||||
break;
|
||||
}
|
||||
if let Some(label) = skip_label(&item.request) {
|
||||
skipped.push(label);
|
||||
}
|
||||
deferred.push(item);
|
||||
}
|
||||
|
||||
self.restore_deferred_items(deferred);
|
||||
|
||||
(
|
||||
selected.map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
item.request
|
||||
}),
|
||||
skipped,
|
||||
)
|
||||
}
|
||||
|
||||
fn restore_deferred_items(&mut self, deferred: Vec<PriorityQueueItem>) {
|
||||
if deferred.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
if deferred.len() > self.heap.len() / 2 {
|
||||
let mut items = std::mem::take(&mut self.heap).into_vec();
|
||||
items.reserve(deferred.len());
|
||||
items.extend(deferred);
|
||||
self.heap = BinaryHeap::from(items);
|
||||
} else {
|
||||
for item in deferred {
|
||||
self.heap.push(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a deduplication key from a heal request
|
||||
pub(super) fn make_dedup_key(request: &HealRequest) -> String {
|
||||
Self::make_dedup_key_for_type(&request.heal_type)
|
||||
}
|
||||
|
||||
pub(super) fn make_dedup_key_for_type(heal_type: &HealType) -> String {
|
||||
match heal_type {
|
||||
HealType::Cluster => "cluster".to_string(),
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
format!("object:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or(""))
|
||||
}
|
||||
HealType::Bucket { bucket } => {
|
||||
format!("bucket:{bucket}")
|
||||
}
|
||||
HealType::Prefix { bucket, prefix } => {
|
||||
format!("prefix:{bucket}/{prefix}")
|
||||
}
|
||||
HealType::ErasureSet { set_disk_id, .. } => {
|
||||
format!("erasure_set:{set_disk_id}")
|
||||
}
|
||||
HealType::Metadata { bucket, object } => {
|
||||
format!("metadata:{bucket}:{object}")
|
||||
}
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
format!("ecdecode:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or(""))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap<String, DedupKeyEntry>, key: &str) {
|
||||
if let Some(entry) = dedup_keys.get_mut(key) {
|
||||
if entry.refcount <= 1 {
|
||||
dedup_keys.remove(key);
|
||||
} else {
|
||||
entry.refcount -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Check if an erasure set heal request for a specific set_disk_id exists
|
||||
pub(super) fn contains_erasure_set(&self, set_disk_id: &str) -> bool {
|
||||
let key = format!("erasure_set:{set_disk_id}");
|
||||
self.dedup_keys.contains_key(&key)
|
||||
}
|
||||
|
||||
/// Iterate queued requests (used by the admin overlap check).
|
||||
pub(super) fn requests(&self) -> impl Iterator<Item = &HealRequest> {
|
||||
self.heap.iter().map(|item| &item.request)
|
||||
}
|
||||
|
||||
pub(super) fn contains_request_id(&self, request_id: &str) -> bool {
|
||||
self.heap.iter().any(|item| item.request.id == request_id)
|
||||
}
|
||||
|
||||
pub(super) fn contains_request_id_matching_path(&self, request_id: &str, heal_path: &str) -> bool {
|
||||
self.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == request_id && heal_type_matches_path(&item.request.heal_type, heal_path))
|
||||
}
|
||||
|
||||
pub(super) fn queued_request_id_for_dedup_key(&self, key: &str) -> Option<&str> {
|
||||
self.dedup_keys.get(key).map(|entry| entry.representative_request_id.as_str())
|
||||
}
|
||||
|
||||
/// Re-elect the representative for `key` from the queue entries holding
|
||||
/// it. Needed after a holder leaves the queue *without* becoming active
|
||||
/// (canceled by id, or displaced): the former opener may be the request
|
||||
/// that just left, and a merge receipt must never name an id that
|
||||
/// resolves nowhere. The scheduler pop path does not need this — the
|
||||
/// popped request surfaces in `active_heals` under the same id and the
|
||||
/// duplicate pre-check consults active heals before the queue. No-op for
|
||||
/// released keys; the survivor scan only runs when a key still has
|
||||
/// holders, which under forced duplicates is the rare admin path.
|
||||
pub(super) fn refresh_dedup_representative(&mut self, key: &str) {
|
||||
if !self.dedup_keys.contains_key(key) {
|
||||
return;
|
||||
}
|
||||
if let Some(id) = self
|
||||
.heap
|
||||
.iter()
|
||||
.find(|item| item.dedup_key == key)
|
||||
.map(|item| item.request.id.clone())
|
||||
&& let Some(entry) = self.dedup_keys.get_mut(key)
|
||||
{
|
||||
entry.representative_request_id = id;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn contains_matching<F>(&self, mut matches: F) -> bool
|
||||
where
|
||||
F: FnMut(&HealRequest) -> bool,
|
||||
{
|
||||
self.heap.iter().any(|item| matches(&item.request))
|
||||
}
|
||||
|
||||
pub(super) fn remove_request_id(&mut self, request_id: &str) -> Option<HealRequest> {
|
||||
let mut retained = BinaryHeap::new();
|
||||
let mut removed = None;
|
||||
let mut affected_key = None;
|
||||
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if removed.is_none() && item.request.id == request_id {
|
||||
let key = item.dedup_key.clone();
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
|
||||
affected_key = Some(key);
|
||||
removed = Some(item.request);
|
||||
} else {
|
||||
retained.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
self.heap = retained;
|
||||
if let Some(key) = affected_key.as_deref() {
|
||||
self.refresh_dedup_representative(key);
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
pub(super) fn remove_matching<F>(&mut self, mut should_remove: F) -> Vec<HealRequest>
|
||||
where
|
||||
F: FnMut(&HealRequest) -> bool,
|
||||
{
|
||||
let mut retained = BinaryHeap::new();
|
||||
let mut removed = Vec::new();
|
||||
let mut affected_keys = Vec::new();
|
||||
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if should_remove(&item.request) {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
affected_keys.push(item.dedup_key);
|
||||
removed.push(item.request);
|
||||
} else {
|
||||
retained.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
self.heap = retained;
|
||||
for key in &affected_keys {
|
||||
self.refresh_dedup_representative(key);
|
||||
}
|
||||
removed
|
||||
}
|
||||
}
|
||||
|
||||
impl RetryingHeal {
|
||||
pub(super) fn status(&self) -> HealTaskStatus {
|
||||
HealTaskStatus::Retrying {
|
||||
error: self.error.clone(),
|
||||
retry_attempt: self.request.retry_attempts,
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,390 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// Unclean-shutdown recovery: durable replacement-intent discovery and healing-marker rewrite.
|
||||
use super::*;
|
||||
|
||||
pub(super) fn durable_replacement_recovery_is_due(state: &ResumeState, task_id: &str) -> bool {
|
||||
state.replacement_generation.as_deref() == Some(task_id)
|
||||
&& !state.replacement_targets.is_empty()
|
||||
&& ((!state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding)
|
||||
&& state.retry_count >= state.max_retries)
|
||||
|| (state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending)))
|
||||
}
|
||||
|
||||
pub(super) fn replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
error: &Error,
|
||||
endpoint: &str,
|
||||
deferred_replacement_endpoints: &HashSet<String>,
|
||||
) -> bool {
|
||||
matches!(error, Error::Disk(DiskError::UnformattedDisk)) && deferred_replacement_endpoints.contains(endpoint)
|
||||
}
|
||||
|
||||
pub(super) fn unblock_replacement_recovery_sets_after_validation(
|
||||
blocked_sets: &mut HashSet<String>,
|
||||
retry_succeeded: HashSet<String>,
|
||||
retry_failed: &HashSet<String>,
|
||||
) {
|
||||
for set_disk_id in retry_succeeded {
|
||||
if !retry_failed.contains(&set_disk_id) {
|
||||
blocked_sets.remove(&set_disk_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HealManager {
|
||||
/// Detect whether the previous run ended without a clean shutdown and, if so,
|
||||
/// enqueue a full erasure-set heal for every local set. Also (re)writes the
|
||||
/// marker for the current run; [`super::super::clear_unclean_shutdown_markers`]
|
||||
/// removes it again during graceful shutdown. Best-effort: failures only log.
|
||||
pub(super) async fn process_unclean_shutdown(&self) {
|
||||
let mut unclean = false;
|
||||
let mut set_disk_ids = HashSet::new();
|
||||
let mut replacement_intents = HashMap::<String, (String, Vec<String>, Vec<String>, String)>::new();
|
||||
let mut replacement_restarts = HashMap::<String, (String, Vec<String>)>::new();
|
||||
let mut conflicted_replacement_sets = HashSet::new();
|
||||
|
||||
{
|
||||
let local_disks = {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
match disk
|
||||
.read_all(super::super::RUSTFS_META_BUCKET, super::super::UNCLEAN_SHUTDOWN_MARKER_PATH)
|
||||
.await
|
||||
{
|
||||
Ok(_) => unclean = true,
|
||||
Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = ?err,
|
||||
"Unclean-shutdown marker check failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let marker = SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs().to_string())
|
||||
.unwrap_or_default();
|
||||
if let Err(err) = disk
|
||||
.write_all(
|
||||
super::super::RUSTFS_META_BUCKET,
|
||||
super::super::UNCLEAN_SHUTDOWN_MARKER_PATH,
|
||||
marker.into(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = ?err,
|
||||
"Unclean-shutdown marker write failed"
|
||||
);
|
||||
}
|
||||
|
||||
let disk_set_disk_id = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
set_disk_ids.insert(set_disk_id.clone());
|
||||
}
|
||||
|
||||
// Legacy flat records are inspected only while starting. The
|
||||
// periodic scanner lists the dedicated replacement directory.
|
||||
if let Err(error) = ResumeUtils::migrate_legacy_replacement_records(disk).await {
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
self.block_replacement_recovery_set(set_disk_id);
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = %error,
|
||||
"Legacy replacement recovery migration failed"
|
||||
);
|
||||
}
|
||||
let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await {
|
||||
Ok(task_ids) => task_ids,
|
||||
Err(error) => {
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
self.block_replacement_recovery_set(set_disk_id);
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = %error,
|
||||
"Replacement recovery discovery failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for task_id in replacement_task_ids {
|
||||
let manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await {
|
||||
Ok(manager) => manager,
|
||||
Err(error) => {
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
self.block_replacement_recovery_set(set_disk_id);
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
task_id,
|
||||
error = %error,
|
||||
"Replacement recovery intent load failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let state = manager.get_state().await;
|
||||
let active_replacement = !state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding);
|
||||
let verified_replacement = state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending);
|
||||
if (active_replacement || verified_replacement)
|
||||
&& state.replacement_generation.as_deref() == Some(task_id.as_str())
|
||||
&& !state.replacement_targets.is_empty()
|
||||
{
|
||||
if matches!(state.replacement_phase, ReplacementPhase::CleanupPending) {
|
||||
replacement_intents.entry(task_id).or_insert((
|
||||
state.set_disk_id,
|
||||
state.replacement_targets,
|
||||
state.replacement_buckets,
|
||||
endpoint.to_string(),
|
||||
));
|
||||
continue;
|
||||
}
|
||||
match self.storage.replacement_target_identities(&state.replacement_targets).await {
|
||||
Ok(identities) if identities == state.replacement_target_identities => {
|
||||
let resume_endpoint = endpoint.to_string();
|
||||
match replacement_intents.entry(task_id) {
|
||||
std::collections::hash_map::Entry::Vacant(entry) => {
|
||||
entry.insert((
|
||||
state.set_disk_id,
|
||||
state.replacement_targets,
|
||||
state.replacement_buckets,
|
||||
resume_endpoint,
|
||||
));
|
||||
}
|
||||
std::collections::hash_map::Entry::Occupied(entry) => {
|
||||
let (existing_set_disk_id, existing_targets, existing_buckets, existing_anchor) =
|
||||
entry.get();
|
||||
if existing_set_disk_id != &state.set_disk_id
|
||||
|| existing_targets != &state.replacement_targets
|
||||
|| existing_buckets != &state.replacement_buckets
|
||||
|| existing_anchor != &resume_endpoint
|
||||
{
|
||||
conflicted_replacement_sets.insert(state.set_disk_id.clone());
|
||||
self.block_replacement_recovery_set(&state.set_disk_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(_) => {
|
||||
if manager.abandon_replacement_intent().await.is_ok() {
|
||||
replacement_restarts
|
||||
.entry(task_id)
|
||||
.or_insert((state.set_disk_id, state.replacement_targets));
|
||||
}
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !unclean && replacement_intents.is_empty() && replacement_restarts.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut recovery_by_set = HashMap::<String, Vec<(Option<String>, Vec<String>, Vec<String>, Option<String>)>>::new();
|
||||
for (task_id, (set_disk_id, heal_endpoints, buckets, resume_endpoint)) in replacement_intents {
|
||||
recovery_by_set
|
||||
.entry(set_disk_id)
|
||||
.or_default()
|
||||
.push((Some(task_id), heal_endpoints, buckets, Some(resume_endpoint)));
|
||||
}
|
||||
for (_abandoned_task_id, (set_disk_id, heal_endpoints)) in replacement_restarts {
|
||||
recovery_by_set
|
||||
.entry(set_disk_id)
|
||||
.or_default()
|
||||
.push((None, heal_endpoints, Vec::new(), None));
|
||||
}
|
||||
|
||||
for (set_disk_id, mut recoveries) in recovery_by_set {
|
||||
let Ok((pool_index, set_index)) = crate::heal::utils::parse_set_disk_id(&set_disk_id) else {
|
||||
continue;
|
||||
};
|
||||
if self.replacement_recovery_set_is_blocked(&set_disk_id) {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
set_disk_id,
|
||||
recovery_count = recoveries.len(),
|
||||
"Replacement recovery deferred because durable recovery validation is blocked"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if conflicted_replacement_sets.contains(&set_disk_id) || recoveries.len() != 1 {
|
||||
self.block_replacement_recovery_set(&set_disk_id);
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
set_disk_id,
|
||||
recovery_count = recoveries.len(),
|
||||
"Replacement recovery deferred because multiple durable generations exist"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let reuse_single_generation = recoveries.len() == 1 && recoveries[0].0.is_some();
|
||||
let mut heal_endpoints = recoveries
|
||||
.iter_mut()
|
||||
.flat_map(|(_, targets, _, _)| std::mem::take(targets))
|
||||
.collect::<Vec<_>>();
|
||||
heal_endpoints.sort_unstable();
|
||||
heal_endpoints.dedup();
|
||||
let buckets = if reuse_single_generation {
|
||||
std::mem::take(&mut recoveries[0].2)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let mut req = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets,
|
||||
set_disk_id: set_disk_id.clone(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(pool_index),
|
||||
set_index: Some(set_index),
|
||||
timeout: None,
|
||||
..HealOptions::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
if reuse_single_generation && let Some(task_id) = recoveries[0].0.take() {
|
||||
req.id = task_id;
|
||||
}
|
||||
let recovery_anchor = reuse_single_generation.then(|| recoveries[0].3.take()).flatten();
|
||||
req.source = HealRequestSource::AutoHeal;
|
||||
req.heal_endpoints = heal_endpoints;
|
||||
let request_id = req.id.clone();
|
||||
if let Some(anchor) = &recovery_anchor {
|
||||
self.replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.insert(request_id.clone(), anchor.clone());
|
||||
}
|
||||
match self.submit_heal_request(req).await {
|
||||
Ok(HealAdmissionResult::Accepted) => {}
|
||||
Ok(_) => {
|
||||
self.replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.remove(&request_id);
|
||||
}
|
||||
Err(err) => {
|
||||
self.replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.remove(&request_id);
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
set_disk_id,
|
||||
error = %err,
|
||||
"Replacement recovery enqueue failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !unclean || set_disk_ids.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
set_count = set_disk_ids.len(),
|
||||
"Unclean shutdown detected; scheduling erasure-set heal for local sets"
|
||||
);
|
||||
|
||||
let buckets = match self.storage.list_buckets().await {
|
||||
Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::<Vec<String>>(),
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
error = %err,
|
||||
"Unclean-shutdown heal skipped: bucket listing failed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
for set_disk_id in set_disk_ids {
|
||||
let mut req = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: buckets.clone(),
|
||||
set_disk_id: set_disk_id.clone(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..HealOptions::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
req.source = HealRequestSource::AutoHeal;
|
||||
if let Err(err) = self.submit_heal_request(req).await {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
set_disk_id,
|
||||
error = %err,
|
||||
"Unclean-shutdown heal enqueue failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
pub mod channel;
|
||||
pub mod erasure_healer;
|
||||
pub mod event;
|
||||
pub mod manager;
|
||||
pub mod mrf_queue;
|
||||
pub mod progress;
|
||||
|
||||
@@ -25,12 +25,9 @@
|
||||
//! set, rewritten on a group-commit cadence (every flush interval or flush
|
||||
//! threshold new intents). A rewrite is atomic at the record level only — a
|
||||
//! torn tail simply truncates during replay because every record carries its
|
||||
//! own CRC32. Losing the last flush window (≤500 ms) is acceptable because
|
||||
//! every producer keeps its own safety net: read-repair re-detects on the
|
||||
//! next failing read, and the scanner's corrupt-metadata branch leaves a
|
||||
//! pending-ledger entry behind even when its MRF intent is accepted
|
||||
//! (backlog#1894 axis A), so a lost intent is retried by the ledger rather
|
||||
//! than waiting for the failed-object TTL to re-scan the path.
|
||||
//! own CRC32. Losing the last flush window (≤500 ms) is acceptable: replayed
|
||||
//! duplicates are merged by the manager's dedup key, and read-repair remains
|
||||
//! the safety net.
|
||||
|
||||
use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
|
||||
use crate::heal::manager::HealManager;
|
||||
@@ -279,26 +276,20 @@ async fn read_journal() -> Option<Vec<u8>> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Write the snapshot to every local disk; returns true when at least one
|
||||
/// disk accepted it, so a total write failure keeps the runtime dirty and
|
||||
/// the next tick retries the persist.
|
||||
async fn write_journal(data: &[u8]) -> bool {
|
||||
async fn write_journal(data: &[u8]) {
|
||||
let payload = bytes::Bytes::copy_from_slice(data);
|
||||
let mut any_persisted = false;
|
||||
for disk in journal_disks().await {
|
||||
match disk
|
||||
if let Err(err) = disk
|
||||
.write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone())
|
||||
.await
|
||||
{
|
||||
Ok(()) => any_persisted = true,
|
||||
Err(err) => warn_mrf_journal_write(&err),
|
||||
warn_mrf_journal_write(&err);
|
||||
}
|
||||
}
|
||||
if !data.is_empty() {
|
||||
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64);
|
||||
any_persisted
|
||||
}
|
||||
|
||||
async fn delete_journal() {
|
||||
@@ -356,28 +347,10 @@ pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
|
||||
request
|
||||
}
|
||||
|
||||
async fn submit_mrf_heal_request(manager: &HealManager, intent: &MrfIntent) -> crate::Result<HealAdmissionResult> {
|
||||
let receipt = manager
|
||||
.submit_mrf_heal_request_with_receipt(
|
||||
build_heal_request(intent),
|
||||
intent.bucket.clone(),
|
||||
intent.object.clone(),
|
||||
intent.version_id,
|
||||
)
|
||||
.await?;
|
||||
Ok(receipt.result)
|
||||
}
|
||||
|
||||
struct MrfRuntime {
|
||||
queue: MrfQueue,
|
||||
config: MrfConsumerConfig,
|
||||
new_since_flush: usize,
|
||||
/// True while the in-memory pending set has changed since the last
|
||||
/// journal flush (push, pop, or an attempts bump that alters the encoded
|
||||
/// bytes). Only a dirty state rewrites the snapshot: a steady backlog
|
||||
/// waiting out an admission backoff must not re-fsync every local disk
|
||||
/// twice a second.
|
||||
dirty: bool,
|
||||
/// True while a journal snapshot exists on disk that no longer reflects
|
||||
/// an all-consumed pending set; the next idle tick removes it (MinIO
|
||||
/// deletes its `list.bin` after replay for the same reason).
|
||||
@@ -387,6 +360,11 @@ struct MrfRuntime {
|
||||
}
|
||||
|
||||
impl MrfRuntime {
|
||||
fn record_accept(&mut self) {
|
||||
// Accepted intents leave the pending set; the next flush persists the
|
||||
// smaller snapshot, which is the journal's compaction.
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
for intent in self.queue.intents() {
|
||||
@@ -396,14 +374,8 @@ impl MrfRuntime {
|
||||
}
|
||||
|
||||
async fn flush(&mut self) {
|
||||
let persisted = write_journal(&self.snapshot()).await;
|
||||
write_journal(&self.snapshot()).await;
|
||||
self.new_since_flush = 0;
|
||||
// Keep the dirty flag when every disk write failed: a clean backlog
|
||||
// would otherwise never rewrite, losing the periodic persist retry a
|
||||
// non-empty queue used to provide.
|
||||
if persisted {
|
||||
self.dirty = false;
|
||||
}
|
||||
self.journal_on_disk = true;
|
||||
}
|
||||
|
||||
@@ -417,15 +389,9 @@ impl MrfRuntime {
|
||||
self.backoff_until = None;
|
||||
}
|
||||
while let Some(mut intent) = self.queue.pop_front() {
|
||||
// Leaving the pending set (consumed or re-queued with a bumped
|
||||
// attempts counter) changes the encoded snapshot; mark it dirty
|
||||
// either way.
|
||||
self.dirty = true;
|
||||
match submit_mrf_heal_request(manager, &intent).await {
|
||||
// Accepted intents leave the pending set; the next flush persists the
|
||||
// smaller snapshot. The scanner ledger is cleared later, when the
|
||||
// canonical heal task reaches a successful terminal completion.
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
||||
let request = build_heal_request(&intent);
|
||||
match manager.submit_heal_request(request).await {
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => self.record_accept(),
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts >= MRF_MAX_ATTEMPTS {
|
||||
@@ -527,7 +493,8 @@ async fn replay_into(
|
||||
// stays armed in `queue` for the consumer's retry loop.
|
||||
if backoff_until.is_none() {
|
||||
while let Some(mut intent) = queue.pop_front() {
|
||||
match submit_mrf_heal_request(manager, &intent).await {
|
||||
let request = build_heal_request(&intent);
|
||||
match manager.submit_heal_request(request).await {
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
@@ -552,7 +519,6 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes),
|
||||
config: config.clone(),
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: false,
|
||||
backoff_until: None,
|
||||
};
|
||||
@@ -560,10 +526,6 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
// Replay: read the journal, re-arm intents (duplicates are merged by the
|
||||
// manager's dedup key), then drop the file so the next flush starts clean.
|
||||
replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
|
||||
// The replay deleted the journal file; anything still pending (e.g. the
|
||||
// manager was full and backoff armed) must be re-persisted by the next
|
||||
// flush or a crash before it would lose those intents.
|
||||
runtime.dirty = runtime.queue.depth() > 0;
|
||||
|
||||
let mut flush_tick = tokio::time::interval(runtime.config.flush_interval);
|
||||
flush_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
@@ -573,13 +535,8 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
tokio::select! {
|
||||
received = receiver.recv_many(&mut batch, runtime.config.replay_batch) => {
|
||||
if received == 0 {
|
||||
// Channel closed: flush once more unless the snapshot is
|
||||
// provably current AND idle (a dirty or pending state
|
||||
// gets one last persist attempt, matching the shutdown
|
||||
// retry the unconditional flush used to provide).
|
||||
if runtime.dirty || runtime.queue.depth() > 0 {
|
||||
runtime.flush().await;
|
||||
}
|
||||
// Channel closed: flush once more and stop.
|
||||
runtime.flush().await;
|
||||
tracing::info!(
|
||||
target: "rustfs::heal::mrf",
|
||||
"MRF channel closed; consumer stopped after final flush"
|
||||
@@ -587,10 +544,8 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
return;
|
||||
}
|
||||
for intent in batch.drain(..) {
|
||||
if runtime.queue.try_push(intent) {
|
||||
runtime.new_since_flush += 1;
|
||||
runtime.dirty = true;
|
||||
}
|
||||
runtime.queue.try_push(intent);
|
||||
runtime.new_since_flush += 1;
|
||||
}
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
if runtime.new_since_flush >= runtime.config.flush_threshold {
|
||||
@@ -598,26 +553,15 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
}
|
||||
}
|
||||
_ = flush_tick.tick() => {
|
||||
match tick_action(runtime.dirty, runtime.queue.depth(), runtime.journal_on_disk) {
|
||||
TickAction::Flush => {
|
||||
runtime.flush().await;
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
}
|
||||
TickAction::Retry => {
|
||||
// Pending set unchanged since the last flush (a
|
||||
// backlog waiting out an admission backoff): skip the
|
||||
// rewrite but keep dispatching so the retry fires on
|
||||
// time.
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
}
|
||||
TickAction::DeleteJournal => {
|
||||
// All intents consumed: remove the journal so a restart
|
||||
// replays nothing (mirrors MinIO's post-replay unlink).
|
||||
delete_journal().await;
|
||||
runtime.journal_on_disk = false;
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
|
||||
}
|
||||
TickAction::Idle => {}
|
||||
if runtime.new_since_flush > 0 || runtime.queue.depth() > 0 {
|
||||
runtime.flush().await;
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
} else if runtime.journal_on_disk {
|
||||
// All intents consumed: remove the journal so a restart
|
||||
// replays nothing (mirrors MinIO's post-replay unlink).
|
||||
delete_journal().await;
|
||||
runtime.journal_on_disk = false;
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64);
|
||||
}
|
||||
@@ -625,33 +569,6 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
}
|
||||
}
|
||||
|
||||
/// What the periodic tick should do, as a pure function of the runtime state
|
||||
/// so the decision table is unit-testable.
|
||||
enum TickAction {
|
||||
/// The pending set changed since the last snapshot: rewrite it, then
|
||||
/// drain.
|
||||
Flush,
|
||||
/// Pending intents exist but the snapshot is current: only drain (an
|
||||
/// admission backoff may have expired).
|
||||
Retry,
|
||||
/// Nothing pending and a stale journal file remains: remove it.
|
||||
DeleteJournal,
|
||||
/// Quiescent: nothing to do.
|
||||
Idle,
|
||||
}
|
||||
|
||||
fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction {
|
||||
if dirty {
|
||||
TickAction::Flush
|
||||
} else if depth > 0 {
|
||||
TickAction::Retry
|
||||
} else if journal_on_disk {
|
||||
TickAction::DeleteJournal
|
||||
} else {
|
||||
TickAction::Idle
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -669,27 +586,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tick_action_table() {
|
||||
use TickAction::*;
|
||||
|
||||
// Dirty dominates: a changed pending set flushes even when idle
|
||||
// otherwise.
|
||||
assert!(matches!(tick_action(true, 0, false), Flush));
|
||||
assert!(matches!(tick_action(true, 3, true), Flush));
|
||||
|
||||
// Clean backlog: no rewrite, but keep draining so an expired
|
||||
// admission backoff retries on time.
|
||||
assert!(matches!(tick_action(false, 1, false), Retry));
|
||||
assert!(matches!(tick_action(false, 2, true), Retry));
|
||||
|
||||
// Quiescent with a stale journal file on disk: remove it.
|
||||
assert!(matches!(tick_action(false, 0, true), DeleteJournal));
|
||||
|
||||
// Fully quiescent: nothing to do.
|
||||
assert!(matches!(tick_action(false, 0, false), Idle));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_enforces_count_and_byte_ceilings() {
|
||||
let mut queue = MrfQueue::new(2, usize::MAX);
|
||||
|
||||
@@ -218,6 +218,15 @@ impl HealStatistics {
|
||||
self.total_bytes_healed += bytes;
|
||||
self.last_update_time = SystemTime::now();
|
||||
}
|
||||
|
||||
pub fn get_success_rate(&self) -> f64 {
|
||||
let total = self.successful_tasks + self.failed_tasks;
|
||||
if total > 0 {
|
||||
(self.successful_tasks as f64 / total as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -530,4 +539,38 @@ mod tests {
|
||||
assert_eq!(stats.total_objects_healed, 8);
|
||||
assert_eq!(stats.total_bytes_healed, 8192);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_statistics_get_success_rate() {
|
||||
let mut stats = HealStatistics::new();
|
||||
stats.successful_tasks = 8;
|
||||
stats.failed_tasks = 2;
|
||||
|
||||
// success_rate = 8 / (8 + 2) * 100 = 80%
|
||||
assert!((stats.get_success_rate() - 80.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_statistics_get_success_rate_zero_total() {
|
||||
let stats = HealStatistics::new();
|
||||
assert_eq!(stats.get_success_rate(), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_statistics_get_success_rate_all_success() {
|
||||
let mut stats = HealStatistics::new();
|
||||
stats.successful_tasks = 10;
|
||||
stats.failed_tasks = 0;
|
||||
|
||||
assert!((stats.get_success_rate() - 100.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_statistics_get_success_rate_all_failure() {
|
||||
let mut stats = HealStatistics::new();
|
||||
stats.successful_tasks = 0;
|
||||
stats.failed_tasks = 5;
|
||||
|
||||
assert_eq!(stats.get_success_rate(), 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
+3231
-14
File diff suppressed because it is too large
Load Diff
@@ -1,351 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{Error, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
|
||||
use super::{
|
||||
LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_FILE, delete_resume_file, path_to_str,
|
||||
validate_resume_task_id,
|
||||
};
|
||||
|
||||
const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state";
|
||||
|
||||
/// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as
|
||||
/// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable
|
||||
/// to the new `compose_key` identities, so a stale checkpoint is discarded.
|
||||
pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 5;
|
||||
|
||||
/// resume checkpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResumeCheckpoint {
|
||||
/// on-disk schema version; absent in legacy snapshots (defaults to 0)
|
||||
#[serde(default)]
|
||||
pub schema_version: u32,
|
||||
/// task id
|
||||
pub task_id: String,
|
||||
/// checkpoint time
|
||||
pub checkpoint_time: u64,
|
||||
/// current bucket index
|
||||
pub current_bucket_index: usize,
|
||||
/// current object index
|
||||
pub current_object_index: usize,
|
||||
/// Objects healed since the last completed page. HashSet: with the
|
||||
/// previous Vec the per-object `contains` was O(n) and made large-bucket
|
||||
/// heals O(N²). Only spans the in-flight page — completed pages are
|
||||
/// covered by `current_object_index`, so `complete_page` prunes the sets.
|
||||
pub processed_objects: HashSet<String>,
|
||||
/// failed objects
|
||||
pub failed_objects: HashSet<String>,
|
||||
/// skipped objects
|
||||
pub skipped_objects: HashSet<String>,
|
||||
}
|
||||
|
||||
impl ResumeCheckpoint {
|
||||
pub fn new(task_id: String) -> Self {
|
||||
Self {
|
||||
schema_version: CURRENT_CHECKPOINT_SCHEMA,
|
||||
task_id,
|
||||
checkpoint_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(),
|
||||
current_bucket_index: 0,
|
||||
current_object_index: 0,
|
||||
processed_objects: HashSet::new(),
|
||||
failed_objects: HashSet::new(),
|
||||
skipped_objects: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_position(&mut self, bucket_index: usize, object_index: usize) {
|
||||
self.current_bucket_index = bucket_index;
|
||||
self.current_object_index = object_index;
|
||||
self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
}
|
||||
|
||||
pub fn add_processed_object(&mut self, object: String) {
|
||||
self.processed_objects.insert(object);
|
||||
}
|
||||
|
||||
pub fn add_failed_object(&mut self, object: String) {
|
||||
self.failed_objects.insert(object);
|
||||
}
|
||||
|
||||
pub fn add_skipped_object(&mut self, object: String) {
|
||||
self.skipped_objects.insert(object);
|
||||
}
|
||||
|
||||
/// Advance past a fully-processed page: objects below `object_index` are
|
||||
/// skipped by position on resume, so the per-object sets no longer need
|
||||
/// their entries and would otherwise grow with the whole bucket.
|
||||
pub fn complete_page(&mut self, bucket_index: usize, object_index: usize) {
|
||||
self.update_position(bucket_index, object_index);
|
||||
self.processed_objects.clear();
|
||||
self.skipped_objects.clear();
|
||||
self.failed_objects.clear();
|
||||
}
|
||||
|
||||
/// Reset the scan to the start and clear the per-object sets so a retry
|
||||
/// re-scans the whole set.
|
||||
pub fn reset_for_retry(&mut self) {
|
||||
self.update_position(0, 0);
|
||||
self.processed_objects.clear();
|
||||
self.skipped_objects.clear();
|
||||
self.failed_objects.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// resume checkpoint manager
|
||||
pub struct CheckpointManager {
|
||||
disk: DiskStore,
|
||||
checkpoint: Arc<RwLock<ResumeCheckpoint>>,
|
||||
throttle: Mutex<PersistThrottle>,
|
||||
}
|
||||
|
||||
impl CheckpointManager {
|
||||
/// create new checkpoint manager
|
||||
pub async fn new(disk: DiskStore, task_id: String) -> Result<Self> {
|
||||
validate_resume_task_id(&task_id)?;
|
||||
let checkpoint = ResumeCheckpoint::new(task_id);
|
||||
let manager = Self {
|
||||
disk,
|
||||
checkpoint: Arc::new(RwLock::new(checkpoint)),
|
||||
throttle: Mutex::new(PersistThrottle::new()),
|
||||
};
|
||||
|
||||
// save initial checkpoint
|
||||
if let Err(e) = manager.save_checkpoint().await {
|
||||
warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
state = "initial_save_failed",
|
||||
error = %e,
|
||||
"Heal checkpoint persistence failed"
|
||||
);
|
||||
}
|
||||
Ok(manager)
|
||||
}
|
||||
|
||||
/// load checkpoint from disk
|
||||
pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result<Self> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?;
|
||||
let mut checkpoint: ResumeCheckpoint =
|
||||
serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to deserialize checkpoint: {e}"),
|
||||
})?;
|
||||
if checkpoint.task_id != task_id {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Resume checkpoint task id does not match filename".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// A checkpoint from an older schema stored latest-only dedup identities
|
||||
// that are not comparable to the new per-version `compose_key`
|
||||
// identities. Discard the stale sets and position, then stamp the
|
||||
// current schema so the scan restarts cleanly.
|
||||
if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!(
|
||||
"Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
|
||||
checkpoint.schema_version
|
||||
),
|
||||
});
|
||||
}
|
||||
if checkpoint.schema_version < CURRENT_CHECKPOINT_SCHEMA {
|
||||
warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
found_schema = checkpoint.schema_version,
|
||||
current_schema = CURRENT_CHECKPOINT_SCHEMA,
|
||||
state = "schema_discarded",
|
||||
"Heal checkpoint schema is stale; discarding dedup sets and position"
|
||||
);
|
||||
checkpoint.processed_objects.clear();
|
||||
checkpoint.failed_objects.clear();
|
||||
checkpoint.skipped_objects.clear();
|
||||
checkpoint.current_bucket_index = 0;
|
||||
checkpoint.current_object_index = 0;
|
||||
checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
disk,
|
||||
checkpoint: Arc::new(RwLock::new(checkpoint)),
|
||||
throttle: Mutex::new(PersistThrottle::new()),
|
||||
})
|
||||
}
|
||||
|
||||
/// check if checkpoint exists
|
||||
pub async fn has_checkpoint(disk: &DiskStore, task_id: &str) -> bool {
|
||||
if validate_resume_task_id(task_id).is_err() {
|
||||
return false;
|
||||
}
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
match path_to_str(&file_path) {
|
||||
Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(data) => !data.is_empty(),
|
||||
Err(_) => false,
|
||||
},
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// get current checkpoint
|
||||
pub async fn get_checkpoint(&self) -> ResumeCheckpoint {
|
||||
self.checkpoint.read().await.clone()
|
||||
}
|
||||
|
||||
/// update position
|
||||
pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.update_position(bucket_index, object_index);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_throttled().await
|
||||
}
|
||||
|
||||
/// Advance past a completed page and prune the per-object sets, then persist.
|
||||
pub async fn complete_page(&self, bucket_index: usize, object_index: usize) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.complete_page(bucket_index, object_index);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_throttled().await
|
||||
}
|
||||
|
||||
/// Reset the checkpoint to the start of the scan for a retry, then persist.
|
||||
pub async fn reset_for_retry(&self) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.reset_for_retry();
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_throttled().await
|
||||
}
|
||||
|
||||
/// Add a processed object. Called once per healed object, so persistence
|
||||
/// is batched (`PERSIST_EVERY_MUTATIONS` / `PERSIST_INTERVAL`); positions
|
||||
/// and page boundaries still persist unconditionally.
|
||||
pub async fn add_processed_object(&self, object: String) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.add_processed_object(object);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_if_due().await
|
||||
}
|
||||
|
||||
/// add failed object (batched, see `add_processed_object`)
|
||||
pub async fn add_failed_object(&self, object: String) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.add_failed_object(object);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_if_due().await
|
||||
}
|
||||
|
||||
/// add skipped object (batched, see `add_processed_object`)
|
||||
pub async fn add_skipped_object(&self, object: String) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.add_skipped_object(object);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_if_due().await
|
||||
}
|
||||
|
||||
async fn save_checkpoint_if_due(&self) -> Result<()> {
|
||||
let should_save = self.throttle.lock().map(|mut throttle| throttle.record()).unwrap_or(true);
|
||||
if !should_save {
|
||||
return Ok(());
|
||||
}
|
||||
self.save_checkpoint_throttled().await
|
||||
}
|
||||
|
||||
async fn save_checkpoint_throttled(&self) -> Result<()> {
|
||||
let result = self.save_checkpoint().await;
|
||||
if result.is_ok()
|
||||
&& let Ok(mut throttle) = self.throttle.lock()
|
||||
{
|
||||
throttle.mark_saved();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// cleanup checkpoint
|
||||
pub async fn cleanup(&self) -> Result<()> {
|
||||
let task_id = self.checkpoint.read().await.task_id.clone();
|
||||
validate_resume_task_id(&task_id)?;
|
||||
|
||||
let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
delete_resume_file(&self.disk, &checkpoint_file).await?;
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
state = "cleaned",
|
||||
"Heal checkpoint cleaned"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// save checkpoint to disk
|
||||
async fn save_checkpoint(&self) -> Result<()> {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
validate_resume_task_id(&checkpoint.task_id)?;
|
||||
let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize checkpoint: {e}"),
|
||||
})?;
|
||||
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE));
|
||||
|
||||
let path_str = path_to_str(&file_path)?;
|
||||
self.disk
|
||||
.write_all(RUSTFS_META_BUCKET, path_str, checkpoint_data.into())
|
||||
.await
|
||||
.map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save checkpoint: {e}"),
|
||||
})?;
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id = %checkpoint.task_id,
|
||||
state = "saved",
|
||||
"Heal checkpoint persisted"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// read checkpoint file from disk
|
||||
async fn read_checkpoint_file(disk: &DiskStore, task_id: &str) -> Result<Vec<u8>> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
|
||||
let path_str = path_to_str(&file_path)?;
|
||||
disk.read_all(RUSTFS_META_BUCKET, path_str)
|
||||
.await
|
||||
.map(|bytes| bytes.to_vec())
|
||||
.map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read checkpoint file: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,688 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{Error, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use super::super::HealDiskExt as _;
|
||||
|
||||
use super::super::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes};
|
||||
use super::{
|
||||
DiskError, DiskStore, RUSTFS_META_BUCKET, ResumeManager, ResumeState, delete_resume_file, ensure_replacement_recovery_dir,
|
||||
injected_replacement_proof_write_error, is_replacement_intent, legacy_replacement_completion_proof_path, path_to_str,
|
||||
replacement_completion_proof_path, replacement_intent_seal_path, replacement_recovery_conflict,
|
||||
replacement_recovery_corruption, validate_resume_task_id,
|
||||
};
|
||||
|
||||
/// Durable-proof schema version.
|
||||
const CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA: u32 = 1;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ReplacementPhase {
|
||||
#[default]
|
||||
None,
|
||||
Intent,
|
||||
Rebuilding,
|
||||
Verified,
|
||||
CleanupPending,
|
||||
Abandoned,
|
||||
}
|
||||
|
||||
/// Target-specific state for a durable automatic replacement generation.
|
||||
///
|
||||
/// This is deliberately separate from the legacy background-heal status
|
||||
/// contract. Consumers must treat [`Self::Unknown`] as non-definitive.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ReplacementRecoveryState {
|
||||
WaitingForReplacement,
|
||||
Running,
|
||||
Incomplete,
|
||||
Unrecoverable,
|
||||
CleanupPending,
|
||||
Completed,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Read-only status derived from one durable replacement generation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ReplacementRecoveryRecord {
|
||||
pub task_id: String,
|
||||
pub state: ReplacementRecoveryState,
|
||||
pub generation: Option<String>,
|
||||
pub set_disk_id: Option<String>,
|
||||
pub target_slots: Vec<String>,
|
||||
pub reason: Option<String>,
|
||||
pub verified_at: Option<u64>,
|
||||
}
|
||||
|
||||
impl ReplacementRecoveryRecord {
|
||||
pub(super) fn from_state(state: ResumeState) -> Option<Self> {
|
||||
if !is_replacement_intent(&state) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let invariant_holds = state.replacement_generation.as_deref() == Some(state.task_id.as_str())
|
||||
&& replacement_targets_match_identities(&state.replacement_targets, &state.replacement_target_identities);
|
||||
if !invariant_holds {
|
||||
return Some(Self::unknown(
|
||||
state.task_id,
|
||||
"durable replacement state violates its generation or target identity binding",
|
||||
));
|
||||
}
|
||||
|
||||
let (state_kind, reason) = if !state.completed && state.retry_count >= state.max_retries {
|
||||
(
|
||||
ReplacementRecoveryState::Unrecoverable,
|
||||
Some("replacement retry budget exhausted".to_string()),
|
||||
)
|
||||
} else if let Some(reason) = state.error_message.clone() {
|
||||
(ReplacementRecoveryState::Incomplete, Some(reason))
|
||||
} else {
|
||||
match state.replacement_phase {
|
||||
ReplacementPhase::Intent => (ReplacementRecoveryState::WaitingForReplacement, None),
|
||||
ReplacementPhase::Rebuilding => (ReplacementRecoveryState::Running, None),
|
||||
ReplacementPhase::Verified | ReplacementPhase::CleanupPending => (ReplacementRecoveryState::CleanupPending, None),
|
||||
ReplacementPhase::Abandoned => (
|
||||
ReplacementRecoveryState::Unrecoverable,
|
||||
Some("replacement generation was abandoned".to_string()),
|
||||
),
|
||||
ReplacementPhase::None => (ReplacementRecoveryState::Unknown, Some("replacement phase is missing".to_string())),
|
||||
}
|
||||
};
|
||||
|
||||
Some(Self {
|
||||
task_id: state.task_id,
|
||||
state: state_kind,
|
||||
generation: state.replacement_generation,
|
||||
set_disk_id: Some(state.set_disk_id),
|
||||
target_slots: state.replacement_targets,
|
||||
reason,
|
||||
verified_at: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn from_completion_proof(proof: &ReplacementCompletionProof) -> Self {
|
||||
Self {
|
||||
task_id: proof.task_id.clone(),
|
||||
state: ReplacementRecoveryState::Completed,
|
||||
generation: Some(proof.replacement_generation.clone()),
|
||||
set_disk_id: Some(proof.set_disk_id.clone()),
|
||||
target_slots: proof.replacement_targets.clone(),
|
||||
reason: None,
|
||||
verified_at: Some(proof.verified_at),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn unknown(task_id: String, reason: &str) -> Self {
|
||||
Self {
|
||||
task_id,
|
||||
state: ReplacementRecoveryState::Unknown,
|
||||
generation: None,
|
||||
set_disk_id: None,
|
||||
target_slots: Vec::new(),
|
||||
reason: Some(reason.to_string()),
|
||||
verified_at: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn replacement_targets_match_identities(targets: &[String], identities: &[ReplacementTargetIdentity]) -> bool {
|
||||
!targets.is_empty()
|
||||
&& targets.len() == identities.len()
|
||||
&& targets.iter().collect::<HashSet<_>>().len() == targets.len()
|
||||
&& identities.iter().map(|identity| &identity.endpoint).eq(targets.iter())
|
||||
}
|
||||
|
||||
/// Stable evidence for the mounted replacement instance that owns a repair
|
||||
/// generation. Endpoint text alone is not sufficient because a later disk can
|
||||
/// be mounted at the same configured path.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ReplacementTargetIdentity {
|
||||
pub endpoint: String,
|
||||
pub canonical_path: String,
|
||||
pub physical_device_ids: Vec<String>,
|
||||
pub filesystem_identity: String,
|
||||
}
|
||||
|
||||
/// Durable terminal evidence for one automatic replacement generation. This
|
||||
/// lives on the healthy non-target anchor rather than in the resumable state,
|
||||
/// because resume cleanup must not erase proof that the generation completed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct ReplacementCompletionProof {
|
||||
pub schema_version: u32,
|
||||
pub task_id: String,
|
||||
pub replacement_generation: String,
|
||||
pub set_disk_id: String,
|
||||
pub replacement_targets: Vec<String>,
|
||||
pub replacement_target_identities: Vec<ReplacementTargetIdentity>,
|
||||
pub verified_at: u64,
|
||||
}
|
||||
|
||||
impl ReplacementCompletionProof {
|
||||
pub(super) fn from_state(state: &ResumeState, verified_at: u64) -> Result<Self> {
|
||||
let replacement_generation = state
|
||||
.replacement_generation
|
||||
.clone()
|
||||
.ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion has no generation for task {}", state.task_id),
|
||||
})?;
|
||||
if replacement_generation != state.task_id
|
||||
|| state.replacement_targets.is_empty()
|
||||
|| state
|
||||
.replacement_target_identities
|
||||
.iter()
|
||||
.map(|identity| &identity.endpoint)
|
||||
.collect::<Vec<_>>()
|
||||
!= state.replacement_targets.iter().collect::<Vec<_>>()
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion identity does not match task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
schema_version: CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA,
|
||||
task_id: state.task_id.clone(),
|
||||
replacement_generation,
|
||||
set_disk_id: state.set_disk_id.clone(),
|
||||
replacement_targets: state.replacement_targets.clone(),
|
||||
replacement_target_identities: state.replacement_target_identities.clone(),
|
||||
verified_at,
|
||||
})
|
||||
}
|
||||
|
||||
fn matches_state(&self, state: &ResumeState) -> bool {
|
||||
self.schema_version == CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA
|
||||
&& self.task_id == state.task_id
|
||||
&& state.replacement_generation.as_deref() == Some(self.replacement_generation.as_str())
|
||||
&& self.set_disk_id == state.set_disk_id
|
||||
&& self.replacement_targets == state.replacement_targets
|
||||
&& self.replacement_target_identities == state.replacement_target_identities
|
||||
}
|
||||
|
||||
fn validate(&self, expected_task_id: &str) -> Result<()> {
|
||||
if self.schema_version != CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof schema {} is unsupported", self.schema_version),
|
||||
});
|
||||
}
|
||||
validate_resume_task_id(expected_task_id)?;
|
||||
if self.task_id != expected_task_id
|
||||
|| self.replacement_generation != self.task_id
|
||||
|| self.set_disk_id.is_empty()
|
||||
|| self.verified_at == 0
|
||||
|| !replacement_targets_match_identities(&self.replacement_targets, &self.replacement_target_identities)
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof does not match task {expected_task_id}"),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn replacement_target_identities_match(
|
||||
expected: &[ReplacementTargetIdentity],
|
||||
actual: &[ReplacementTargetIdentity],
|
||||
) -> bool {
|
||||
let mut expected = expected.to_vec();
|
||||
let mut actual = actual.to_vec();
|
||||
expected.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
actual.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
expected == actual
|
||||
}
|
||||
|
||||
/// Build the canonical, provably-injective dedup identity for an object
|
||||
/// version. Length-prefixing the object key makes the encoding injective: no
|
||||
/// two distinct `(object, version_id)` pairs can collide, even for adversarial
|
||||
/// keys containing `:` or embedded null bytes. This is the single source of
|
||||
/// truth for per-version dedup across the heal loop and the checkpoint sets.
|
||||
pub fn compose_key(object: &str, version_id: Option<&str>) -> String {
|
||||
format!("{}:{}{}", object.len(), object, version_id.unwrap_or(""))
|
||||
}
|
||||
|
||||
impl ResumeManager {
|
||||
/// Seal a durably published intent before the caller may format a target.
|
||||
/// A torn intent without this seal is known to have failed before its
|
||||
/// creator returned and can be atomically recreated on retry.
|
||||
pub(super) async fn ensure_replacement_intent_seal(&self) -> Result<()> {
|
||||
let task_id = self.state.read().await.task_id.clone();
|
||||
validate_resume_task_id(&task_id)?;
|
||||
let path = replacement_intent_seal_path(&task_id);
|
||||
let path = path_to_str(&path)?;
|
||||
match self.disk.read_all(RUSTFS_META_BUCKET, path).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(DiskError::FileNotFound) => {}
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read replacement intent seal: {error}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
self.disk
|
||||
.write_all(RUSTFS_META_BUCKET, path, b"sealed".as_slice().into())
|
||||
.await
|
||||
.map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save replacement intent seal: {error}"),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn mark_replacement_rebuilding(
|
||||
&self,
|
||||
mut replacement_target_identities: Vec<ReplacementTargetIdentity>,
|
||||
) -> Result<()> {
|
||||
replacement_target_identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
replacement_target_identities.dedup_by(|left, right| left.endpoint == right.endpoint);
|
||||
let mut state = self.state.write().await;
|
||||
if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement intent is not active for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
if replacement_target_identities
|
||||
.iter()
|
||||
.map(|identity| &identity.endpoint)
|
||||
.collect::<Vec<_>>()
|
||||
!= state.replacement_targets.iter().collect::<Vec<_>>()
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement identities do not match targets for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
if !replacement_target_identities_match(&state.replacement_target_identities, &replacement_target_identities) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement target changed after format for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
state.replacement_phase = ReplacementPhase::Rebuilding;
|
||||
state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
/// Persist survivor-anchor completion proof before transitioning this
|
||||
/// resumable state to `Verified`. If proof persistence fails, this state
|
||||
/// stays rebuildable and the caller must retain the healing marker.
|
||||
pub async fn mark_replacement_completed_and_verified(&self) -> Result<()> {
|
||||
let state = self.state.read().await.clone();
|
||||
if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement verification is not active for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
let proof = self.write_replacement_completion_proof(&state, None).await?;
|
||||
|
||||
let mut state = self.state.write().await;
|
||||
if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement verification changed for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
state.mark_completed();
|
||||
state.replacement_phase = ReplacementPhase::Verified;
|
||||
state.last_update = proof.verified_at;
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
/// Verify or backfill the terminal proof before marker removal or resume
|
||||
/// cleanup. This supports restart recovery from a `Verified` state written
|
||||
/// by a prior binary that did not yet have a separate proof record.
|
||||
pub(crate) async fn ensure_replacement_completion_proof(&self) -> Result<ReplacementCompletionProof> {
|
||||
let state = self.state.read().await.clone();
|
||||
if !state.completed || !matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion is not verified for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
self.write_replacement_completion_proof(&state, Some(state.last_update)).await
|
||||
}
|
||||
|
||||
/// Record that the healing markers have been removed, so a later retry can
|
||||
/// safely delete the remaining resume artifacts without touching markers.
|
||||
pub async fn mark_replacement_cleanup_pending(&self) -> Result<()> {
|
||||
let mut state = self.state.write().await;
|
||||
if !state.completed || !matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement cleanup is not ready for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
state.replacement_phase = ReplacementPhase::CleanupPending;
|
||||
state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
/// Load the durable terminal proof from the healthy survivor anchor.
|
||||
pub(crate) async fn load_replacement_completion_proof(disk: DiskStore, task_id: &str) -> Result<ReplacementCompletionProof> {
|
||||
Self::replacement_completion_proof_if_present(disk, task_id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read replacement completion proof: proof is missing for task {task_id}"),
|
||||
})
|
||||
}
|
||||
|
||||
async fn replacement_completion_proof_if_present(
|
||||
disk: DiskStore,
|
||||
task_id: &str,
|
||||
) -> Result<Option<ReplacementCompletionProof>> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let mut proofs = Vec::new();
|
||||
for path in [
|
||||
replacement_completion_proof_path(task_id),
|
||||
legacy_replacement_completion_proof_path(task_id),
|
||||
] {
|
||||
let path_str = path_to_str(&path)?;
|
||||
let bytes = match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(DiskError::FileNotFound) => continue,
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
let proof: ReplacementCompletionProof =
|
||||
serde_json::from_slice(&bytes).map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to deserialize replacement completion proof: {error}"),
|
||||
})?;
|
||||
proof.validate(task_id)?;
|
||||
proofs.push(proof);
|
||||
}
|
||||
|
||||
match proofs.as_slice() {
|
||||
[] => Ok(None),
|
||||
[proof] => Ok(Some(proof.clone())),
|
||||
[proof, legacy_proof] if proof == legacy_proof => Ok(Some(proof.clone())),
|
||||
_ => Err(replacement_recovery_conflict(format!(
|
||||
"Replacement completion proof conflicts with legacy proof for task {task_id}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reconcile the proof-first publication order after a crash. A matching
|
||||
/// proof is durable evidence that rebuilding finished, so it must win over
|
||||
/// an older active state before a retry may format the target again.
|
||||
pub(super) async fn reconcile_replacement_completion_proof(&self) -> Result<()> {
|
||||
let task_id = self.state.read().await.task_id.clone();
|
||||
let Some(proof) = Self::replacement_completion_proof_if_present(self.disk.clone(), &task_id).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut state = self.state.write().await;
|
||||
if !proof.matches_state(&state) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof does not match active intent for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
if state.completed && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) {
|
||||
return Ok(());
|
||||
}
|
||||
if state.completed || !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) {
|
||||
return Err(replacement_recovery_conflict(format!(
|
||||
"Replacement completion proof conflicts with state for task {}",
|
||||
state.task_id
|
||||
)));
|
||||
}
|
||||
|
||||
state.mark_completed();
|
||||
state.replacement_phase = ReplacementPhase::Verified;
|
||||
state.last_update = proof.verified_at;
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
pub(super) async fn migrate_legacy_replacement_completion_proof(disk: &DiskStore, task_id: &str) -> Result<bool> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let legacy_path = legacy_replacement_completion_proof_path(task_id);
|
||||
let legacy_path_str = path_to_str(&legacy_path)?;
|
||||
let legacy_bytes = match disk.read_all(RUSTFS_META_BUCKET, legacy_path_str).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(DiskError::FileNotFound) => return Ok(false),
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read legacy replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
let legacy_proof: ReplacementCompletionProof = serde_json::from_slice(&legacy_bytes).map_err(|error| {
|
||||
replacement_recovery_corruption(format!("Failed to deserialize legacy replacement completion proof: {error}"))
|
||||
})?;
|
||||
legacy_proof
|
||||
.validate(task_id)
|
||||
.map_err(|error| replacement_recovery_corruption(format!("Invalid legacy replacement completion proof: {error}")))?;
|
||||
|
||||
ensure_replacement_recovery_dir(disk)
|
||||
.await
|
||||
.map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to create replacement recovery directory: {error}"),
|
||||
})?;
|
||||
let path = replacement_completion_proof_path(task_id);
|
||||
let path_str = path_to_str(&path)?;
|
||||
for _ in 0..2 {
|
||||
match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(bytes) => {
|
||||
let proof: ReplacementCompletionProof =
|
||||
serde_json::from_slice(&bytes).map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to deserialize replacement completion proof: {error}"),
|
||||
})?;
|
||||
proof.validate(task_id).map_err(|error| {
|
||||
replacement_recovery_corruption(format!("Invalid replacement completion proof: {error}"))
|
||||
})?;
|
||||
if proof != legacy_proof {
|
||||
return Err(replacement_recovery_conflict(format!(
|
||||
"Replacement completion proof conflicts with legacy proof for task {task_id}"
|
||||
)));
|
||||
}
|
||||
delete_resume_file(disk, &legacy_path).await?;
|
||||
return Ok(true);
|
||||
}
|
||||
Err(DiskError::FileNotFound) => {}
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path_str,
|
||||
None,
|
||||
Some(legacy_bytes.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => {
|
||||
delete_resume_file(disk, &legacy_path).await?;
|
||||
return Ok(true);
|
||||
}
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue,
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to migrate replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof changed while migrating task {task_id}"),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn abandon_replacement_intent(&self) -> Result<()> {
|
||||
let mut state = self.state.write().await;
|
||||
if matches!(state.replacement_phase, ReplacementPhase::Abandoned) {
|
||||
return Ok(());
|
||||
}
|
||||
state.replacement_phase = ReplacementPhase::Abandoned;
|
||||
state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
pub async fn set_replacement_targets(&self, replacement_targets: Vec<String>) -> Result<()> {
|
||||
{
|
||||
let mut state = self.state.write().await;
|
||||
state.replacement_targets = replacement_targets;
|
||||
}
|
||||
self.save_state().await
|
||||
}
|
||||
|
||||
pub(super) async fn publish_new_replacement_intent(&self, expected: Option<EcstoreDiskBytes>) -> Result<()> {
|
||||
let state = self.state.read().await.clone();
|
||||
validate_resume_task_id(&state.task_id)?;
|
||||
let state_data = EcstoreDiskBytes::from(serde_json::to_vec(&state).map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize resume state: {error}"),
|
||||
})?);
|
||||
let path = self.state_file.path(&state.task_id);
|
||||
let path = path_to_str(&path)?;
|
||||
|
||||
ensure_replacement_recovery_dir(&self.disk)
|
||||
.await
|
||||
.map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to create replacement recovery directory: {error}"),
|
||||
})?;
|
||||
match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file(
|
||||
self.disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
expected,
|
||||
Some(state_data),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => Ok(()),
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => {
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement intent changed before publication for task {}", state.task_id),
|
||||
})
|
||||
}
|
||||
Err(error) => Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save resume state: {error}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_replacement_completion_proof(
|
||||
&self,
|
||||
state: &ResumeState,
|
||||
verified_at: Option<u64>,
|
||||
) -> Result<ReplacementCompletionProof> {
|
||||
ensure_replacement_recovery_dir(&self.disk)
|
||||
.await
|
||||
.map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to create replacement recovery directory: {error}"),
|
||||
})?;
|
||||
let path = replacement_completion_proof_path(&state.task_id);
|
||||
let path_str = path_to_str(&path)?;
|
||||
let proof = ReplacementCompletionProof::from_state(
|
||||
state,
|
||||
verified_at.unwrap_or_else(|| SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()),
|
||||
)?;
|
||||
let proof_data = EcstoreDiskBytes::from(serde_json::to_vec(&proof).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize replacement completion proof: {e}"),
|
||||
})?);
|
||||
if let Some(error) = injected_replacement_proof_write_error(path_str) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
|
||||
// Publish through the disk CAS primitive: `write_all` can expose a
|
||||
// partially written proof to a crash/restart reader. If a prior
|
||||
// version left torn bytes behind, replace exactly the observed bytes;
|
||||
// a concurrently published valid proof is never overwritten.
|
||||
for _ in 0..2 {
|
||||
let expected = match self.disk.read_all(RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(existing) => match serde_json::from_slice::<ReplacementCompletionProof>(&existing) {
|
||||
Ok(existing_proof) => {
|
||||
existing_proof.validate(&state.task_id)?;
|
||||
if existing_proof.matches_state(state) {
|
||||
return Ok(existing_proof);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof does not match task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
Err(_) => Some(existing),
|
||||
},
|
||||
Err(DiskError::FileNotFound) => None,
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file(
|
||||
self.disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path_str,
|
||||
expected,
|
||||
Some(proof_data.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(proof),
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue,
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof changed while publishing task {}", state.task_id),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn write_replacement_intent_state(
|
||||
&self,
|
||||
path: &str,
|
||||
state_data: EcstoreDiskBytes,
|
||||
) -> std::result::Result<(), DiskError> {
|
||||
ensure_replacement_recovery_dir(&self.disk).await?;
|
||||
for _ in 0..2 {
|
||||
let expected = match self.disk.read_all(RUSTFS_META_BUCKET, path).await {
|
||||
Ok(existing) => Some(existing),
|
||||
Err(DiskError::FileNotFound) => None,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file(
|
||||
self.disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
expected,
|
||||
Some(state_data.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(()),
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
Err(DiskError::other("replacement intent changed while publishing"))
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,311 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{Error, Result};
|
||||
use std::collections::HashSet;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tracing::{debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
|
||||
use super::replacement::{ReplacementPhase, ReplacementRecoveryRecord};
|
||||
use super::{
|
||||
EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
|
||||
REPLACEMENT_INTENT_FILE, RESUME_STATE_FILE, ResumeManager, ResumeStateFile, is_replacement_intent, path_to_str,
|
||||
replacement_recovery_corruption_for_state_load, replacement_recovery_dir, validate_resume_task_id,
|
||||
};
|
||||
|
||||
/// resume utils
|
||||
pub struct ResumeUtils;
|
||||
|
||||
impl ResumeUtils {
|
||||
/// generate unique task id
|
||||
pub fn generate_task_id() -> String {
|
||||
Uuid::new_v4().to_string()
|
||||
}
|
||||
|
||||
/// check if task can be resumed
|
||||
pub async fn can_resume_task(disk: &DiskStore, task_id: &str) -> bool {
|
||||
ResumeManager::has_resume_state(disk, task_id).await
|
||||
}
|
||||
|
||||
/// get all resumable task ids
|
||||
pub async fn get_resumable_tasks(disk: &DiskStore) -> Result<Vec<String>> {
|
||||
// List all files in the buckets metadata directory
|
||||
let entries = match disk.list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
state = "list_failed",
|
||||
error = %e,
|
||||
"Heal resume state listing failed"
|
||||
);
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
};
|
||||
|
||||
let mut task_ids = Vec::new();
|
||||
|
||||
// Filter files that end with ahm_resume_state.json and extract task IDs
|
||||
for entry in entries {
|
||||
if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) {
|
||||
// Extract task ID from filename: {task_id}_ahm_resume_state.json
|
||||
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}"))
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
{
|
||||
task_ids.push(task_id.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_count = task_ids.len(),
|
||||
state = "listed",
|
||||
"Heal resume states listed"
|
||||
);
|
||||
Ok(task_ids)
|
||||
}
|
||||
|
||||
/// Return replacement intent task IDs from the dedicated recovery
|
||||
/// directory. Periodic recovery must never enumerate the ordinary resume
|
||||
/// directory, whose cardinality is unrelated to replacement work.
|
||||
pub async fn get_replacement_intent_tasks(disk: &DiskStore) -> Result<Vec<String>> {
|
||||
let entries = Self::replacement_recovery_entries(disk).await?;
|
||||
let suffix = format!("_{REPLACEMENT_INTENT_FILE}");
|
||||
let mut task_ids = HashSet::new();
|
||||
|
||||
for entry in entries {
|
||||
if let Some(task_id) = entry.strip_suffix(&suffix)
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
{
|
||||
task_ids.insert(task_id.to_string());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let mut task_ids = task_ids.into_iter().collect::<Vec<_>>();
|
||||
task_ids.sort_unstable();
|
||||
Ok(task_ids)
|
||||
}
|
||||
|
||||
async fn replacement_recovery_entries(disk: &DiskStore) -> Result<Vec<String>> {
|
||||
let recovery_dir = replacement_recovery_dir();
|
||||
let recovery_dir = path_to_str(&recovery_dir)?;
|
||||
match disk.list_dir("", RUSTFS_META_BUCKET, recovery_dir, -1).await {
|
||||
Ok(entries) => Ok(entries),
|
||||
Err(DiskError::FileNotFound) => Ok(Vec::new()),
|
||||
Err(error @ DiskError::UnformattedDisk) => Err(error.into()),
|
||||
Err(error) => Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to list replacement recovery records: {error}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Migrate flat replacement artifacts from earlier builds exactly once at
|
||||
/// manager startup. The normal scanner only uses the dedicated directory;
|
||||
/// ordinary resume JSON is never read on its periodic path.
|
||||
pub async fn migrate_legacy_replacement_records(disk: &DiskStore) -> Result<()> {
|
||||
let entries = disk
|
||||
.list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1)
|
||||
.await
|
||||
.map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to list legacy replacement records: {error}"),
|
||||
})?;
|
||||
let ordinary_suffix = format!("_{RESUME_STATE_FILE}");
|
||||
let intent_suffix = format!("_{REPLACEMENT_INTENT_FILE}");
|
||||
let proof_suffix = format!("_{REPLACEMENT_COMPLETION_PROOF_FILE}");
|
||||
let mut ordinary_task_ids = HashSet::new();
|
||||
let mut intent_task_ids = HashSet::new();
|
||||
let mut proof_task_ids = HashSet::new();
|
||||
|
||||
for entry in entries {
|
||||
if let Some(task_id) = entry.strip_suffix(&intent_suffix)
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
{
|
||||
intent_task_ids.insert(task_id.to_string());
|
||||
continue;
|
||||
}
|
||||
if let Some(task_id) = entry.strip_suffix(&ordinary_suffix)
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
{
|
||||
ordinary_task_ids.insert(task_id.to_string());
|
||||
continue;
|
||||
}
|
||||
if let Some(task_id) = entry.strip_suffix(&proof_suffix)
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
{
|
||||
proof_task_ids.insert(task_id.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let mut state_task_ids = intent_task_ids.into_iter().collect::<Vec<_>>();
|
||||
state_task_ids.extend(ordinary_task_ids);
|
||||
state_task_ids.sort_unstable();
|
||||
state_task_ids.dedup();
|
||||
for task_id in state_task_ids {
|
||||
let has_flat_intent = ResumeManager::has_state_file(disk, &task_id, ResumeStateFile::LegacyReplacementIntent).await;
|
||||
if !has_flat_intent {
|
||||
let manager = ResumeManager::load_from_disk(disk.clone(), &task_id).await.map_err(|error| {
|
||||
replacement_recovery_corruption_for_state_load(
|
||||
format!("Failed to load legacy replacement recovery candidate {task_id}"),
|
||||
error,
|
||||
)
|
||||
})?;
|
||||
if !is_replacement_intent(&manager.get_state().await) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
ResumeManager::load_replacement_intent(disk.clone(), &task_id).await?;
|
||||
}
|
||||
|
||||
for task_id in proof_task_ids {
|
||||
ResumeManager::migrate_legacy_replacement_completion_proof(disk, &task_id).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return all durable replacement states and completion proofs stored on
|
||||
/// one survivor disk. Unlike the legacy resumable-task helper, listing
|
||||
/// failures are returned to the caller so an observability surface cannot
|
||||
/// silently turn an unreadable durable record into a green result.
|
||||
pub async fn get_replacement_recovery_records(disk: &DiskStore) -> Result<Vec<ReplacementRecoveryRecord>> {
|
||||
let entries = Self::replacement_recovery_entries(disk).await?;
|
||||
let proof_suffix = format!("_{REPLACEMENT_COMPLETION_PROOF_FILE}");
|
||||
let mut records = Vec::new();
|
||||
let mut intent_task_ids = HashSet::new();
|
||||
|
||||
for task_id in Self::get_replacement_intent_tasks(disk).await? {
|
||||
let state = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
|
||||
.await?
|
||||
.get_state()
|
||||
.await;
|
||||
intent_task_ids.insert(task_id.clone());
|
||||
records.push(ReplacementRecoveryRecord::from_state(state).unwrap_or_else(|| {
|
||||
ReplacementRecoveryRecord::unknown(
|
||||
task_id,
|
||||
"isolated replacement intent violates its generation or target identity binding",
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
for entry in entries {
|
||||
let Some(task_id) = entry.strip_suffix(&proof_suffix) else {
|
||||
continue;
|
||||
};
|
||||
if validate_resume_task_id(task_id).is_err() {
|
||||
continue;
|
||||
}
|
||||
if intent_task_ids.contains(task_id) {
|
||||
continue;
|
||||
}
|
||||
let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), task_id).await?;
|
||||
records.push(ReplacementRecoveryRecord::from_completion_proof(&proof));
|
||||
}
|
||||
|
||||
records.sort_by(|left, right| left.task_id.cmp(&right.task_id).then(left.state.cmp(&right.state)));
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
/// cleanup expired resume states
|
||||
pub async fn cleanup_expired_states(disk: &DiskStore, max_age_hours: u64) -> Result<()> {
|
||||
let task_ids = Self::get_resumable_tasks(disk).await?;
|
||||
let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
|
||||
|
||||
for task_id in task_ids {
|
||||
if let Ok(resume_manager) = ResumeManager::load_from_disk(disk.clone(), &task_id).await {
|
||||
let state = resume_manager.get_state().await;
|
||||
let age_hours = current_time.saturating_sub(state.last_update) / 3600;
|
||||
|
||||
if !state.completed && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if age_hours > max_age_hours {
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
age_hours,
|
||||
state = "expired_cleanup_started",
|
||||
"Heal resume cleanup started"
|
||||
);
|
||||
if let Err(e) = resume_manager.cleanup().await {
|
||||
warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
age_hours,
|
||||
state = "expired_cleanup_failed",
|
||||
error = %e,
|
||||
"Heal resume state cleanup failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for task_id in Self::get_replacement_intent_tasks(disk).await? {
|
||||
if let Ok(resume_manager) = ResumeManager::load_replacement_intent(disk.clone(), &task_id).await {
|
||||
let state = resume_manager.get_state().await;
|
||||
let age_hours = current_time.saturating_sub(state.last_update) / 3600;
|
||||
|
||||
if !state.completed && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if age_hours > max_age_hours
|
||||
&& let Err(e) = resume_manager.cleanup().await
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
age_hours,
|
||||
state = "expired_cleanup_failed",
|
||||
error = %e,
|
||||
"Replacement intent cleanup failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
+550
-93
@@ -27,7 +27,7 @@ use super::storage_api::storage::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
|
||||
ObjectOperations as _, StorageAdminApi,
|
||||
};
|
||||
use super::{DiskStore, ECStore, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity};
|
||||
use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity};
|
||||
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
@@ -42,10 +42,7 @@ pub struct HealLifecycleExpiryContext {
|
||||
|
||||
enum HealLifecycleExpiryContextInner {
|
||||
Ecstore(EcstoreHealLifecycleExpiryContext),
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "constructed by the #[cfg(test)] `test()` helper; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
#[allow(dead_code)]
|
||||
Test,
|
||||
}
|
||||
|
||||
@@ -68,6 +65,7 @@ const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_READ_LIMIT: &str = "heal_storage_object_read_limit";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_VERIFY: &str = "heal_storage_object_verify";
|
||||
const EVENT_HEAL_STORAGE_ADMIN_OP: &str = "heal_storage_admin_op";
|
||||
const EVENT_HEAL_STORAGE_REPAIR_OP: &str = "heal_storage_repair_op";
|
||||
|
||||
@@ -314,23 +312,56 @@ pub struct HealListItem {
|
||||
pub is_delete_marker: bool,
|
||||
}
|
||||
|
||||
/// Disk status for heal operations
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum DiskStatus {
|
||||
/// Ok
|
||||
Ok,
|
||||
/// Offline
|
||||
Offline,
|
||||
/// Corrupt
|
||||
Corrupt,
|
||||
/// Missing
|
||||
Missing,
|
||||
/// Permission denied
|
||||
PermissionDenied,
|
||||
/// Faulty
|
||||
Faulty,
|
||||
/// Root mount
|
||||
RootMount,
|
||||
/// Unknown
|
||||
Unknown,
|
||||
/// Unformatted
|
||||
Unformatted,
|
||||
}
|
||||
|
||||
/// Heal storage layer interface
|
||||
#[async_trait]
|
||||
pub trait HealStorageAPI: Send + Sync {
|
||||
/// Get object meta
|
||||
///
|
||||
/// Reserved for HS-01 MRF wiring (rustfs/backlog#1865): MRF intents
|
||||
/// currently execute through `heal_object`; keep this entry point for the
|
||||
/// metadata-corruption variant that must inspect metadata first.
|
||||
async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<HealObjectInfo>>;
|
||||
|
||||
/// Get object data
|
||||
async fn get_object_data(&self, bucket: &str, object: &str) -> Result<Option<Vec<u8>>>;
|
||||
|
||||
/// Put object data
|
||||
async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()>;
|
||||
|
||||
/// Delete object
|
||||
async fn delete_object(&self, bucket: &str, object: &str) -> Result<()>;
|
||||
|
||||
/// Check object integrity
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result<bool>;
|
||||
|
||||
/// EC decode rebuild
|
||||
///
|
||||
/// Reserved for HS-01 MRF wiring (rustfs/backlog#1865): urgent ECDecode
|
||||
/// requests currently execute through `heal_object`; keep the explicit
|
||||
/// rebuild-and-read path for the decode-failure fast variant.
|
||||
async fn ec_decode_rebuild(&self, bucket: &str, object: &str) -> Result<Vec<u8>>;
|
||||
|
||||
/// Get disk status
|
||||
async fn get_disk_status(&self, endpoint: &Endpoint) -> Result<DiskStatus>;
|
||||
|
||||
/// Format disk
|
||||
async fn format_disk(&self, endpoint: &Endpoint) -> Result<()>;
|
||||
|
||||
/// Get bucket info
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>>;
|
||||
|
||||
@@ -356,12 +387,21 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Fix bucket metadata
|
||||
async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>;
|
||||
|
||||
/// Get all buckets
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>>;
|
||||
|
||||
/// Check object exists
|
||||
async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool>;
|
||||
|
||||
/// Get object size
|
||||
async fn get_object_size(&self, bucket: &str, object: &str) -> Result<Option<u64>>;
|
||||
|
||||
/// Get object checksum
|
||||
async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result<Option<String>>;
|
||||
|
||||
/// Heal object using ecstore
|
||||
async fn heal_object(
|
||||
&self,
|
||||
@@ -413,6 +453,12 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// List object versions for healing (returns all versions, may use significant memory for large buckets)
|
||||
///
|
||||
/// WARNING: This method loads all object versions into memory at once. For buckets with many
|
||||
/// objects/versions, consider using `list_objects_for_heal_page` instead to process versions in pages.
|
||||
async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result<Vec<HealListItem>>;
|
||||
|
||||
/// List object versions for healing with pagination (returns one page and continuation token)
|
||||
/// Returns (versions, next_continuation_token, is_truncated). The continuation token is an
|
||||
/// opaque composite `(marker, version_marker)` value — see `encode_heal_token`/`decode_heal_token`.
|
||||
@@ -481,11 +527,89 @@ impl ECStoreHealStorage {
|
||||
pub fn new(ecstore: Arc<ECStore>) -> Self {
|
||||
Self { ecstore }
|
||||
}
|
||||
}
|
||||
|
||||
fn is_transient_object_exists_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
|
||||
[
|
||||
"failed to acquire read lock",
|
||||
"lock acquisition failed",
|
||||
"lock acquisition timeout",
|
||||
"quorum not reached",
|
||||
"deadline has elapsed",
|
||||
"timed out",
|
||||
"network error",
|
||||
"transport error",
|
||||
"connection refused",
|
||||
]
|
||||
.iter()
|
||||
.any(|pattern| message.contains(pattern))
|
||||
}
|
||||
|
||||
fn is_transient_object_exists_error(err: &StorageError) -> bool {
|
||||
if err.is_quorum_error() {
|
||||
return true;
|
||||
}
|
||||
|
||||
match err {
|
||||
StorageError::Lock(lock_err) => lock_err.is_retryable() || is_transient_object_exists_message(&lock_err.to_string()),
|
||||
StorageError::Io(io_err) => is_transient_object_exists_message(&io_err.to_string()),
|
||||
StorageError::SlowDown | StorageError::OperationCanceled => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl HealStorageAPI for ECStoreHealStorage {
|
||||
async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<HealObjectInfo>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
"Heal storage request started"
|
||||
);
|
||||
|
||||
match self.ecstore.get_object_info(bucket, object, &Default::default()).await {
|
||||
Ok(info) => Ok(Some(info)),
|
||||
Err(e) => {
|
||||
// Map ObjectNotFound to None to align with Option return type
|
||||
if matches!(e, StorageError::ObjectNotFound(_, _)) {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
result = "not_found",
|
||||
"Heal storage object metadata missing"
|
||||
);
|
||||
Ok(None)
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage request failed"
|
||||
);
|
||||
Err(Error::other(e))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read back an object's bytes, capped to bound memory.
|
||||
///
|
||||
/// Private support for the reserved `ec_decode_rebuild` (HS-01); not part
|
||||
/// of the storage trait surface.
|
||||
async fn get_object_data(&self, bucket: &str, object: &str) -> Result<Option<Vec<u8>>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -571,85 +695,196 @@ impl ECStoreHealStorage {
|
||||
}
|
||||
Ok(Some(buf))
|
||||
}
|
||||
}
|
||||
|
||||
fn is_transient_object_exists_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
|
||||
[
|
||||
"failed to acquire read lock",
|
||||
"lock acquisition failed",
|
||||
"lock acquisition timeout",
|
||||
"quorum not reached",
|
||||
"deadline has elapsed",
|
||||
"timed out",
|
||||
"network error",
|
||||
"transport error",
|
||||
"connection refused",
|
||||
]
|
||||
.iter()
|
||||
.any(|pattern| message.contains(pattern))
|
||||
}
|
||||
|
||||
fn is_transient_object_exists_error(err: &StorageError) -> bool {
|
||||
if err.is_quorum_error() {
|
||||
return true;
|
||||
}
|
||||
|
||||
match err {
|
||||
StorageError::Lock(lock_err) => lock_err.is_retryable() || is_transient_object_exists_message(&lock_err.to_string()),
|
||||
StorageError::Io(io_err) => is_transient_object_exists_message(&io_err.to_string()),
|
||||
StorageError::SlowDown | StorageError::OperationCanceled => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl HealStorageAPI for ECStoreHealStorage {
|
||||
async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<HealObjectInfo>> {
|
||||
async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
operation = "put_object_data",
|
||||
bucket,
|
||||
object,
|
||||
bytes = data.len(),
|
||||
"Heal storage request started"
|
||||
);
|
||||
|
||||
let mut reader = HealPutObjReader::from_vec(data.to_vec());
|
||||
match (*self.ecstore)
|
||||
.put_object(bucket, object, &mut reader, &Default::default())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "put_object_data",
|
||||
bucket,
|
||||
object,
|
||||
result = "ok",
|
||||
"Heal storage object write completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "put_object_data",
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage request failed"
|
||||
);
|
||||
Err(Error::other(e))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_object(&self, bucket: &str, object: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "delete_object",
|
||||
bucket,
|
||||
object,
|
||||
"Heal storage request started"
|
||||
);
|
||||
|
||||
match self.ecstore.get_object_info(bucket, object, &Default::default()).await {
|
||||
Ok(info) => Ok(Some(info)),
|
||||
match self.ecstore.delete_object(bucket, object, Default::default()).await {
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "delete_object",
|
||||
bucket,
|
||||
object,
|
||||
result = "ok",
|
||||
"Heal storage object delete completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
// Map ObjectNotFound to None to align with Option return type
|
||||
if matches!(e, StorageError::ObjectNotFound(_, _)) {
|
||||
debug!(
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "delete_object",
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage request failed"
|
||||
);
|
||||
Err(Error::other(e))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result<bool> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
bucket,
|
||||
object,
|
||||
state = "started",
|
||||
"Heal storage object verification started"
|
||||
);
|
||||
|
||||
// Check object metadata first
|
||||
match self.get_object_meta(bucket, object).await? {
|
||||
Some(obj_info) => {
|
||||
if obj_info.size < 0 {
|
||||
warn!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
result = "not_found",
|
||||
"Heal storage object metadata missing"
|
||||
state = "invalid_size",
|
||||
"Heal storage object verification failed"
|
||||
);
|
||||
Ok(None)
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage request failed"
|
||||
);
|
||||
Err(Error::other(e))
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Stream-read the object to a sink to avoid loading into memory
|
||||
match (*self.ecstore)
|
||||
.get_object_reader(bucket, object, None, Default::default(), &Default::default())
|
||||
.await
|
||||
{
|
||||
Ok(reader) => {
|
||||
let mut stream = reader.stream;
|
||||
match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await {
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
bucket,
|
||||
object,
|
||||
state = "ok",
|
||||
"Heal storage object verified"
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
bucket,
|
||||
object,
|
||||
state = "stream_read_failed",
|
||||
error = %e,
|
||||
"Heal storage object verification failed"
|
||||
);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
bucket,
|
||||
object,
|
||||
state = "reader_open_failed",
|
||||
error = %e,
|
||||
"Heal storage object verification failed"
|
||||
);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
None => {
|
||||
warn!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
bucket,
|
||||
object,
|
||||
state = "metadata_missing",
|
||||
"Heal storage object verification failed"
|
||||
);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -741,6 +976,81 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_disk_status(&self, endpoint: &Endpoint) -> Result<DiskStatus> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_disk_status",
|
||||
endpoint = ?endpoint,
|
||||
state = "started",
|
||||
"Heal storage admin operation started"
|
||||
);
|
||||
|
||||
// TODO: implement disk status check using ecstore
|
||||
// For now, return Ok status
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_disk_status",
|
||||
endpoint = ?endpoint,
|
||||
result = "ok",
|
||||
disk_status = "ok",
|
||||
"Heal storage disk status resolved"
|
||||
);
|
||||
Ok(DiskStatus::Ok)
|
||||
}
|
||||
|
||||
async fn format_disk(&self, endpoint: &Endpoint) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "format_disk",
|
||||
endpoint = ?endpoint,
|
||||
state = "started",
|
||||
"Heal storage admin operation started"
|
||||
);
|
||||
|
||||
// Use ecstore's heal_format
|
||||
match self.heal_format(false).await {
|
||||
Ok((_, error)) => {
|
||||
if error.is_some() {
|
||||
return Err(Error::other(format!("Format failed: {error:?}")));
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "format_disk",
|
||||
endpoint = ?endpoint,
|
||||
result = "ok",
|
||||
"Heal storage disk format completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "format_disk",
|
||||
endpoint = ?endpoint,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage admin operation failed"
|
||||
);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -851,6 +1161,61 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "heal_bucket_metadata",
|
||||
bucket,
|
||||
state = "started",
|
||||
"Heal storage repair started"
|
||||
);
|
||||
|
||||
let heal_opts = HealOpts {
|
||||
recursive: true,
|
||||
dry_run: false,
|
||||
remove: false,
|
||||
recreate: false,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
update_parity: false,
|
||||
no_lock: false,
|
||||
pool: None,
|
||||
set: None,
|
||||
};
|
||||
|
||||
match self.heal_bucket(bucket, &heal_opts).await {
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "heal_bucket_metadata",
|
||||
bucket,
|
||||
result = "ok",
|
||||
"Heal storage bucket metadata repaired"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "heal_bucket_metadata",
|
||||
bucket,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage repair failed"
|
||||
);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -950,6 +1315,48 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_size(&self, bucket: &str, object: &str) -> Result<Option<u64>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_size",
|
||||
bucket,
|
||||
object,
|
||||
"Heal storage request started"
|
||||
);
|
||||
|
||||
match self.get_object_meta(bucket, object).await {
|
||||
Ok(Some(obj_info)) => Ok(Some(obj_info.size as u64)),
|
||||
Ok(None) => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result<Option<String>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_checksum",
|
||||
bucket,
|
||||
object,
|
||||
"Heal storage request started"
|
||||
);
|
||||
|
||||
match self.get_object_meta(bucket, object).await {
|
||||
Ok(Some(obj_info)) => {
|
||||
// Convert checksum bytes to hex string
|
||||
let checksum = obj_info.checksum.iter().map(|b| format!("{b:02x}")).collect::<String>();
|
||||
Ok(Some(checksum))
|
||||
}
|
||||
Ok(None) => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1140,6 +1547,65 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result<Vec<HealListItem>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "list_objects_for_heal",
|
||||
bucket,
|
||||
prefix,
|
||||
state = "started",
|
||||
"Heal storage admin operation started"
|
||||
);
|
||||
warn!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "list_objects_for_heal",
|
||||
bucket,
|
||||
prefix,
|
||||
state = "memory_heavy",
|
||||
"Heal storage version listing loads all versions into memory (footprint is per-version, not per-object)"
|
||||
);
|
||||
|
||||
let mut all_objects: Vec<HealListItem> = Vec::new();
|
||||
let mut continuation_token: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let (page_objects, next_token, is_truncated) = self
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false)
|
||||
.await?;
|
||||
|
||||
all_objects.extend(page_objects);
|
||||
|
||||
if !is_truncated {
|
||||
break;
|
||||
}
|
||||
|
||||
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
|
||||
if continuation_token.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "list_objects_for_heal",
|
||||
bucket,
|
||||
prefix,
|
||||
object_count = all_objects.len(),
|
||||
result = "ok",
|
||||
"Heal storage object listing completed"
|
||||
);
|
||||
Ok(all_objects)
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1202,22 +1668,13 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
let version_id = obj.version_id.map(|u| u.to_string());
|
||||
let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos());
|
||||
let is_delete_marker = obj.delete_marker;
|
||||
if include_lifecycle_object_info {
|
||||
HealListItem {
|
||||
name: obj.name.clone(),
|
||||
version_id,
|
||||
mod_time_unix_nanos,
|
||||
lifecycle_object_info: Some(obj),
|
||||
is_delete_marker,
|
||||
}
|
||||
} else {
|
||||
HealListItem {
|
||||
name: obj.name,
|
||||
version_id,
|
||||
mod_time_unix_nanos,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker,
|
||||
}
|
||||
let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone());
|
||||
HealListItem {
|
||||
name: obj.name,
|
||||
version_id,
|
||||
mod_time_unix_nanos,
|
||||
lifecycle_object_info,
|
||||
is_delete_marker,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
+4099
-48
File diff suppressed because it is too large
Load Diff
@@ -1,450 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
|
||||
use super::*;
|
||||
|
||||
impl HealTask {
|
||||
pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
stage = "start",
|
||||
recursive = self.options.recursive,
|
||||
"Heal bucket started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("bucket: {bucket}")));
|
||||
progress.update_progress(0, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 1: Check if bucket exists
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
stage = "check_existence",
|
||||
"Heal bucket stage entered"
|
||||
);
|
||||
self.check_control_flags().await?;
|
||||
let bucket_exists = self.await_with_control(self.storage.get_bucket_info(bucket)).await?.is_some();
|
||||
if !bucket_exists {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
result = "missing",
|
||||
"Heal bucket failed because the bucket does not exist"
|
||||
);
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Bucket not found: {bucket}"),
|
||||
});
|
||||
}
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(1, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 2: Perform bucket heal using ecstore
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
stage = "heal_with_ecstore",
|
||||
dry_run = self.options.dry_run,
|
||||
"Heal bucket stage entered"
|
||||
);
|
||||
let heal_opts = HealOpts {
|
||||
recursive: self.options.recursive,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: if self.options.recursive {
|
||||
false
|
||||
} else {
|
||||
self.options.remove_corrupted
|
||||
},
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
let heal_result = self.await_with_control(self.storage.heal_bucket(bucket, &heal_opts)).await;
|
||||
|
||||
match heal_result {
|
||||
Ok(result) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
drives_healed = result.drives_healed(),
|
||||
drives_total = result.drives_reported(),
|
||||
recursive = self.options.recursive,
|
||||
result = "ok",
|
||||
"Heal bucket completed"
|
||||
);
|
||||
self.record_result_item(result).await;
|
||||
|
||||
if self.options.recursive {
|
||||
self.heal_bucket_objects(bucket, "").await?;
|
||||
}
|
||||
|
||||
if !self.options.recursive {
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal bucket failed"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal bucket {bucket}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn heal_cluster(&self) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
stage = "cluster_recursive",
|
||||
"Heal cluster started"
|
||||
);
|
||||
|
||||
let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
|
||||
let mut failed = 0_u64;
|
||||
let mut retryable = 0_u64;
|
||||
let mut permanent = 0_u64;
|
||||
let mut first_object = None;
|
||||
let mut first_error = None;
|
||||
for bucket_info in bucket_infos {
|
||||
self.check_control_flags().await?;
|
||||
let mut retry_attempt = 0_u32;
|
||||
loop {
|
||||
match self.heal_bucket(&bucket_info.name).await {
|
||||
Ok(()) => break,
|
||||
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
|
||||
Err(err) => {
|
||||
if let Some(failure) = self.take_batch_failure().await {
|
||||
failed = failed.saturating_add(failure.failed);
|
||||
retryable = retryable.saturating_add(failure.retryable);
|
||||
permanent = permanent.saturating_add(failure.permanent);
|
||||
first_object.get_or_insert(failure.first_object);
|
||||
first_error.get_or_insert(failure.first_error);
|
||||
break;
|
||||
}
|
||||
if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
retry_attempt = retry_attempt.saturating_add(1);
|
||||
self.await_with_control(async {
|
||||
tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
continue;
|
||||
}
|
||||
failed = failed.saturating_add(1);
|
||||
if err.is_recoverable_heal() {
|
||||
retryable = retryable.saturating_add(1);
|
||||
} else {
|
||||
permanent = permanent.saturating_add(1);
|
||||
}
|
||||
first_object.get_or_insert(bucket_info.name.clone());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
let failure = BatchHealFailure {
|
||||
scope: "cluster".to_string(),
|
||||
failed,
|
||||
retryable,
|
||||
permanent,
|
||||
first_object: first_object.unwrap_or_default(),
|
||||
first_error: first_error.unwrap_or_default(),
|
||||
};
|
||||
return Err(self.record_batch_failure(failure).await);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn heal_prefix(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
prefix,
|
||||
stage = "prefix_recursive",
|
||||
"Heal prefix started"
|
||||
);
|
||||
|
||||
self.heal_bucket_objects(bucket, prefix).await
|
||||
}
|
||||
|
||||
#[hotpath::measure]
|
||||
async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||
let mut continuation_token: Option<String> = None;
|
||||
let mut scanned = 0u64;
|
||||
let mut healed = 0u64;
|
||||
let mut failed = 0u64;
|
||||
let mut retryable_failed = 0u64;
|
||||
let mut permanent_failed = 0u64;
|
||||
let mut bytes = 0u64;
|
||||
let mut first_failed_object = None;
|
||||
let mut first_error = None;
|
||||
let mut failure_samples_logged = 0_u64;
|
||||
|
||||
let heal_opts = HealOpts {
|
||||
recursive: false,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: self.options.remove_corrupted,
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
loop {
|
||||
self.check_control_flags().await?;
|
||||
let (objects, next_token, is_truncated) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut pending = objects;
|
||||
let mut retry_attempt = 0_u32;
|
||||
while !pending.is_empty() {
|
||||
if retry_attempt > 0 {
|
||||
self.await_with_control(async {
|
||||
tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
let mut retry = Vec::with_capacity(pending.len());
|
||||
for item in pending {
|
||||
self.check_control_flags().await?;
|
||||
let object = item.name.as_str();
|
||||
if retry_attempt == 0 {
|
||||
scanned = scanned.saturating_add(1);
|
||||
}
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
progress.update_progress(scanned, healed, failed, bytes);
|
||||
}
|
||||
|
||||
let error = match self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((result, None)) => {
|
||||
healed = healed.saturating_add(1);
|
||||
bytes = bytes.saturating_add(u64::try_from(result.object_size).unwrap_or_default());
|
||||
self.record_result_item(result).await;
|
||||
None
|
||||
}
|
||||
Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
|
||||
healed = healed.saturating_add(1);
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "object_dir_not_found_skipped",
|
||||
"Heal bucket object-dir candidate skipped after not-found result"
|
||||
);
|
||||
None
|
||||
}
|
||||
Ok((_, Some(err))) | Err(err) => Some(err),
|
||||
};
|
||||
|
||||
if let Some(err) = error {
|
||||
if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "transient_skip",
|
||||
error = %err,
|
||||
"Heal bucket object repair skipped due to transient metadata error"
|
||||
);
|
||||
} else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt = retry_attempt.saturating_add(1),
|
||||
error = %err,
|
||||
result = "object_retry_scheduled",
|
||||
"Heal bucket object retry scheduled"
|
||||
);
|
||||
retry.push(item);
|
||||
} else {
|
||||
failed = failed.saturating_add(1);
|
||||
if err.is_recoverable_heal() {
|
||||
retryable_failed = retryable_failed.saturating_add(1);
|
||||
} else {
|
||||
permanent_failed = permanent_failed.saturating_add(1);
|
||||
}
|
||||
first_failed_object.get_or_insert_with(|| object.to_string());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
if take_failure_log_sample(&mut failure_samples_logged) {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt,
|
||||
error = %err,
|
||||
result = "object_failed",
|
||||
"Heal bucket object repair failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(scanned, healed, failed, bytes);
|
||||
}
|
||||
pending = retry;
|
||||
retry_attempt = retry_attempt.saturating_add(1);
|
||||
}
|
||||
|
||||
if !is_truncated {
|
||||
break;
|
||||
}
|
||||
|
||||
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
|
||||
if continuation_token.is_none() {
|
||||
// Truncated but no continuation token: end of listing.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
let failure = BatchHealFailure {
|
||||
scope: format!("bucket:{bucket}"),
|
||||
failed,
|
||||
retryable: retryable_failed,
|
||||
permanent: permanent_failed,
|
||||
first_object: first_failed_object.unwrap_or_default(),
|
||||
first_error: first_error.unwrap_or_default(),
|
||||
};
|
||||
return Err(self.record_batch_failure(failure).await);
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
prefix,
|
||||
scanned,
|
||||
healed,
|
||||
failed,
|
||||
bytes_processed = bytes,
|
||||
result = "recursive_ok",
|
||||
"Heal bucket recursive pass completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> {
|
||||
let baseline = match self
|
||||
.await_with_control(self.storage.erasure_set_usage_baseline(buckets))
|
||||
.await
|
||||
{
|
||||
Ok(Some(baseline)) => baseline,
|
||||
Ok(None) => return Ok(()),
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(_) => return Ok(()),
|
||||
};
|
||||
|
||||
let HealBucketUsageBaseline { objects_count, bytes } = baseline;
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_total_baseline(objects_count, bytes);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,506 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// erasure-set heal: drives the ErasureSetHealer across the set's buckets
|
||||
use super::*;
|
||||
|
||||
impl HealTask {
|
||||
pub(super) async fn heal_erasure_set(&self, buckets: Vec<String>, set_disk_id: String) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
bucket_count = buckets.len(),
|
||||
stage = "start",
|
||||
"Heal erasure set started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("erasure_set: {} ({} buckets)", set_disk_id, buckets.len())));
|
||||
progress.update_progress(0, 4, 0, 0);
|
||||
}
|
||||
|
||||
let is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty();
|
||||
let replacement_resume_disk = if is_auto_replacement {
|
||||
let mut requested_targets = self.heal_endpoints.clone();
|
||||
requested_targets.sort_unstable();
|
||||
requested_targets.dedup();
|
||||
let selection = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.get_replacement_resume_disk(&set_disk_id, &self.id, &self.heal_endpoints),
|
||||
)
|
||||
.await?;
|
||||
let disk = match selection {
|
||||
crate::heal::storage::ReplacementResumeDisk::Existing(disk) => {
|
||||
if let Some(anchor) = &self.replacement_resume_endpoint
|
||||
&& disk.endpoint().to_string() != *anchor
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement resume anchor changed for automatic heal {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
Some(disk)
|
||||
}
|
||||
crate::heal::storage::ReplacementResumeDisk::Fresh => {
|
||||
if self.replacement_resume_endpoint.is_some() {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement resume anchor is unavailable for automatic heal {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(disk) = disk.as_ref()
|
||||
&& ResumeManager::has_replacement_intent(disk, &self.id).await
|
||||
{
|
||||
let resume_manager = ResumeManager::load_replacement_intent(disk.clone(), &self.id).await?;
|
||||
let state = resume_manager.get_state().await;
|
||||
if state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::CleanupPending)
|
||||
&& state.set_disk_id == set_disk_id
|
||||
&& state.replacement_targets == requested_targets
|
||||
&& state.replacement_generation.as_deref() == Some(self.id.as_str())
|
||||
{
|
||||
resume_manager.ensure_replacement_completion_proof().await?;
|
||||
if CheckpointManager::has_checkpoint(disk, &self.id).await {
|
||||
CheckpointManager::load_from_disk(disk.clone(), &self.id)
|
||||
.await?
|
||||
.cleanup()
|
||||
.await?;
|
||||
}
|
||||
resume_manager.cleanup().await?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
disk
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if is_auto_replacement
|
||||
&& !self
|
||||
.await_with_control(self.storage.replacement_targets_ready(&self.heal_endpoints))
|
||||
.await?
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement target is no longer ready for automatic heal {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
|
||||
let replacement_resume_disk = if is_auto_replacement {
|
||||
Some(match replacement_resume_disk {
|
||||
Some(disk) => disk,
|
||||
None => {
|
||||
self.await_with_control(self.storage.get_disk_for_resume_excluding(&set_disk_id, &self.heal_endpoints))
|
||||
.await?
|
||||
}
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut buckets = if buckets.is_empty() {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "list_buckets",
|
||||
"Heal erasure set bucket list resolved"
|
||||
);
|
||||
let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
|
||||
bucket_infos.into_iter().map(|info| info.name).collect()
|
||||
} else {
|
||||
buckets
|
||||
};
|
||||
|
||||
// Persist automatic replacement intent on a surviving disk before the
|
||||
// first target format write. A task retry keeps this id; a newly
|
||||
// admitted blank replacement gets a fresh id and cannot reuse cursor
|
||||
// progress from an older disk at the same endpoint.
|
||||
let replacement_resume = if is_auto_replacement {
|
||||
let identities = self
|
||||
.await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints))
|
||||
.await?;
|
||||
let disk = replacement_resume_disk.clone().ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Replacement resume disk is missing for automatic heal {set_disk_id}"),
|
||||
})?;
|
||||
let manager = ResumeManager::new_replacement_intent(
|
||||
disk.clone(),
|
||||
self.id.clone(),
|
||||
set_disk_id.clone(),
|
||||
buckets.clone(),
|
||||
self.heal_endpoints.clone(),
|
||||
identities.clone(),
|
||||
)
|
||||
.await?;
|
||||
buckets = manager.get_state().await.replacement_buckets;
|
||||
Some((disk, manager, identities))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
self.apply_erasure_set_usage_baseline(&buckets).await?;
|
||||
|
||||
let healing_marker = format!("{set_disk_id}:{}", self.id);
|
||||
if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() {
|
||||
let state = resume_manager.get_state().await;
|
||||
if state.completed && matches!(state.replacement_phase, ReplacementPhase::Verified) {
|
||||
resume_manager.ensure_replacement_completion_proof().await?;
|
||||
super::super::clear_healing_markers_after_verified(&self.heal_endpoints, &healing_marker).await?;
|
||||
resume_manager.mark_replacement_cleanup_pending().await?;
|
||||
if CheckpointManager::has_checkpoint(disk, &self.id).await {
|
||||
CheckpointManager::load_from_disk(disk.clone(), &self.id)
|
||||
.await?
|
||||
.cleanup()
|
||||
.await?;
|
||||
}
|
||||
resume_manager.cleanup().await?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
// Step 1: Perform disk format heal using ecstore
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "heal_format",
|
||||
"Heal erasure set stage entered"
|
||||
);
|
||||
if is_auto_replacement {
|
||||
let Some((_, _, expected_identities)) = replacement_resume.as_ref() else {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement intent is missing for automatic heal {set_disk_id}"),
|
||||
});
|
||||
};
|
||||
self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "format")
|
||||
.await?;
|
||||
}
|
||||
let format_result = if is_auto_replacement {
|
||||
let pool_index = self.options.pool_index.ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Missing pool scope for automatic replacement heal {set_disk_id}"),
|
||||
})?;
|
||||
let set_index = self.options.set_index.ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Missing set scope for automatic replacement heal {set_disk_id}"),
|
||||
})?;
|
||||
self.await_with_control(self.storage.heal_replacement_format(
|
||||
self.options.dry_run,
|
||||
pool_index,
|
||||
set_index,
|
||||
&self.heal_endpoints,
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
self.await_with_control(self.storage.heal_format(self.options.dry_run)).await
|
||||
};
|
||||
|
||||
match format_result {
|
||||
Ok((result, error)) => {
|
||||
if let Some(e) = error {
|
||||
if Self::is_no_heal_required_error(&e) {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
result = "format_noop",
|
||||
"Heal erasure set format repair skipped because no format heal was required"
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
result = "format_failed",
|
||||
error = %e,
|
||||
"Heal erasure set failed"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
drives_healed = result.drives_healed(),
|
||||
drives_total = result.drives_reported(),
|
||||
result = "format_ok",
|
||||
"Heal erasure set format repaired"
|
||||
);
|
||||
}
|
||||
if !self.options.dry_run && !target_outcomes_complete(&result, &self.heal_endpoints) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to verify formatted replacement targets for {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
if let Some((_, replacement_resume, expected_identities)) = &replacement_resume {
|
||||
let identities = self
|
||||
.await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints))
|
||||
.await?;
|
||||
if !replacement_target_identities_match(expected_identities, &identities) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement target changed after format for automatic heal {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
replacement_resume.mark_replacement_rebuilding(identities).await?;
|
||||
}
|
||||
}
|
||||
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
result = "format_failed",
|
||||
error = %e,
|
||||
"Heal erasure set failed"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(1, 4, 0, 0);
|
||||
}
|
||||
|
||||
// The rebuilt disks are formatted now: mark them as healing so
|
||||
// DiskInfo.healing reflects the rebuild until it completes.
|
||||
super::super::set_healing_markers(&self.heal_endpoints, &healing_marker).await?;
|
||||
|
||||
// Step 2: Get disk for resume functionality
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "resolve_resume_disk",
|
||||
"Heal erasure set stage entered"
|
||||
);
|
||||
let replacement_target_identities = replacement_resume.as_ref().map(|(_, _, identities)| identities.clone());
|
||||
let disk = match replacement_resume.as_ref() {
|
||||
Some((disk, _, _)) => disk.clone(),
|
||||
None => {
|
||||
self.await_with_control(self.storage.get_disk_for_resume(&set_disk_id))
|
||||
.await?
|
||||
}
|
||||
};
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(2, 4, 0, 0);
|
||||
}
|
||||
|
||||
// Step 3: Heal bucket structure
|
||||
// Check control flags before each iteration to ensure timely cancellation.
|
||||
let bucket_heal_opts = HealOpts {
|
||||
recursive: false,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: false,
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
for bucket in buckets.iter() {
|
||||
// Check control flags before starting each bucket heal
|
||||
self.check_control_flags().await?;
|
||||
if let Some(expected_identities) = replacement_target_identities.as_ref() {
|
||||
self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "bucket prepass")
|
||||
.await?;
|
||||
}
|
||||
let heal_result = self
|
||||
.await_with_control(self.storage.heal_bucket(bucket, &bucket_heal_opts))
|
||||
.await;
|
||||
match heal_result {
|
||||
Ok(result) => {
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
bucket,
|
||||
result = "bucket_failed",
|
||||
error = %err,
|
||||
"Heal erasure set bucket prepass failed"
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create erasure set healer with resume support
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "build_resumable_healer",
|
||||
"Heal erasure set stage entered"
|
||||
);
|
||||
let heal_opts = HealOpts {
|
||||
recursive: self.options.recursive,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: self.options.remove_corrupted,
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
let erasure_healer = ErasureSetHealer::new(
|
||||
self.storage.clone(),
|
||||
self.progress.clone(),
|
||||
self.cancel_token.clone(),
|
||||
disk,
|
||||
heal_opts,
|
||||
self.source,
|
||||
)
|
||||
.with_replacement_targets(self.heal_endpoints.clone(), is_auto_replacement.then(|| self.id.clone()))
|
||||
.with_replacement_identity_fence(replacement_target_identities.clone());
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 4, 0, 0);
|
||||
}
|
||||
|
||||
// Step 4: Execute erasure set heal with resume
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "execute_resumable_heal",
|
||||
"Heal erasure set stage entered"
|
||||
);
|
||||
let result = self
|
||||
.await_with_control(erasure_healer.heal_erasure_set(&buckets, &set_disk_id))
|
||||
.await;
|
||||
|
||||
// Keep the markers on failure: the resume state also persists, and the
|
||||
// next run of this set heal re-marks and eventually clears them.
|
||||
let result = match result {
|
||||
Ok(()) => {
|
||||
if let Some(expected_identities) = replacement_target_identities.as_ref() {
|
||||
self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "marker completion")
|
||||
.await?;
|
||||
}
|
||||
super::super::clear_healing_markers_after_verified(&self.heal_endpoints, &healing_marker).await?;
|
||||
if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() {
|
||||
resume_manager.mark_replacement_cleanup_pending().await?;
|
||||
if CheckpointManager::has_checkpoint(disk, &self.id).await {
|
||||
CheckpointManager::load_from_disk(disk.clone(), &self.id)
|
||||
.await?
|
||||
.cleanup()
|
||||
.await?;
|
||||
}
|
||||
resume_manager.cleanup().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
let bytes_processed = progress.bytes_processed;
|
||||
progress.update_progress(4, 4, 0, bytes_processed);
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
bucket_count = buckets.len(),
|
||||
result = "ok",
|
||||
"Heal erasure set repaired"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal erasure set failed"
|
||||
);
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal erasure set {set_disk_id}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user