mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-09 13:46:05 +00:00
Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8552c9f8aa | |||
| 27d66c159f | |||
| e546ae9c62 | |||
| 35b5cfcf8d | |||
| d7c2fc7587 | |||
| 7db206466b | |||
| e549252ac6 | |||
| 17807b05fb | |||
| 789f1832a4 | |||
| a722fa80d5 | |||
| 9ecb500cbf | |||
| 8462b3492b | |||
| ac44f8968e | |||
| 46907c05cf |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=f0d15f2d1183be319d9d977d20b48bb05a29a424c266eed910c67aaa6f1ff955
|
||||
sha256-linux=340aa702576ebed5266b7c47e591f267a11178fa04bf3828d64f53fe92eb0907
|
||||
sha256-darwin=f0c78fdb93471575d9a64c5c46eae6c806bdd0bc10a6e33d7fb574aabd8db5a3
|
||||
sha256-linux=03ed7016cab672de9320e31375a0358eceacb4408b0e79cf063614fa7c878b87
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=12d30fff5ed48fe95bbfb310dd507048f81782b2954e00eb72435708b1133f9c
|
||||
sha256-linux=e917b2fdb303d01e6008ac4f1836c7b268698afe8383bd2aa4e697061df85cf6
|
||||
sha256-darwin=364f2329a7b72eb9f1608dbe1a3af37af4095354014f3cbe23ca448492d89961
|
||||
sha256-linux=60983f1ebe7068cf660d473c5f76c76a650410ccc99d71934ddca7fd67607987
|
||||
|
||||
@@ -89,6 +89,7 @@ offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offl
|
||||
test-wiring-check: ## Check tests stay registered and selected by their intended runners
|
||||
@echo "🧪 Checking test wiring..."
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --check-workflow
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
||||
|
||||
@@ -39,6 +39,7 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_python_bin.sh
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
|
||||
|
||||
@@ -197,6 +197,12 @@ test-group = 'e2e-cluster-nightly'
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# This four-disk, 65-member rollback probe already drives up to 32 concurrent
|
||||
# durable deletions. Reserve this nextest run's capacity for its progress oracle.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(=store::init::tests::dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"schema": 2,
|
||||
"schema": 1,
|
||||
"cases": {
|
||||
"background-target-restart": {
|
||||
"gate": "G14",
|
||||
@@ -13,7 +13,6 @@
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"erasure": {"data_blocks": 2, "parity_blocks": 2},
|
||||
"scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
},
|
||||
"background-target-crash": {
|
||||
@@ -28,284 +27,30 @@
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"erasure": {"data_blocks": 2, "parity_blocks": 2},
|
||||
"scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
},
|
||||
"ec84-target-drive-restart": {
|
||||
"gate": "G14",
|
||||
"task": "W20/W21",
|
||||
"lane": "e2e-distributed",
|
||||
"suite": "e2e_test",
|
||||
"name": "distributed::heal_test::three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart",
|
||||
"oracle": "ec84-target-drive-restart.json",
|
||||
"evidence": "process-restart",
|
||||
"unclean_shutdown_marker": false,
|
||||
"min_objects": 5,
|
||||
"max_objects": 5,
|
||||
"topology": {"nodes": 3, "drives_per_node": 4},
|
||||
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||
"erasure_set_drive_count": 12,
|
||||
"scope": "3-node x 4-drive single-set EC8+4, graceful target restart, preformatted replacement drive, exact unversioned S3 bodies and physical target shards; not mixed-version, multi-pool or long-window ABBA."
|
||||
},
|
||||
"background-target-restart-ec8-4": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_restart",
|
||||
"oracle": "background-target-restart-ec8-4.json",
|
||||
"evidence": "process-restart",
|
||||
"unclean_shutdown_marker": false,
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 3, "drives_per_node": 4},
|
||||
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||
"erasure_set_drive_count": 12,
|
||||
"scope": "Target process restart during partial background rebuild on a single 3x4 EC8+4 set; exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
|
||||
},
|
||||
"background-target-crash-ec8-4": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_crash",
|
||||
"oracle": "background-target-crash-ec8-4.json",
|
||||
"evidence": "process-crash-restart",
|
||||
"unclean_shutdown_marker": true,
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 3, "drives_per_node": 4},
|
||||
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||
"erasure_set_drive_count": 12,
|
||||
"scope": "Target process killed during partial background rebuild on a single 3x4 EC8+4 set; real unclean-shutdown marker, exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
|
||||
}
|
||||
},
|
||||
"release_lanes": {
|
||||
"single-set-restart": {
|
||||
"status": "implemented",
|
||||
"cases": ["background-target-restart", "background-target-crash"],
|
||||
"covers": ["four-node one-drive topology", "unversioned objects", "target restart/crash"]
|
||||
},
|
||||
"authority-coverage": {
|
||||
"status": "pending",
|
||||
"gates": ["G01", "G12"],
|
||||
"requires": ["root authority coverage", "quota authority coverage"]
|
||||
},
|
||||
"checkpoint-and-crash": {
|
||||
"status": "pending",
|
||||
"gates": ["G02", "G04", "R-E"],
|
||||
"requires": ["bounded checkpoint progress", "boundary crash matrix", "fixed-budget restart evidence"]
|
||||
},
|
||||
"status-and-outcome": {
|
||||
"status": "pending",
|
||||
"gates": ["G05", "G06", "R-D"],
|
||||
"requires": ["per-object outcomes", "legacy status clients", "manager/event/ledger disposition"]
|
||||
},
|
||||
"mrf-responsibility": {
|
||||
"status": "pending",
|
||||
"gates": ["G07", "G08", "P4"],
|
||||
"requires": ["durable MRF responsibility", "disk-full and replica-loss matrix", "MRF replay cost"]
|
||||
},
|
||||
"mixed-version-rollback": {
|
||||
"status": "pending",
|
||||
"gates": ["G03", "G09", "R-L"],
|
||||
"requires": ["mixed-version peers", "rollback payloads", "crash-safe source retirement"]
|
||||
},
|
||||
"scheduler-pressure": {
|
||||
"status": "pending",
|
||||
"gates": ["G10", "P1", "P2", "P3"],
|
||||
"requires": ["bounded scheduling", "foreground latency and throughput", "two-hour pressure evidence"]
|
||||
},
|
||||
"maintenance-producers": {
|
||||
"status": "pending",
|
||||
"gates": ["G11", "G13"],
|
||||
"requires": ["complete producer coverage", "quorum-minus-one and remount matrix"]
|
||||
},
|
||||
"ec8-4-multiset": {
|
||||
"status": "pending",
|
||||
"gates": ["G14"],
|
||||
"requires": ["3x4 EC8+4 topology", "multi-set coverage", "multi-pool coverage"]
|
||||
}
|
||||
},
|
||||
"release_requirements": [
|
||||
{
|
||||
"gate": "G01",
|
||||
"task": "W02/W04",
|
||||
"lane": "authority-coverage",
|
||||
"status": "pending",
|
||||
"description": "Complete root and quota authority coverage",
|
||||
"requires": ["root authority evidence", "quota authority evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G02",
|
||||
"task": "W03",
|
||||
"lane": "checkpoint-and-crash",
|
||||
"status": "pending",
|
||||
"description": "Bounded checkpoint progress and independent version inventory",
|
||||
"requires": ["bounded checkpoint oracle", "independent version inventory"]
|
||||
},
|
||||
{
|
||||
"gate": "G03",
|
||||
"task": "W17/W18",
|
||||
"lane": "mixed-version-rollback",
|
||||
"status": "pending",
|
||||
"description": "Exact scoped ACK with durable publication and mixed peers",
|
||||
"requires": ["durable scoped ACK publication", "mixed-peer evidence"],
|
||||
"evidence_fields": [
|
||||
"durable_root_publication_proof",
|
||||
"scoped_ack_request_identity",
|
||||
"participating_peer_capability_snapshot",
|
||||
"mixed_peer_ack_fallback_oracle"
|
||||
]
|
||||
},
|
||||
{
|
||||
"gate": "G04",
|
||||
"task": "W03/W15/W16",
|
||||
"lane": "checkpoint-and-crash",
|
||||
"status": "pending",
|
||||
"description": "Crash at every cache, root, floor and intent boundary",
|
||||
"requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G05",
|
||||
"task": "W06/W07",
|
||||
"lane": "status-and-outcome",
|
||||
"status": "pending",
|
||||
"description": "Per-object outcomes and bounded terminal retention",
|
||||
"requires": ["per-object outcome oracle", "terminal retention bounds"]
|
||||
},
|
||||
{
|
||||
"gate": "G06",
|
||||
"task": "W06/W08/W23",
|
||||
"lane": "status-and-outcome",
|
||||
"status": "pending",
|
||||
"description": "Concurrent status, legacy clients and truncation",
|
||||
"requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"]
|
||||
},
|
||||
{
|
||||
"gate": "G07",
|
||||
"task": "W12/W13/W14",
|
||||
"lane": "mrf-responsibility",
|
||||
"status": "pending",
|
||||
"description": "Durable MRF responsibility at every commit boundary",
|
||||
"requires": ["MRF responsibility oracle", "commit-boundary crash matrix"]
|
||||
},
|
||||
{
|
||||
"gate": "G08",
|
||||
"task": "W12/W13/W14",
|
||||
"lane": "mrf-responsibility",
|
||||
"status": "pending",
|
||||
"description": "MRF capacity, disk-full and replica-loss matrix",
|
||||
"requires": ["MRF capacity evidence", "disk-full matrix", "replica-loss matrix"]
|
||||
},
|
||||
{
|
||||
"gate": "G09",
|
||||
"task": "W13/W18/W23",
|
||||
"lane": "mixed-version-rollback",
|
||||
"status": "pending",
|
||||
"description": "Actual mixed-version reader/writer and rollback payloads",
|
||||
"requires": ["mixed-version reader evidence", "mixed-version writer evidence", "rollback payload evidence"],
|
||||
"evidence_fields": [
|
||||
"mixed_version_reader_evidence",
|
||||
"mixed_version_writer_evidence",
|
||||
"rollback_payload_evidence"
|
||||
]
|
||||
},
|
||||
{
|
||||
"gate": "G10",
|
||||
"task": "W05/W09/W10/W11",
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Bounded scheduling and pressure recovery",
|
||||
"requires": ["scheduler bound evidence", "pressure recovery evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G11",
|
||||
"task": "W04/W19/W24",
|
||||
"lane": "maintenance-producers",
|
||||
"status": "pending",
|
||||
"description": "Maintenance and complete producer coverage",
|
||||
"requires": ["maintenance producer matrix", "complete producer inventory"]
|
||||
},
|
||||
{
|
||||
"gate": "G12",
|
||||
"task": "W02/W15/W16",
|
||||
"lane": "authority-coverage",
|
||||
"status": "pending",
|
||||
"description": "Both quota paths during reset and settlement",
|
||||
"requires": ["reset quota-path evidence", "settlement quota-path evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G13",
|
||||
"task": "W07/W14",
|
||||
"lane": "maintenance-producers",
|
||||
"status": "pending",
|
||||
"description": "Quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
||||
"requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G14",
|
||||
"task": "W20/W21",
|
||||
"lane": "ec8-4-multiset",
|
||||
"status": "pending",
|
||||
"description": "Same-window field evidence with 3x4 EC8+4 and multi-set/pool coverage",
|
||||
"requires": ["same-window field evidence", "3x4 EC8+4 evidence", "multi-set evidence", "multi-pool evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "P1",
|
||||
"task": "W20",
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Measured cold-walk share and foreground latency/throughput",
|
||||
"requires": ["cold-walk share measurement", "foreground latency/throughput measurement"]
|
||||
},
|
||||
{
|
||||
"gate": "P2",
|
||||
"task": "W20/W24",
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Measured post-stop convergence and cold segment reuse",
|
||||
"requires": ["post-stop convergence measurement", "cold segment reuse measurement"]
|
||||
},
|
||||
{
|
||||
"gate": "P3",
|
||||
"task": "W20",
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Measured two-hour pressure/heal capacity and recovery window",
|
||||
"requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"]
|
||||
},
|
||||
{
|
||||
"gate": "P4",
|
||||
"task": "W20",
|
||||
"lane": "mrf-responsibility",
|
||||
"status": "pending",
|
||||
"description": "Measured MRF scale and replay cost with retained responsibility",
|
||||
"requires": ["MRF scale measurement", "MRF replay-cost measurement", "retained responsibility evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "R-E",
|
||||
"task": "W03/W05",
|
||||
"lane": "checkpoint-and-crash",
|
||||
"status": "pending",
|
||||
"description": "Fixed-budget real process restart through enumeration and classification",
|
||||
"requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "R-D",
|
||||
"task": "W07/W14",
|
||||
"lane": "status-and-outcome",
|
||||
"status": "pending",
|
||||
"description": "Manager-to-event-to-ledger exact disposition, including grace",
|
||||
"requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"]
|
||||
},
|
||||
{
|
||||
"gate": "R-L",
|
||||
"task": "W13/W14",
|
||||
"lane": "mixed-version-rollback",
|
||||
"status": "pending",
|
||||
"description": "Legacy source conflicts, migration gaps and crash-safe source retirement",
|
||||
"requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"]
|
||||
}
|
||||
]
|
||||
"release_pending": {
|
||||
"G01": "W02/W04 complete root and quota authority coverage",
|
||||
"G02": "W03 bounded checkpoint progress and independent version inventory",
|
||||
"G03": "W17/W18 exact scoped ACK with durable publication and mixed peers",
|
||||
"G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary",
|
||||
"G05": "W06/W07 per-object outcomes and bounded terminal retention",
|
||||
"G06": "W06/W08/W23 concurrent status, legacy clients and truncation",
|
||||
"G07": "W12/W13/W14 durable MRF responsibility at every commit boundary",
|
||||
"G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix",
|
||||
"G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads",
|
||||
"G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery",
|
||||
"G11": "W04/W19/W24 maintenance and complete producer coverage",
|
||||
"G12": "W02/W15/W16 both quota paths during reset and settlement",
|
||||
"G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
||||
"G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage",
|
||||
"P1": "W20 measured cold-walk share and foreground latency/throughput",
|
||||
"P2": "W20/W24 measured post-stop convergence and cold segment reuse",
|
||||
"P3": "W20 measured two-hour pressure/heal capacity and recovery window",
|
||||
"P4": "W20 measured MRF scale and replay cost with retained responsibility",
|
||||
"R-E": "W03/W05 fixed-budget real process restart through enumeration and classification",
|
||||
"R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace",
|
||||
"R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,10 +111,6 @@ runs:
|
||||
shell: bash
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
- name: Check CI paths stay in sync
|
||||
shell: bash
|
||||
run: ./scripts/check_ci_paths_sync.sh
|
||||
|
||||
- name: Check io_uring lane --lib precondition
|
||||
shell: bash
|
||||
run: ./scripts/check_uring_lane_lib_only.sh
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Security Audit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, release ]
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- '**/Cargo.toml'
|
||||
- '**/Cargo.lock'
|
||||
@@ -32,7 +32,7 @@ on:
|
||||
- 'scripts/security/check_workflow_pins.sh'
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
branches: [ main, release ]
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- '**/Cargo.toml'
|
||||
- '**/Cargo.lock'
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Reports the existing required checks for paths excluded by ci.yml.
|
||||
# Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared
|
||||
# action to keep validation coverage aligned. Keep this paths list in sync with
|
||||
# ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh.
|
||||
|
||||
name: Continuous Integration (docs only)
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened ]
|
||||
branches: [ main, release ]
|
||||
paths:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
- "deploy/**"
|
||||
- "scripts/dev_*.sh"
|
||||
- "scripts/probe.sh"
|
||||
- "LICENSE*"
|
||||
- ".gitignore"
|
||||
- ".dockerignore"
|
||||
- "README*"
|
||||
- "**/*.png"
|
||||
- "**/*.jpg"
|
||||
- "**/*.svg"
|
||||
- ".github/workflows/build.yml"
|
||||
- ".github/workflows/docker.yml"
|
||||
- ".github/workflows/audit.yml"
|
||||
- "flake.lock"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
quick-checks:
|
||||
name: Quick Checks
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
test-and-lint:
|
||||
name: Test and Lint
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Docs-only PRs skip the full code CI, but they are exactly where a
|
||||
# planning-type document could be slipped in (git add -f bypasses
|
||||
# .gitignore). Run the guard here so the required "Test and Lint" check
|
||||
# stays meaningful for docs-only changes.
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
- name: Satisfy required check for docs-only changes
|
||||
run: echo "Docs-only change — code CI is skipped by paths-ignore; planning-docs guard passed, reporting success for the required 'Test and Lint' check."
|
||||
+81
-90
@@ -16,7 +16,7 @@ name: Continuous Integration
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, release ]
|
||||
branches: [ main ]
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
@@ -36,26 +36,7 @@ on:
|
||||
- "flake.lock"
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
branches: [ main, release ]
|
||||
# Keep this list in sync with the `paths` list in ci-docs-only.yml, which
|
||||
# reports the required "Test and Lint" check for PRs skipped here.
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
- "deploy/**"
|
||||
- "scripts/dev_*.sh"
|
||||
- "scripts/probe.sh"
|
||||
- "LICENSE*"
|
||||
- ".gitignore"
|
||||
- ".dockerignore"
|
||||
- "README*"
|
||||
- "**/*.png"
|
||||
- "**/*.jpg"
|
||||
- "**/*.svg"
|
||||
- ".github/workflows/build.yml"
|
||||
- ".github/workflows/docker.yml"
|
||||
- ".github/workflows/audit.yml"
|
||||
- "flake.lock"
|
||||
branches: [ main ]
|
||||
merge_group:
|
||||
types: [ checks_requested ]
|
||||
schedule:
|
||||
@@ -88,6 +69,32 @@ jobs:
|
||||
- name: Explain cancellation run
|
||||
run: echo "PR closed; this run only cancels older runs in the same concurrency group."
|
||||
|
||||
classify-changes:
|
||||
name: Select CI scope
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
mode: ${{ steps.scope.outputs.mode }}
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
fetch-depth: 2
|
||||
persist-credentials: false
|
||||
- name: Select scope using the base revision's policy
|
||||
id: scope
|
||||
env:
|
||||
CI_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
elif [[ "$CI_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && git show "$CI_BASE_SHA:scripts/ci_gate.py" > "$RUNNER_TEMP/ci-gate-base.py"; then
|
||||
python3 -I "$RUNNER_TEMP/ci-gate-base.py" select
|
||||
else
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
|
||||
typos:
|
||||
name: Typos
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
@@ -100,7 +107,7 @@ jobs:
|
||||
- name: Typos check with custom config file
|
||||
uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
|
||||
|
||||
# Fail early with compile-free checks shared with docs-only CI.
|
||||
# Fail early with compile-free checks for every pull request.
|
||||
quick-checks:
|
||||
name: Quick Checks
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
@@ -116,9 +123,9 @@ jobs:
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
test-and-lint:
|
||||
name: Test and Lint
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
name: Workspace Test and Lint
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
@@ -289,45 +296,6 @@ jobs:
|
||||
- name: Run rebalance/decommission migration proofs
|
||||
run: ./scripts/check_migration_gate_count.sh
|
||||
|
||||
# Record the reason before this job completes as FAILURE. A separate
|
||||
# dependent job cancels sibling lanes only after GitHub has preserved this
|
||||
# required check's failure verdict.
|
||||
- name: Annotate early-stop reason
|
||||
if: >-
|
||||
failure() && github.event_name == 'pull_request'
|
||||
&& github.event.pull_request.head.repo.full_name == github.repository
|
||||
run: |
|
||||
{
|
||||
echo "## CI early-stop"
|
||||
echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; a follow-up job will cancel sibling lanes to free runners."
|
||||
echo "Sibling jobs showing **cancelled** were stopped by the early-stop follow-up, not by their own failure."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Preserve the required Test and Lint FAILURE verdict before stopping sibling
|
||||
# lanes. Cancelling from inside test-and-lint changed its own conclusion to
|
||||
# CANCELLED and hid the actionable failure in the PR checks UI.
|
||||
cancel-after-test-and-lint-failure:
|
||||
name: Cancel siblings after Test and Lint failure
|
||||
if: >-
|
||||
failure() && needs.test-and-lint.result == 'failure'
|
||||
&& github.event_name == 'pull_request'
|
||||
&& github.event.pull_request.head.repo.full_name == github.repository
|
||||
needs: [ test-and-lint ]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
actions: write
|
||||
steps:
|
||||
- name: Cancel remaining jobs
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
curl -fsS -X POST \
|
||||
-H "Authorization: Bearer ${GH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel"
|
||||
|
||||
# Dedicated serial lane for the ILM / lifecycle integration tests. These tests
|
||||
# drive the object layer through process-global singletons (the GLOBAL_ENV
|
||||
# ECStore, the global tier-config manager, background-expiry workers) and bind
|
||||
@@ -340,8 +308,8 @@ jobs:
|
||||
# See rustfs/backlog#1148 (ilm-1) and #1155.
|
||||
test-ilm-integration-serial:
|
||||
name: ILM Integration (serial)
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
@@ -408,8 +376,8 @@ jobs:
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
@@ -449,8 +417,8 @@ jobs:
|
||||
|
||||
connect-short-credential-boundary:
|
||||
name: Connect Short Credential Boundary
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
@@ -507,8 +475,8 @@ jobs:
|
||||
|
||||
test-and-lint-protocols:
|
||||
name: "Test and Lint (${{ matrix.features.name }})"
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
@@ -561,8 +529,8 @@ jobs:
|
||||
|
||||
build-rustfs-debug-binary:
|
||||
name: Build RustFS Debug Binary
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
@@ -684,8 +652,8 @@ jobs:
|
||||
# job had neither, so each closed/merged PR really ran the whole io_uring
|
||||
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
|
||||
# 30662728539) and kept the cancellation run in progress for minutes.
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
|
||||
# a container, applies no seccomp filter that would block io_uring_setup — so
|
||||
# the probe succeeds and the tests exercise the real UringBackend/FdCache/
|
||||
@@ -872,14 +840,13 @@ jobs:
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
|
||||
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
|
||||
# every PR, so it is gated to main/release pushes, the merge queue, and manual
|
||||
# every PR, so it is gated to main pushes, the merge queue, and manual
|
||||
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
|
||||
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'push' &&
|
||||
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release'))
|
||||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 55
|
||||
@@ -1002,22 +969,11 @@ jobs:
|
||||
# debug binary; each test spawns its own rustfs server on a random port.
|
||||
- name: Run e2e full suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-full-logs
|
||||
RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG: rustfs::heal=debug,rustfs_heal=debug,rustfs_ecstore::disk=debug,rustfs_lock=debug,rustfs=error
|
||||
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence
|
||||
run: cargo nextest run --profile e2e-full -p e2e_test
|
||||
|
||||
- name: Upload e2e full diagnostics
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-full-diagnostics-${{ github.run_number }}
|
||||
path: ${{ runner.temp }}/rustfs-e2e-full-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload junit
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
@@ -1224,9 +1180,44 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
required-checks:
|
||||
name: Test and Lint
|
||||
if: always() && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs:
|
||||
- classify-changes
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- test-ilm-integration-serial
|
||||
- test-and-lint-rio-v2
|
||||
- connect-short-credential-boundary
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
- s3-implemented-tests
|
||||
- s3-lifecycle-behavior-tests
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- e2e-tests-rio-v2
|
||||
- e2e-full
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Require the expected result of every CI lane
|
||||
env:
|
||||
CI_NEEDS: ${{ toJSON(needs) }}
|
||||
shell: bash
|
||||
run: python3 scripts/ci_gate.py verify
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs:
|
||||
- classify-changes
|
||||
- connect-short-credential-boundary
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# Upgrade cases use an independent 1.0.0-rc.2 pin defined below.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
# independent `statfs` capacity. This job runs on GitHub-hosted
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
HTTP_PROXY: ""
|
||||
HTTPS_PROXY: ""
|
||||
# Independent 1.0.0-rc.2 source pin for distributed::upgrade_test.
|
||||
# Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
|
||||
UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
|
||||
UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
|
||||
UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
|
||||
|
||||
@@ -19,17 +19,27 @@ on:
|
||||
- cron: "7 0 * * *"
|
||||
timezone: "Asia/Shanghai"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
description: 'Branch/ref to build and publish as the nightly (empty = scheduled source, see NIGHTLY_BUILD_REF)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Scheduled builds follow the NIGHTLY_BRANCH repo variable so the channel can
|
||||
# be pointed at e.g. `release` for the GA cycle and back to `main` afterwards
|
||||
# without touching this file. Manual runs take the `branch` input, falling
|
||||
# back to the branch the run was dispatched from.
|
||||
concurrency:
|
||||
group: nightly-gnu-build-main-${{ github.event_name }}
|
||||
group: nightly-gnu-build-${{ github.event_name }}-${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
|
||||
cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
NIGHTLY_BUILD_REF: ${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@@ -43,6 +53,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -152,13 +163,104 @@ jobs:
|
||||
|
||||
fakeroot dpkg-deb --build "${PKG_DIR}"
|
||||
ls -lh "${DEB_FILE}"
|
||||
echo "deb_date=${DEB_DATE}" >> "${GITHUB_OUTPUT}"
|
||||
echo "deb_file=${DEB_FILE}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Same packaging scheme as .github/workflows/package.yml (fpm), but from
|
||||
# the locally built nightly binary instead of a release artifact, with a
|
||||
# date-based version that mirrors the DEB.
|
||||
- name: Build RPM package
|
||||
id: rpm
|
||||
shell: bash
|
||||
env:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v fpm >/dev/null 2>&1; then
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} apt-get update -qq && ${SUDO} apt-get install -y -qq ruby ruby-dev build-essential rpm >/dev/null
|
||||
${SUDO} gem install fpm --no-document >/dev/null
|
||||
fi
|
||||
|
||||
RPM_FILE="rustfs-nightly-${DEB_DATE}.rpm"
|
||||
RPM_VERSION="0"
|
||||
RPM_RELEASE="0.nightly.${DEB_DATE//-/.}"
|
||||
|
||||
echo "Building RPM: ${RPM_FILE} (version ${RPM_VERSION}-${RPM_RELEASE})"
|
||||
|
||||
# fpm wants the config file to exist before packaging.
|
||||
mkdir -p ./tmp-pkg/etc/default
|
||||
cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
--name rustfs \
|
||||
--version "$RPM_VERSION" \
|
||||
--iteration "$RPM_RELEASE" \
|
||||
--architecture x86_64 \
|
||||
--package "$RPM_FILE" \
|
||||
--depends "glibc >= 2.31" \
|
||||
--maintainer "RustFS Team <support@rustfs.com>" \
|
||||
--description "High-performance distributed object storage" \
|
||||
--url "https://rustfs.com" \
|
||||
--license "Apache-2.0" \
|
||||
--after-install <(cat << 'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTINST
|
||||
) \
|
||||
--before-remove <(cat << 'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
) \
|
||||
--after-remove <(cat << 'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
) \
|
||||
--config-files /etc/default/rustfs \
|
||||
"rustfs-nightly-${DEB_DATE}/usr/bin/rustfs=/usr/bin/rustfs" \
|
||||
./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
|
||||
deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
|
||||
LICENSE=/usr/share/doc/rustfs/LICENSE \
|
||||
README.md=/usr/share/doc/rustfs/README.md
|
||||
|
||||
[[ -f "$RPM_FILE" ]] || { echo "RPM build failed"; exit 1; }
|
||||
rpm -qpl "$RPM_FILE" | grep -Fx '/usr/bin/rustfs' >/dev/null
|
||||
stat --printf='%n %s bytes\n' "$RPM_FILE"
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload DEB artifact
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: ${{ steps.deb.outputs.deb_file }}
|
||||
path: ${{ steps.deb.outputs.deb_file }}
|
||||
- name: Upload RPM artifact
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: ${{ steps.rpm.outputs.rpm_file }}
|
||||
path: ${{ steps.rpm.outputs.rpm_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Persist the nightly deb on Cloudflare R2 (same channel as package.yml)
|
||||
@@ -187,11 +289,10 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# The candidate manifest must describe the tree that was actually
|
||||
# built. With a ref override (NIGHTLY_BRANCH / dispatch input) that
|
||||
# is not necessarily GITHUB_SHA, so always advertise HEAD.
|
||||
SOURCE_SHA="$(git rev-parse HEAD)"
|
||||
if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
|
||||
echo "Checkout SHA does not match the nightly build run" >&2
|
||||
exit 1
|
||||
fi
|
||||
DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
|
||||
CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
|
||||
CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
|
||||
@@ -247,6 +348,20 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
@@ -284,6 +399,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -372,6 +488,7 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
|
||||
@@ -82,6 +82,12 @@ jobs:
|
||||
performance-test:
|
||||
runs-on: pf-testing
|
||||
timeout-minutes: 900
|
||||
env:
|
||||
RUSTFS_BENCH_SCRIPT: ${{ github.workspace }}/auto-testing/rustfs_performance_testing.sh
|
||||
RUSTFS_WARP_METHODS: ${{ inputs.test_method }}
|
||||
RUSTFS_WARP_SIZES: ${{ inputs.object_size }}
|
||||
RUSTFS_WARP_DURATION: ${{ inputs.warp_duration || '5m' }}
|
||||
RUSTFS_WARP_CONCURRENCY: ${{ inputs.warp_concurrency || '64' }}
|
||||
# Run on manual dispatch, or when the nightly build completed successfully.
|
||||
# Skipped when nightly failed.
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
@@ -158,19 +164,15 @@ jobs:
|
||||
- name: Run benchmark (GET/PUT/MIXED)
|
||||
id: benchmark
|
||||
run: |
|
||||
# Empty on automatic (workflow_run) runs -> full 30 rounds.
|
||||
# Manual dispatch can restrict method(s)/size(s).
|
||||
export WARP_METHODS="${{ inputs.test_method }}"
|
||||
export WARP_SIZES="${{ inputs.object_size }}"
|
||||
./auto-testing/rustfs_performance_test.sh \
|
||||
--step 5 -y \
|
||||
--warp-duration "${{ inputs.warp_duration || '5m' }}" \
|
||||
--warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
|
||||
--log-file "${LOG_FILE}"
|
||||
|
||||
- name: Analyze results
|
||||
if: ${{ steps.benchmark.conclusion == 'success' }}
|
||||
run: |
|
||||
export WARP_METHODS="${RUSTFS_WARP_METHODS}" WARP_SIZES="${RUSTFS_WARP_SIZES}"
|
||||
export WARP_DURATION="${RUSTFS_WARP_DURATION}" WARP_CONCURRENCY="${RUSTFS_WARP_CONCURRENCY}"
|
||||
./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
|
||||
|
||||
- name: Collect RustFS version info
|
||||
|
||||
@@ -52,6 +52,7 @@ docs
|
||||
__pycache__/
|
||||
!docs/
|
||||
docs/*
|
||||
!docs/README.md
|
||||
!docs/architecture/
|
||||
!docs/architecture/**
|
||||
!docs/operations/
|
||||
|
||||
@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Replication
|
||||
|
||||
- Object Lock replication PUTs now carry a required integrity header, fixing target rejection introduced by the plain-payload default ([#7097](https://github.com/rustfs/rustfs/pull/7097)). This changes the default outbound request for locked objects but adds no persisted format.
|
||||
- Multipart source objects stay on the multipart transport even when their checksum record is a whole-object checksum, so objects above the single-PUT limit remain replicable ([#7047](https://github.com/rustfs/rustfs/pull/7047)).
|
||||
- Targets that mint their own version IDs now use a per-target version ledger for tag, retention, legal-hold, and permanent-delete mutations; ambiguous pre-ledger matches fail with backoff instead of guessing ([#7368](https://github.com/rustfs/rustfs/pull/7368)). This adds dual-prefixed internal metadata keys that older readers ignore.
|
||||
- Single-part source checksums are forwarded as `x-amz-checksum-*` headers instead of user metadata, so the replica preserves checksum responses ([#7313](https://github.com/rustfs/rustfs/pull/7313)). This changes the default outbound headers for checksummed objects.
|
||||
- Site-replication outage recovery now uses a bounded 30-second retry drain plus the 600-second full reconciliation pass, persists destructive liabilities before local deletion, and fences replay settlement and peer edits ([#7148](https://github.com/rustfs/rustfs/pull/7148)). Persisted additions are optional and ignored by older readers.
|
||||
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
|
||||
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
|
||||
|
||||
### Security
|
||||
- **Presigned URLs honour only signed headers** (GHSA-g8w9-qw9q-fghr): a SigV4 presigned request that carries an `x-amz-*` request header not listed in `X-Amz-SignedHeaders` is now rejected with `403 AccessDenied` ("There were headers present in the request which were not signed"), matching AWS S3. Previously the holder of a presigned `PutObject` URL could add unsigned `x-amz-tagging`, `x-amz-storage-class`, `x-amz-website-redirect-location`, ACL, metadata, Object Lock or SSE headers and have them applied. Presigners that intend a property must set it before signing so the SDK lists the header in `SignedHeaders`; `x-amz-cf-id` (CloudFront) remains tolerated unsigned. Header-signed SigV4 and SigV2 requests are unchanged.
|
||||
|
||||
|
||||
@@ -211,7 +211,10 @@ For developers who want to build RustFS Docker images from source with multi-arc
|
||||
|
||||
```bash
|
||||
# Build multi-architecture images locally
|
||||
./docker-buildx.sh --build-arg RELEASE=latest
|
||||
./docker-buildx.sh
|
||||
|
||||
# Build a single-platform image locally
|
||||
./docker-buildx.sh -p linux/amd64
|
||||
|
||||
# Build and push to registry
|
||||
./docker-buildx.sh --push
|
||||
|
||||
+4
-1
@@ -150,7 +150,10 @@ docker compose -f docker-compose-simple.yml up -d
|
||||
|
||||
```bash
|
||||
# 在本地构建多架构镜像
|
||||
./docker-buildx.sh --build-arg RELEASE=latest
|
||||
./docker-buildx.sh
|
||||
|
||||
# 在本地构建单平台镜像
|
||||
./docker-buildx.sh -p linux/amd64
|
||||
|
||||
# 构建并推送到仓库
|
||||
./docker-buildx.sh --push
|
||||
|
||||
@@ -145,38 +145,6 @@ pub fn consume_verified_mrf_repair_events(anchors: &mut Vec<MrfDurableRepairAnch
|
||||
before.saturating_sub(anchors.len())
|
||||
}
|
||||
|
||||
/// Consume recorded verified repairs for one bucket without draining
|
||||
/// unrelated or still-unmatched proofs. If the caller crashes before
|
||||
/// persisting the retained anchor set, the proof may be replayed by repair
|
||||
/// instead of silently deleting the old responsibility.
|
||||
pub fn consume_recorded_verified_mrf_repair_events_for(bucket: &str, anchors: &mut Vec<MrfDurableRepairAnchor>) -> usize {
|
||||
let Some(registry) = MRF_VERIFIED_REPAIR_EVENTS.get() else {
|
||||
return 0;
|
||||
};
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
return 0;
|
||||
};
|
||||
let before = anchors.len();
|
||||
let mut retained = std::collections::VecDeque::with_capacity(events.len());
|
||||
while let Some(event) = events.pop_front() {
|
||||
if event.bucket.as_ref() != bucket {
|
||||
retained.push_back(event);
|
||||
continue;
|
||||
}
|
||||
let mut matched = false;
|
||||
anchors.retain(|anchor| {
|
||||
let proven = anchor.is_proven_by(&event);
|
||||
matched |= proven;
|
||||
!proven
|
||||
});
|
||||
if !matched {
|
||||
retained.push_back(event);
|
||||
}
|
||||
}
|
||||
*events = retained;
|
||||
before.saturating_sub(anchors.len())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct MrfScope {
|
||||
pub pool_index: u32,
|
||||
@@ -853,71 +821,6 @@ mod tests {
|
||||
assert!(retained.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_verified_repair_consumer_retains_unmatched_proofs() {
|
||||
let bucket = Arc::<str>::from(format!("recorded-proof-{}", Uuid::new_v4()));
|
||||
let other_bucket = Arc::<str>::from(format!("recorded-proof-other-{}", Uuid::new_v4()));
|
||||
let incarnation = Uuid::new_v4();
|
||||
let lease = MrfIngressLease::new(21);
|
||||
let retained_anchor = MrfDurableRepairAnchor {
|
||||
kind: MrfKind::PartialWrite,
|
||||
bucket: bucket.clone(),
|
||||
object: Arc::from("retained"),
|
||||
version_id: Some([7; 16]),
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 1,
|
||||
set_index: 2,
|
||||
}),
|
||||
lease,
|
||||
bucket_incarnation_id: incarnation,
|
||||
};
|
||||
let waiting_anchor = MrfDurableRepairAnchor {
|
||||
object: Arc::from("waiting"),
|
||||
lease: MrfIngressLease::new(22),
|
||||
..retained_anchor.clone()
|
||||
};
|
||||
let matched_event = MrfVerifiedRepairEvent {
|
||||
kind: retained_anchor.kind,
|
||||
bucket: bucket.clone(),
|
||||
object: retained_anchor.object.clone(),
|
||||
version_id: retained_anchor.version_id,
|
||||
scope: retained_anchor.scope,
|
||||
lease: Some(retained_anchor.lease),
|
||||
bucket_incarnation_id: retained_anchor.bucket_incarnation_id,
|
||||
disposition: MrfVerifiedRepairDisposition::Repaired,
|
||||
};
|
||||
let same_bucket_unmatched = MrfVerifiedRepairEvent {
|
||||
object: Arc::from("future"),
|
||||
lease: Some(MrfIngressLease::new(23)),
|
||||
..matched_event.clone()
|
||||
};
|
||||
let other_bucket_event = MrfVerifiedRepairEvent {
|
||||
bucket: other_bucket.clone(),
|
||||
..matched_event.clone()
|
||||
};
|
||||
|
||||
note_mrf_verified_repair(matched_event);
|
||||
note_mrf_verified_repair(same_bucket_unmatched.clone());
|
||||
note_mrf_verified_repair(other_bucket_event.clone());
|
||||
|
||||
let mut anchors = vec![retained_anchor, waiting_anchor.clone()];
|
||||
assert_eq!(consume_recorded_verified_mrf_repair_events_for(&bucket, &mut anchors), 1);
|
||||
assert_eq!(anchors, vec![waiting_anchor]);
|
||||
|
||||
let remaining_bucket_events = take_mrf_verified_repair_events_for(&bucket);
|
||||
assert_eq!(
|
||||
remaining_bucket_events,
|
||||
vec![same_bucket_unmatched],
|
||||
"same-bucket proofs without a retained anchor must remain available"
|
||||
);
|
||||
let remaining_other_events = take_mrf_verified_repair_events_for(&other_bucket);
|
||||
assert_eq!(
|
||||
remaining_other_events,
|
||||
vec![other_bucket_event],
|
||||
"proofs for other buckets must not be drained by this consumer"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn try_send_delivers_and_respects_capacity() {
|
||||
let mut receiver = init_mrf_channel().expect("first initialization should succeed");
|
||||
|
||||
@@ -44,7 +44,6 @@ use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::error::Error;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::time::{Duration, Instant, sleep};
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
use walkdir::WalkDir;
|
||||
@@ -375,33 +374,6 @@ pub(crate) fn census_object_version_on_disk(
|
||||
})
|
||||
}
|
||||
|
||||
/// Wait for the background PUT tail to commit every physical part on one disk.
|
||||
/// Invalid metadata remains an immediate error instead of a retryable absence.
|
||||
pub(crate) async fn wait_for_complete_physical_shard_on_disk(
|
||||
disk: &Path,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: Option<&str>,
|
||||
timeout: Duration,
|
||||
) -> ChaosResult<VersionShardCensus> {
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
let census = census_object_version_on_disk(disk, bucket, key, version_id)?;
|
||||
if census.is_complete() && !census.expected_part_numbers.is_empty() {
|
||||
return Ok(census);
|
||||
}
|
||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return Err(format!(
|
||||
"physical shard for {bucket}/{key}@{version_id:?} on {} did not become complete within {timeout:?}: {census:?}",
|
||||
disk.display()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(remaining.min(Duration::from_millis(50))).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// `POST` a signed (SigV4, service `s3`) admin request without relying on the
|
||||
/// external `awscurl` binary. Mirrors the admin heal calls used by the heal
|
||||
/// regression suite.
|
||||
@@ -479,30 +451,4 @@ mod tests {
|
||||
assert!(expected.matches_manifest(&expected));
|
||||
assert!(!changed.matches_manifest(&expected));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn physical_shard_readiness_fails_closed_with_last_census() {
|
||||
let disk = tempfile::tempdir().expect("temporary disk");
|
||||
let error = wait_for_complete_physical_shard_on_disk(disk.path(), "bucket", "missing", None, Duration::ZERO)
|
||||
.await
|
||||
.expect_err("missing physical shards must fail the baseline gate");
|
||||
assert!(error.to_string().contains("has_xl_meta: false"));
|
||||
assert!(error.to_string().contains("bucket/missing"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn physical_shard_readiness_does_not_retry_invalid_metadata() {
|
||||
let disk = tempfile::tempdir().expect("temporary disk");
|
||||
let object = disk.path().join("bucket").join("corrupt");
|
||||
std::fs::create_dir_all(&object).expect("object directory");
|
||||
std::fs::write(object.join("xl.meta"), b"invalid metadata").expect("corrupt metadata fixture");
|
||||
let error = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
wait_for_complete_physical_shard_on_disk(disk.path(), "bucket", "corrupt", None, Duration::from_secs(30)),
|
||||
)
|
||||
.await
|
||||
.expect("corrupt metadata must fail immediately")
|
||||
.expect_err("invalid metadata must not be accepted as a complete baseline");
|
||||
assert!(!error.to_string().contains("did not become complete"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use serde_json;
|
||||
use std::ffi::OsStr;
|
||||
use std::fs as stdfs;
|
||||
use std::io::ErrorKind;
|
||||
use std::net::{Ipv6Addr, SocketAddr};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Once;
|
||||
@@ -1468,9 +1468,7 @@ impl RustFSTestClusterEnvironment {
|
||||
return Err(format!("a volume proxy is already configured for node {node_idx}").into());
|
||||
}
|
||||
let target = self.nodes[node_idx].address.parse::<SocketAddr>()?;
|
||||
// Endpoint locality requires the server port to match. IPv6 loopback
|
||||
// keeps that port while leaving the direct IPv4 S3 listener available.
|
||||
let proxy = crate::fault_proxy::FaultProxy::start_on((Ipv6Addr::LOCALHOST, target.port()).into(), target).await?;
|
||||
let proxy = crate::fault_proxy::FaultProxy::start(target).await?;
|
||||
self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr());
|
||||
Ok(proxy)
|
||||
}
|
||||
@@ -2375,7 +2373,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn volume_proxy_rewrites_cluster_volume_endpoint() {
|
||||
let mut env = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(2, 2))
|
||||
let mut env = RustFSTestClusterEnvironment::new(1)
|
||||
.await
|
||||
.expect("cluster environment should allocate a node");
|
||||
let direct = env.nodes[0].address.clone();
|
||||
@@ -2390,24 +2388,6 @@ mod tests {
|
||||
assert!(!volumes.contains(&direct), "volumes must not retain the direct address: {volumes}");
|
||||
|
||||
proxy.shutdown().await;
|
||||
|
||||
for node in &env.nodes {
|
||||
let local_port = node.address.parse::<SocketAddr>().expect("node address").port();
|
||||
let local_paths = volumes
|
||||
.split_whitespace()
|
||||
.filter_map(|volume| {
|
||||
let endpoint = reqwest::Url::parse(volume).expect("volume endpoint");
|
||||
rustfs_utils::is_local_host(
|
||||
endpoint.host().expect("volume endpoint host"),
|
||||
endpoint.port().expect("volume endpoint port"),
|
||||
local_port,
|
||||
)
|
||||
.expect("endpoint locality")
|
||||
.then(|| endpoint.path().to_string())
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(local_paths, node.data_dirs, "the proxy must preserve local disk ownership");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -59,9 +59,6 @@ const POOL_META_V3_ENV: [(&str, &str); 2] = [
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) enum DistLayout {
|
||||
/// 3 nodes × 4 drives, one erasure pool. With `EC:4` this is the
|
||||
/// release-evidence EC8+4 geometry.
|
||||
ThreeByFourEc84,
|
||||
/// 4 nodes × 4 drives, one erasure pool spanning every endpoint.
|
||||
FourByFour,
|
||||
/// 4 nodes × 1 drive, one erasure pool (minimum 4-node 4-disk layout).
|
||||
@@ -97,7 +94,6 @@ impl DistCluster {
|
||||
|
||||
pub async fn new_stopped_with_env(layout: DistLayout, extra_env: &[(&str, &str)]) -> TestResult<Self> {
|
||||
let topology = match layout {
|
||||
DistLayout::ThreeByFourEc84 => ClusterTopology::single_pool_multidrive(3, DRIVES_PER_NODE),
|
||||
DistLayout::FourByFour => ClusterTopology::single_pool_multidrive(NODE_COUNT, DRIVES_PER_NODE),
|
||||
DistLayout::FourNodeFourDisk => ClusterTopology::single_pool(NODE_COUNT),
|
||||
DistLayout::SingleNodeFourDrive => ClusterTopology::per_node_pools(DRIVES_PER_NODE, vec![vec![0]]),
|
||||
@@ -105,7 +101,7 @@ impl DistCluster {
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology).await?;
|
||||
let pool_storage_roots = match layout {
|
||||
DistLayout::SingleNodeFourDrive => Some(configured_pool_storage_roots()?),
|
||||
DistLayout::ThreeByFourEc84 | DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
|
||||
DistLayout::FourByFour | DistLayout::FourNodeFourDisk => None,
|
||||
};
|
||||
let mut owned_pool_dirs = Vec::new();
|
||||
if let Some(roots) = pool_storage_roots.as_deref() {
|
||||
|
||||
@@ -1,250 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_inventory, payload_for, put_object, unique_bucket, wait_until};
|
||||
use crate::chaos::{
|
||||
VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post, wait_for_complete_physical_shard_on_disk,
|
||||
};
|
||||
use crate::common::{init_logging, rustfs_binary_path};
|
||||
use crate::scanner_heal_evidence::{EvidenceTopology, RestartObservation, ScannerHealEvidenceCase, restart_evidence_run};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
const EC84_NODE_COUNT: usize = 3;
|
||||
const EC84_DRIVES_PER_NODE: usize = 4;
|
||||
const EC84_DATA_BLOCKS: usize = 8;
|
||||
const EC84_PARITY_BLOCKS: usize = 4;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ExpectedShard {
|
||||
key: String,
|
||||
body: Vec<u8>,
|
||||
baseline: VersionShardCensus,
|
||||
}
|
||||
|
||||
fn assert_ec84_geometry(census: &VersionShardCensus, key: &str) -> TestResult {
|
||||
if census.data_blocks != Some(EC84_DATA_BLOCKS) || census.parity_blocks != Some(EC84_PARITY_BLOCKS) {
|
||||
return Err(format!("object {key} did not use EC8+4 geometry: {census:?}").into());
|
||||
}
|
||||
let erasure_index = census
|
||||
.erasure_index
|
||||
.ok_or_else(|| format!("object {key} did not record an erasure index: {census:?}"))?;
|
||||
if !(1..=EC84_DATA_BLOCKS + EC84_PARITY_BLOCKS).contains(&erasure_index) {
|
||||
return Err(format!("object {key} has out-of-range erasure index {erasure_index}: {census:?}").into());
|
||||
}
|
||||
if !census.is_complete() || census.expected_part_numbers.is_empty() {
|
||||
return Err(format!("object {key} does not have complete physical shard evidence: {census:?}").into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> TestResult {
|
||||
for key in keys {
|
||||
let census = census_object_version_on_disk(drive, bucket, key, None)?;
|
||||
if census.has_xl_meta {
|
||||
return Err(format!("replacement drive unexpectedly retained {bucket}/{key}: {census:?}").into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<ExpectedShard>> {
|
||||
let mut expected = Vec::new();
|
||||
for index in 0..4 {
|
||||
let key = format!("ec84/prefix-{}/object-{index:04}.bin", index % 2);
|
||||
let body = payload_for(&key, 10 * 1024 * 1024);
|
||||
put_object(client, bucket, &key, body.clone()).await?;
|
||||
expected.push(ExpectedShard {
|
||||
key,
|
||||
body,
|
||||
baseline: VersionShardCensus {
|
||||
version_id: None,
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
data_blocks: None,
|
||||
parity_blocks: None,
|
||||
expected_part_numbers: Default::default(),
|
||||
present_part_fingerprints: Default::default(),
|
||||
inline_data_fingerprint: None,
|
||||
},
|
||||
});
|
||||
}
|
||||
Ok(expected)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart() -> TestResult {
|
||||
init_logging();
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_run = restart_evidence_run(
|
||||
&server_binary,
|
||||
ScannerHealEvidenceCase {
|
||||
id: "ec84-target-drive-restart",
|
||||
oracle: "ec84-target-drive-restart.json",
|
||||
evidence: "process-restart",
|
||||
unclean_shutdown_marker: false,
|
||||
topology: EvidenceTopology::new(3, 4),
|
||||
storage_class_standard: Some("EC:4"),
|
||||
erasure_set_drive_count: Some("12"),
|
||||
},
|
||||
)?;
|
||||
let mut dist = DistCluster::start_with_env(
|
||||
DistLayout::ThreeByFourEc84,
|
||||
&[
|
||||
("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
|
||||
("RUSTFS_HEAL_ENABLED", "true"),
|
||||
("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false"),
|
||||
("RUSTFS_HEAL_MRF_ENABLE", "false"),
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(dist.cluster.nodes.len(), EC84_NODE_COUNT);
|
||||
assert_eq!(dist.cluster.topology.drives_per_node, EC84_DRIVES_PER_NODE);
|
||||
|
||||
let bucket = unique_bucket("healec84");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let mut expected = put_large_inventory(&writer, &bucket).await?;
|
||||
let replaced_node = 1;
|
||||
let replaced_drive_index = 2;
|
||||
let replaced_drive = PathBuf::from(&dist.cluster.nodes[replaced_node].data_dirs[replaced_drive_index]);
|
||||
|
||||
for item in &mut expected {
|
||||
item.baseline =
|
||||
wait_for_complete_physical_shard_on_disk(&replaced_drive, &bucket, &item.key, None, Duration::from_secs(10)).await?;
|
||||
assert_ec84_geometry(&item.baseline, &item.key)?;
|
||||
}
|
||||
|
||||
let format_path = replaced_drive.join(".rustfs.sys").join("format.json");
|
||||
let format_json = std::fs::read(&format_path)?;
|
||||
let pid_before = dist.cluster.nodes[replaced_node]
|
||||
.process
|
||||
.as_ref()
|
||||
.ok_or("target process is absent")?
|
||||
.id();
|
||||
dist.cluster.stop_node_gracefully(replaced_node).await?;
|
||||
let unclean_shutdown_marker = Path::new(&dist.cluster.nodes[replaced_node].data_dir)
|
||||
.join(".rustfs.sys")
|
||||
.join("unclean-shutdown")
|
||||
.is_file();
|
||||
assert!(!unclean_shutdown_marker, "graceful target shutdown must remove its unclean marker");
|
||||
let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
|
||||
std::fs::rename(&replaced_drive, &retired_drive)?;
|
||||
std::fs::create_dir_all(format_path.parent().ok_or("replacement format path has no parent")?)?;
|
||||
std::fs::write(&format_path, format_json)?;
|
||||
assert_replaced_drive_empty(
|
||||
&replaced_drive,
|
||||
&bucket,
|
||||
&expected.iter().map(|item| item.key.clone()).collect::<Vec<_>>(),
|
||||
)?;
|
||||
|
||||
let outage_key = "ec84/written-while-node-restarting.bin";
|
||||
let outage_body = payload_for(outage_key, 10 * 1024 * 1024);
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(outage_key)
|
||||
.body(ByteStream::from(outage_body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
dist.cluster.start_node(replaced_node).await?;
|
||||
let heal_body =
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url);
|
||||
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
|
||||
wait_until(
|
||||
Duration::from_secs(120),
|
||||
|| async {
|
||||
for item in &expected {
|
||||
let current = census_object_version_on_disk(&replaced_drive, &bucket, &item.key, None)?;
|
||||
if !current.matches_manifest(&item.baseline) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
let outage = census_object_version_on_disk(&replaced_drive, &bucket, outage_key, None)?;
|
||||
Ok(outage.is_complete()
|
||||
&& outage.data_blocks == Some(EC84_DATA_BLOCKS)
|
||||
&& outage.parity_blocks == Some(EC84_PARITY_BLOCKS))
|
||||
},
|
||||
"EC8+4 replacement drive rebuilt baseline and outage shards",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let inventory = expected
|
||||
.iter()
|
||||
.map(|item| (item.key.clone(), item.body.clone()))
|
||||
.chain(std::iter::once((outage_key.to_string(), outage_body.clone())))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let expected_keys = inventory.keys().cloned().collect::<HashSet<_>>();
|
||||
let mut node_listings = Vec::new();
|
||||
for node_index in 0..dist.cluster.nodes.len() {
|
||||
let client = dist.client(node_index)?;
|
||||
assert_inventory(&client, &bucket, &inventory).await?;
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
let observed = listing
|
||||
.contents()
|
||||
.iter()
|
||||
.filter_map(|object| object.key().map(str::to_owned))
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(observed, expected_keys, "node {node_index} listing diverged after EC8+4 heal");
|
||||
let mut keys = observed.into_iter().collect::<Vec<_>>();
|
||||
keys.sort();
|
||||
node_listings.push(keys);
|
||||
}
|
||||
|
||||
if let Some(evidence_run) = evidence_run {
|
||||
let target_client = dist.client(replaced_node)?;
|
||||
let mut objects = Vec::with_capacity(inventory.len());
|
||||
for (key, body) in &inventory {
|
||||
let response = target_client.get_object().bucket(&bucket).key(key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(actual.as_ref(), body.as_slice(), "object body changed for {key}");
|
||||
let physical = census_object_version_on_disk(&replaced_drive, &bucket, key, None)?;
|
||||
assert_ec84_geometry(&physical, key)?;
|
||||
let baseline = expected.iter().find(|item| item.key == *key).map(|item| &item.baseline);
|
||||
objects.push(serde_json::json!({
|
||||
"key": key, "version_id": null,
|
||||
"expected_bytes": body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(body), "actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": baseline, "physical": physical,
|
||||
}));
|
||||
}
|
||||
let pid_after = dist.cluster.nodes[replaced_node]
|
||||
.process
|
||||
.as_ref()
|
||||
.ok_or("restarted target is absent")?
|
||||
.id();
|
||||
evidence_run.write(
|
||||
&server_binary,
|
||||
RestartObservation {
|
||||
nodes: dist.cluster.nodes.len(),
|
||||
drives_per_node: dist.cluster.topology.drives_per_node,
|
||||
pid_before,
|
||||
pid_after,
|
||||
unclean_shutdown_marker,
|
||||
objects,
|
||||
node_listings,
|
||||
},
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -25,7 +25,6 @@ mod data_integrity_movement_test;
|
||||
mod expand_decommission_rebalance_test;
|
||||
mod extra_test;
|
||||
mod harness;
|
||||
mod heal_test;
|
||||
mod object_lock_test;
|
||||
mod observability_test;
|
||||
mod replication_quota_test;
|
||||
|
||||
@@ -498,9 +498,7 @@ mod tests {
|
||||
.stderr(log)
|
||||
.spawn()?,
|
||||
);
|
||||
// macOS evaluates each fresh binary copy before its capability hook can run.
|
||||
let probe_timeout = if cfg!(target_os = "macos") { 60 } else { 10 };
|
||||
let status = tokio::time::timeout(Duration::from_secs(probe_timeout), async {
|
||||
let status = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
if let Some(status) = child.0.try_wait()? {
|
||||
return Ok::<_, std::io::Error>(status);
|
||||
|
||||
@@ -114,12 +114,7 @@ impl FaultProxy {
|
||||
/// Bind a listener on `127.0.0.1:0` and start forwarding accepted
|
||||
/// connections to `target`. Starts in [`FaultMode::Pass`].
|
||||
pub async fn start(target: SocketAddr) -> io::Result<Self> {
|
||||
Self::start_on((Ipv4Addr::LOCALHOST, 0).into(), target).await
|
||||
}
|
||||
|
||||
/// Bind the selected address and forward accepted connections to `target`.
|
||||
pub(crate) async fn start_on(address: SocketAddr, target: SocketAddr) -> io::Result<Self> {
|
||||
let listener = TcpListener::bind(address).await?;
|
||||
let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?;
|
||||
let listen_addr = listener.local_addr()?;
|
||||
|
||||
let (mode_tx, mode_rx) = watch::channel(FaultMode::Pass);
|
||||
|
||||
@@ -401,11 +401,11 @@ mod tests {
|
||||
let mp_view = get_full(&client, multipart_key).await?;
|
||||
assert_eq!(mp_view.sha256, sha256_hex(&multipart_body), "degraded baseline multipart body mismatch");
|
||||
baseline_degraded.insert(multipart_key.to_string(), mp_view);
|
||||
// Stop disk writers before restoring the complete layout reused by Phase B.
|
||||
harness.kill_server();
|
||||
// Restore the disk so Phase B restarts from a clean, complete disk set.
|
||||
harness.bring_disk_online(0)?;
|
||||
|
||||
// ---- Phase B: codec streaming (gates opened) ----
|
||||
harness.kill_server();
|
||||
for (k, v) in codec_env() {
|
||||
harness.set_env(k, v);
|
||||
}
|
||||
@@ -497,8 +497,6 @@ mod tests {
|
||||
let mp_view = get_full(&client, multipart_key).await?;
|
||||
assert_eq!(mp_view.sha256, sha256_hex(&multipart_body), "degraded codec multipart body mismatch");
|
||||
codec_degraded.insert(multipart_key.to_string(), mp_view);
|
||||
// All server reads are complete; stop disk writers before restoring disk0.
|
||||
harness.kill_server();
|
||||
harness.bring_disk_online(0)?;
|
||||
|
||||
// A/B under parity reconstruction: codec == legacy, byte-for-byte and
|
||||
|
||||
@@ -16,20 +16,18 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::{
|
||||
VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post,
|
||||
wait_for_complete_physical_shard_on_disk,
|
||||
};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
|
||||
use crate::common::{
|
||||
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
|
||||
rustfs_binary_path,
|
||||
};
|
||||
use crate::scanner_heal_evidence::{EvidenceTopology, RestartObservation, ScannerHealEvidenceCase, restart_evidence_run};
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
@@ -41,14 +39,34 @@ mod tests {
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RestartEvidenceRun {
|
||||
schema: u32,
|
||||
run_id: String,
|
||||
source_revision: String,
|
||||
test_build: serde_json::Value,
|
||||
binary: EvidenceBuild,
|
||||
test_binary: EvidenceBuild,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ScannerHealEvidenceCase {
|
||||
id: &'static str,
|
||||
oracle: &'static str,
|
||||
evidence: &'static str,
|
||||
unclean_shutdown_marker: bool,
|
||||
}
|
||||
|
||||
const BACKGROUND_TARGET_RESTART_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-restart",
|
||||
oracle: "background-target-restart.json",
|
||||
evidence: "process-restart",
|
||||
unclean_shutdown_marker: false,
|
||||
topology: EvidenceTopology::new(4, 1),
|
||||
storage_class_standard: None,
|
||||
erasure_set_drive_count: None,
|
||||
};
|
||||
|
||||
const BACKGROUND_TARGET_CRASH_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
@@ -56,30 +74,82 @@ mod tests {
|
||||
oracle: "background-target-crash.json",
|
||||
evidence: "process-crash-restart",
|
||||
unclean_shutdown_marker: true,
|
||||
topology: EvidenceTopology::new(4, 1),
|
||||
storage_class_standard: None,
|
||||
erasure_set_drive_count: None,
|
||||
};
|
||||
|
||||
const BACKGROUND_TARGET_RESTART_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-restart-ec8-4",
|
||||
oracle: "background-target-restart-ec8-4.json",
|
||||
evidence: "process-restart",
|
||||
unclean_shutdown_marker: false,
|
||||
topology: EvidenceTopology::new(3, 4),
|
||||
storage_class_standard: Some("EC:4"),
|
||||
erasure_set_drive_count: Some("12"),
|
||||
};
|
||||
struct RestartEvidenceContext {
|
||||
directory: PathBuf,
|
||||
run: RestartEvidenceRun,
|
||||
case: ScannerHealEvidenceCase,
|
||||
}
|
||||
|
||||
const BACKGROUND_TARGET_CRASH_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-crash-ec8-4",
|
||||
oracle: "background-target-crash-ec8-4.json",
|
||||
evidence: "process-crash-restart",
|
||||
unclean_shutdown_marker: true,
|
||||
topology: EvidenceTopology::new(3, 4),
|
||||
storage_class_standard: Some("EC:4"),
|
||||
erasure_set_drive_count: Some("12"),
|
||||
};
|
||||
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file.read(&mut buffer)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..read]);
|
||||
}
|
||||
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||
}
|
||||
|
||||
fn restart_evidence_run(
|
||||
binary: &Path,
|
||||
case: ScannerHealEvidenceCase,
|
||||
) -> Result<Option<RestartEvidenceContext>, Box<dyn Error + Send + Sync>> {
|
||||
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||
return Ok(None);
|
||||
};
|
||||
if case.id.is_empty()
|
||||
|| case.oracle.is_empty()
|
||||
|| !case.oracle.ends_with(".json")
|
||||
|| case.oracle.contains('/')
|
||||
|| case.oracle.contains('\\')
|
||||
|| case.oracle.contains("..")
|
||||
|| !matches!(case.evidence, "process-restart" | "process-crash-restart")
|
||||
|| (case.evidence == "process-crash-restart") != case.unclean_shutdown_marker
|
||||
{
|
||||
return Err("invalid scanner/heal evidence case".into());
|
||||
}
|
||||
let directory = PathBuf::from(directory);
|
||||
let receipt = directory.join("run.json");
|
||||
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||
return Err("oversized scanner/heal execution receipt".into());
|
||||
}
|
||||
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
|
||||
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
|
||||
return Err("invalid scanner/heal execution identity".into());
|
||||
}
|
||||
let built = compiled_test_identity();
|
||||
for key in ["source_revision", "dirty", "lock_blob", "features"] {
|
||||
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
|
||||
}
|
||||
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
|
||||
assert_eq!(
|
||||
file_sha256(&std::env::current_exe()?)?,
|
||||
run.test_binary.sha256,
|
||||
"test executable must match the run receipt"
|
||||
);
|
||||
if directory.join(case.oracle).exists() {
|
||||
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||
}
|
||||
Ok(Some(RestartEvidenceContext { directory, run, case }))
|
||||
}
|
||||
|
||||
fn compiled_test_identity() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
|
||||
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
|
||||
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
|
||||
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
|
||||
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
|
||||
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
|
||||
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
|
||||
})
|
||||
}
|
||||
|
||||
struct TcpPortBlackhole {
|
||||
port: u16,
|
||||
@@ -262,10 +332,11 @@ mod tests {
|
||||
|
||||
// Healing may rewrite non-identity bookkeeping in xl.meta. The census
|
||||
// therefore compares the canonical selected metadata fields plus every
|
||||
// physical shard.
|
||||
// physical shard, while the payload seed makes object mix-ups observable.
|
||||
#[derive(Debug)]
|
||||
struct PhysicalObjectManifest {
|
||||
key: String,
|
||||
payload_seed: u8,
|
||||
shard_census: VersionShardCensus,
|
||||
}
|
||||
|
||||
@@ -385,79 +456,6 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
async fn wait_for_admin_cluster_start_log(
|
||||
log_path: &Path,
|
||||
client_token: &str,
|
||||
deadline: Instant,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
loop {
|
||||
let coordinator_log = std::fs::read_to_string(log_path)?;
|
||||
if coordinator_log
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.any(|event| {
|
||||
event["event"] == "heal_task_state"
|
||||
&& event["task_id"] == client_token
|
||||
&& event["heal_type"] == "cluster"
|
||||
&& event["state"] == "started"
|
||||
})
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(
|
||||
format!("node 0 must have started the exact admin task before interruption: task_id={client_token}").into(),
|
||||
);
|
||||
}
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_admin_cluster_start_log_waits_for_exact_delayed_event() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
use std::io::Write;
|
||||
|
||||
let mut log = tempfile::NamedTempFile::new()?;
|
||||
for (task_id, heal_type, state) in [
|
||||
("other-task", "cluster", "started"),
|
||||
("admin-task", "object", "started"),
|
||||
("admin-task", "cluster", "completed"),
|
||||
] {
|
||||
writeln!(
|
||||
log,
|
||||
"{}",
|
||||
serde_json::json!({"event": "heal_task_state", "task_id": task_id, "heal_type": heal_type, "state": state})
|
||||
)?;
|
||||
}
|
||||
let log_path = log.path().to_path_buf();
|
||||
let started = wait_for_admin_cluster_start_log(&log_path, "admin-task", Instant::now() + Duration::from_secs(1));
|
||||
tokio::pin!(started);
|
||||
// Poll the reader before publishing the start event, without depending
|
||||
// on scheduling or a fixed writer delay to reproduce log visibility.
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut started => panic!("unrelated events must leave the exact start pending: {result:?}"),
|
||||
_ = std::future::ready(()) => {}
|
||||
}
|
||||
writeln!(
|
||||
log,
|
||||
"{}",
|
||||
serde_json::json!({"event": "heal_task_state", "task_id": "admin-task", "heal_type": "cluster", "state": "started"})
|
||||
)?;
|
||||
started.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_admin_cluster_start_log_respects_existing_deadline() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let log = tempfile::NamedTempFile::new()?;
|
||||
let error = wait_for_admin_cluster_start_log(log.path(), "admin-task", Instant::now())
|
||||
.await
|
||||
.expect_err("missing exact start must fail at the supplied deadline");
|
||||
assert!(error.to_string().contains("task_id=admin-task"), "{error}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cluster_heal_is_idle(status: &serde_json::Value) -> bool {
|
||||
let operations = &status["healOperations"];
|
||||
status["clusterStatusComplete"] == serde_json::Value::Bool(true)
|
||||
@@ -894,10 +892,7 @@ mod tests {
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
previous_cycle_end: u64,
|
||||
) -> Result<u64, Box<dyn Error + Send + Sync>> {
|
||||
let started = Instant::now();
|
||||
let mut deadline = started + Duration::from_secs(60);
|
||||
let catch_up_deadline = deadline + Duration::from_secs(300);
|
||||
let mut catch_up_wait_observed = false;
|
||||
let deadline = Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
let mut latest_cycle_end = 0;
|
||||
let mut versions_observed = false;
|
||||
@@ -925,61 +920,24 @@ mod tests {
|
||||
let versions_scanned = metrics["versions_scanned"]
|
||||
.as_u64()
|
||||
.ok_or("scanner status is missing its version-coverage counter")?;
|
||||
let cycle_result = metrics["last_cycle_result"]
|
||||
.as_str()
|
||||
.ok_or("scanner status is missing its cycle result")?;
|
||||
if cycle_result == "success" {
|
||||
latest_cycle_end = latest_cycle_end.max(cycle_end);
|
||||
}
|
||||
latest_cycle_end = latest_cycle_end.max(cycle_end);
|
||||
versions_observed |= versions_scanned > 0;
|
||||
let backlog = &status["pause_backlog"];
|
||||
if !catch_up_wait_observed
|
||||
&& backlog["persistence_state"].as_str() == Some("healthy")
|
||||
&& backlog["durable"].as_bool() == Some(true)
|
||||
&& backlog["phase"].as_str() == Some("catching_up")
|
||||
&& backlog["rate_limited"].as_bool() == Some(true)
|
||||
&& backlog["retry_exhausted"].as_bool() == Some(false)
|
||||
{
|
||||
let next_attempt = backlog["next_attempt_at_unix_secs"]
|
||||
.as_u64()
|
||||
.ok_or("rate-limited scanner backlog is missing its next attempt")?;
|
||||
let interval = backlog["thresholds"]["catch_up_min_interval_seconds"]
|
||||
.as_u64()
|
||||
.ok_or("rate-limited scanner backlog is missing its catch-up interval")?;
|
||||
let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_secs();
|
||||
let remaining = next_attempt.saturating_sub(now);
|
||||
if remaining > 0 {
|
||||
if interval > 300 || remaining > interval {
|
||||
return Err(
|
||||
format!("scanner catch-up schedule exceeds the bounded recovery budget: {backlog}").into()
|
||||
);
|
||||
}
|
||||
// The durable catch-up interval overrides SCANNER_CYCLE=1.
|
||||
// Honor one observed retry without restarting the deadline on every poll.
|
||||
deadline = deadline
|
||||
.max(Instant::now() + Duration::from_secs(remaining + 60))
|
||||
.min(catch_up_deadline);
|
||||
catch_up_wait_observed = true;
|
||||
}
|
||||
}
|
||||
observations.push(format!(
|
||||
"node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}, backlog={}",
|
||||
"node{node_index}: end={cycle_end}, versions={versions_scanned}, cycle={}, active={}, leader={}, result={}",
|
||||
metrics["current_cycle"],
|
||||
metrics["current_cycle_active"],
|
||||
metrics["leader_lock_state"],
|
||||
metrics["last_cycle_result"],
|
||||
backlog,
|
||||
));
|
||||
}
|
||||
// Only a successful coordinator cycle counts as completion; deferred
|
||||
// and superseded attempts also advance its end timestamp. Remote
|
||||
// workers record version coverage, so the witnesses can span nodes.
|
||||
// The coordinator records cycle completion, but remote workers
|
||||
// record scanned versions. Both witnesses need not share a node.
|
||||
if latest_cycle_end > previous_cycle_end && versions_observed {
|
||||
return Ok(latest_cycle_end);
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"enabled scanner did not complete a successful object-scanning cycle after {previous_cycle_end}: {observations:?}"
|
||||
"enabled scanner did not complete an object-scanning cycle after {previous_cycle_end}: {observations:?}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
@@ -999,7 +957,7 @@ mod tests {
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_background_target_restart()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(720),
|
||||
Duration::from_secs(420),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestart),
|
||||
)
|
||||
.await?
|
||||
@@ -1009,37 +967,17 @@ mod tests {
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_background_target_crash()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(720),
|
||||
Duration::from_secs(420),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrash),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_ec84_shards_after_background_target_restart()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(720),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_ec84_shards_after_background_target_crash()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(720),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrashEc84),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
timeout(
|
||||
Duration::from_secs(720),
|
||||
Duration::from_secs(420),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundCoordinatorRestart),
|
||||
)
|
||||
.await?
|
||||
@@ -1072,38 +1010,29 @@ mod tests {
|
||||
IsolatedTargetRestart,
|
||||
BackgroundTargetRestart,
|
||||
BackgroundTargetCrash,
|
||||
BackgroundTargetRestartEc84,
|
||||
BackgroundTargetCrashEc84,
|
||||
BackgroundCoordinatorRestart,
|
||||
TargetEndpointBlackhole,
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_case = match scenario {
|
||||
InterruptionScenario::BackgroundTargetRestart => Some(BACKGROUND_TARGET_RESTART_EVIDENCE),
|
||||
InterruptionScenario::BackgroundTargetCrash => Some(BACKGROUND_TARGET_CRASH_EVIDENCE),
|
||||
InterruptionScenario::BackgroundTargetRestartEc84 => Some(BACKGROUND_TARGET_RESTART_EC84_EVIDENCE),
|
||||
InterruptionScenario::BackgroundTargetCrashEc84 => Some(BACKGROUND_TARGET_CRASH_EC84_EVIDENCE),
|
||||
let evidence_run = match scenario {
|
||||
InterruptionScenario::BackgroundTargetRestart => {
|
||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_RESTART_EVIDENCE)?
|
||||
}
|
||||
InterruptionScenario::BackgroundTargetCrash => {
|
||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_CRASH_EVIDENCE)?
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let evidence_run = match evidence_case {
|
||||
Some(case) => restart_evidence_run(&server_binary, case)?,
|
||||
None => None,
|
||||
};
|
||||
let mut evidence_objects = Vec::new();
|
||||
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
||||
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
||||
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
|
||||
InterruptionScenario::BackgroundTargetCrash => (true, 1, "background_target_crash"),
|
||||
InterruptionScenario::BackgroundTargetRestartEc84 => (true, 1, "background_target_restart_ec8_4"),
|
||||
InterruptionScenario::BackgroundTargetCrashEc84 => (true, 1, "background_target_crash_ec8_4"),
|
||||
InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"),
|
||||
InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"),
|
||||
};
|
||||
let topology = evidence_case
|
||||
.map(|case| case.topology)
|
||||
.unwrap_or_else(|| EvidenceTopology::new(4, 1));
|
||||
init_logging();
|
||||
info!(
|
||||
event = "heal_interruption_started",
|
||||
@@ -1115,15 +1044,11 @@ mod tests {
|
||||
"Starting root-heal interruption test"
|
||||
);
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology.cluster_topology()).await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
if let Some(storage_class) = evidence_case.and_then(|case| case.storage_class_standard) {
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", storage_class);
|
||||
}
|
||||
if let Some(erasure_set_drive_count) = evidence_case.and_then(|case| case.erasure_set_drive_count) {
|
||||
cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", erasure_set_drive_count);
|
||||
}
|
||||
// Capture physical baselines after the PUT rename fanout has drained.
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
// Heal control uses the first lexicographically sorted grid host.
|
||||
// Keep that coordinator distinct from the remote target at index 1.
|
||||
cluster.nodes.sort_by(|left, right| left.url.cmp(&right.url));
|
||||
@@ -1146,19 +1071,10 @@ mod tests {
|
||||
let server_rust_log = std::env::var("RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG")
|
||||
.unwrap_or_else(|_| "rustfs::heal::task=info,rustfs=error".to_string());
|
||||
cluster.set_env("RUST_LOG", server_rust_log);
|
||||
let log_dir = if let Some(directory) = std::env::var_os("RUSTFS_HEAL_CHAOS_LOG_DIR") {
|
||||
PathBuf::from(directory)
|
||||
} else if let Some(directory) = std::env::var_os("RUSTFS_E2E_LOG_DIR") {
|
||||
let cluster_name = Path::new(&cluster.temp_dir)
|
||||
.file_name()
|
||||
.ok_or("cluster directory has no name")?;
|
||||
PathBuf::from(directory).join(cluster_name).join("heal")
|
||||
} else {
|
||||
PathBuf::from(&cluster.temp_dir).join("logs")
|
||||
};
|
||||
let log_dir = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR").unwrap_or_else(|_| format!("{}/logs", cluster.temp_dir));
|
||||
std::fs::create_dir_all(&log_dir)?;
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, log_dir.join(format!("node{node_index}.log")).to_string_lossy())?;
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
}
|
||||
cluster.start_with_binary(&server_binary).await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
@@ -1181,22 +1097,11 @@ mod tests {
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.unwrap_or(4 * 1024 * 1024)
|
||||
.clamp(1024 * 1024, 16 * 1024 * 1024);
|
||||
let mut created_online_objects = Vec::with_capacity(online_object_count);
|
||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||
let mut unclean_shutdown_marker_observed = None;
|
||||
let mut attempt_count = 0usize;
|
||||
let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
|
||||
while expected_manifests.len() < online_object_count {
|
||||
if attempt_count >= max_online_attempts {
|
||||
return Err(format!(
|
||||
"target replacement drive held only {}/{} baseline object shards after {attempt_count} writes",
|
||||
expected_manifests.len(),
|
||||
online_object_count
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let key = format!("cluster/online/object-{attempt_count:04}.bin");
|
||||
let payload_seed = ((attempt_count % 251) + 1) as u8;
|
||||
for index in 0..online_object_count {
|
||||
let key = format!("cluster/online/object-{index:04}.bin");
|
||||
let payload_seed = u8::try_from(index + 1).expect("clamped object count must fit in u8");
|
||||
timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[0]
|
||||
@@ -1208,13 +1113,6 @@ mod tests {
|
||||
)
|
||||
.await??;
|
||||
let shard_census = census_object_version_on_disk(&replaced_disk, bucket, &key, None)?;
|
||||
if !shard_census.has_xl_meta {
|
||||
timeout(Duration::from_secs(30), clients[0].delete_object().bucket(bucket).key(&key).send()).await??;
|
||||
attempt_count += 1;
|
||||
continue;
|
||||
}
|
||||
let shard_census =
|
||||
wait_for_complete_physical_shard_on_disk(&replaced_disk, bucket, &key, None, Duration::from_secs(10)).await?;
|
||||
assert!(
|
||||
shard_census.is_complete(),
|
||||
"node 1 should hold a complete baseline shard for {key}: {shard_census:?}"
|
||||
@@ -1223,9 +1121,11 @@ mod tests {
|
||||
!shard_census.expected_part_numbers.is_empty(),
|
||||
"chaos objects must use physical part shards rather than inline data: {shard_census:?}"
|
||||
);
|
||||
created_online_objects.push((key.clone(), payload_seed));
|
||||
expected_manifests.push(PhysicalObjectManifest { key, shard_census });
|
||||
attempt_count += 1;
|
||||
expected_manifests.push(PhysicalObjectManifest {
|
||||
key,
|
||||
payload_seed,
|
||||
shard_census,
|
||||
});
|
||||
}
|
||||
|
||||
let expected_pool_metadata = if background_enabled {
|
||||
@@ -1271,38 +1171,31 @@ mod tests {
|
||||
if node_index == 1 {
|
||||
continue;
|
||||
}
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
let census = census_object_version_on_disk(Path::new(drive), bucket, outage_key, None)?;
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census.erasure_index.ok_or_else(|| {
|
||||
format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}")
|
||||
})?;
|
||||
assert!(
|
||||
(1..=topology.total_drives()).contains(&erasure_index),
|
||||
"online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online drives"
|
||||
);
|
||||
}
|
||||
let census = census_object_version_on_disk(Path::new(&node.data_dir), bucket, outage_key, None)?;
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census
|
||||
.erasure_index
|
||||
.ok_or_else(|| format!("online node {node_index} outage-object shard has no erasure index: {census:?}"))?;
|
||||
assert!(
|
||||
(1..=cluster.nodes.len()).contains(&erasure_index),
|
||||
"online node {node_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online nodes"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
outage_peer_erasure_indices.len(),
|
||||
topology.total_drives().saturating_sub(cluster.nodes[1].data_dirs.len()),
|
||||
"every online drive must contribute one unique outage-object erasure index"
|
||||
);
|
||||
let missing_outage_erasure_indices = (1..=topology.total_drives())
|
||||
.filter(|index| !outage_peer_erasure_indices.contains(index))
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
missing_outage_erasure_indices.len(),
|
||||
cluster.nodes[1].data_dirs.len(),
|
||||
"the stopped node must account for every missing outage-object erasure index"
|
||||
cluster.nodes.len().saturating_sub(1),
|
||||
"every online node must contribute one unique outage-object erasure index"
|
||||
);
|
||||
let expected_outage_target_erasure_index = (1..=cluster.nodes.len())
|
||||
.find(|index| !outage_peer_erasure_indices.contains(index))
|
||||
.ok_or("online outage-object shards leave no erasure index for the replacement target")?;
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
if !background_enabled {
|
||||
@@ -1401,9 +1294,6 @@ mod tests {
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
// Task execution and its non-blocking log writer advance independently.
|
||||
// Observe the exact start before taking the partial-rebuild snapshot.
|
||||
wait_for_admin_cluster_start_log(&log_dir.join("node0.log"), client_token, partial_deadline).await?;
|
||||
let (partial_count, partial_manifest) = loop {
|
||||
// Hash one committed shard to prove progress without letting a
|
||||
// full-corpus hash pass consume the interruption window.
|
||||
@@ -1438,6 +1328,19 @@ mod tests {
|
||||
let pre_interrupt_status: serde_json::Value = serde_json::from_str(&pre_interrupt_status_body)
|
||||
.map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {pre_interrupt_status_body}"))?;
|
||||
let pre_interrupt_replacement = replacement_recovery_status(&cluster).await?;
|
||||
let coordinator_log = std::fs::read_to_string(format!("{log_dir}/node0.log"))?;
|
||||
assert!(
|
||||
coordinator_log
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.any(|event| {
|
||||
event["event"] == "heal_task_state"
|
||||
&& event["task_id"] == client_token
|
||||
&& event["heal_type"] == "cluster"
|
||||
&& event["state"] == "started"
|
||||
}),
|
||||
"node 0 must have started the exact admin task before interruption"
|
||||
);
|
||||
let pre_interrupt_operations = &pre_interrupt_status["healOperations"];
|
||||
assert_eq!(
|
||||
pre_interrupt_operations["activeBySource"]["admin"].as_u64(),
|
||||
@@ -1562,11 +1465,7 @@ mod tests {
|
||||
"Restored target endpoint forwarding"
|
||||
);
|
||||
} else {
|
||||
let graceful_restart = matches!(
|
||||
scenario,
|
||||
InterruptionScenario::BackgroundTargetRestart | InterruptionScenario::BackgroundTargetRestartEc84
|
||||
);
|
||||
if graceful_restart {
|
||||
if scenario == InterruptionScenario::BackgroundTargetRestart {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
} else {
|
||||
cluster.stop_node(interruption_node)?;
|
||||
@@ -1588,7 +1487,7 @@ mod tests {
|
||||
if background_enabled {
|
||||
let marker_exists = unclean_shutdown_marker.is_file();
|
||||
unclean_shutdown_marker_observed = Some(marker_exists);
|
||||
let expected_marker = !graceful_restart;
|
||||
let expected_marker = !matches!(scenario, InterruptionScenario::BackgroundTargetRestart);
|
||||
assert!(
|
||||
marker_exists == expected_marker,
|
||||
"background restart/crash lane observed unexpected unclean-shutdown marker state"
|
||||
@@ -1644,19 +1543,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
if Instant::now() >= heal_deadline {
|
||||
let mut mismatched_manifests = Vec::new();
|
||||
for expected in &expected_manifests {
|
||||
let actual = census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?;
|
||||
if !actual.matches_manifest(&expected.shard_census) {
|
||||
mismatched_manifests.push(serde_json::json!({
|
||||
"key": expected.key,
|
||||
"expected": expected.shard_census,
|
||||
"actual": actual,
|
||||
}));
|
||||
}
|
||||
}
|
||||
let matching = expected_manifests.len() - mismatched_manifests.len();
|
||||
let mismatched_manifests = serde_json::Value::Array(mismatched_manifests);
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let pool_metadata =
|
||||
census_object_version_on_disk(&replaced_disk, RUSTFS_META_BUCKET, POOL_METADATA_OBJECT, None)?;
|
||||
@@ -1679,7 +1566,7 @@ mod tests {
|
||||
Err(_) => "replacement status request exceeded 5s diagnostic budget".to_string(),
|
||||
};
|
||||
return Err(format!(
|
||||
"root heal did not recover after {interruption_kind} within {heal_timeout_secs}s: baseline={matching}/{}, mismatched_manifests={mismatched_manifests}, outage={outage_census:?}, pool_metadata={pool_metadata:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, pre_interrupt_replacement={pre_interrupt_replacement}, replacement_status={replacement_status}",
|
||||
"root heal did not recover after {interruption_kind} within {heal_timeout_secs}s: baseline={matching}/{}, outage={outage_census:?}, pool_metadata={pool_metadata:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, pre_interrupt_replacement={pre_interrupt_replacement}, replacement_status={replacement_status}",
|
||||
expected_manifests.len()
|
||||
)
|
||||
.into());
|
||||
@@ -1701,35 +1588,29 @@ mod tests {
|
||||
"outage object must have a complete target shard: {outage_census:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
outage_census
|
||||
.erasure_index
|
||||
.filter(|index| missing_outage_erasure_indices.contains(index)),
|
||||
outage_census.erasure_index,
|
||||
"the outage object must be rebuilt into one of the stopped node's missing erasure slots"
|
||||
Some(expected_outage_target_erasure_index),
|
||||
"the outage object must be rebuilt into its own missing erasure slot"
|
||||
);
|
||||
|
||||
if let Some(cycle_end) = scanner_cycle_floor {
|
||||
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
||||
}
|
||||
|
||||
let mut expected_keys = created_online_objects
|
||||
let mut expected_keys = expected_manifests
|
||||
.iter()
|
||||
.map(|(key, _)| key.clone())
|
||||
.map(|manifest| manifest.key.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(expected_keys.insert(outage_key.to_string()));
|
||||
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
for (key, payload_seed) in &created_online_objects {
|
||||
let response = target_client.get_object().bucket(bucket).key(key).send().await?;
|
||||
for expected in &expected_manifests {
|
||||
let response = target_client.get_object().bucket(bucket).key(&expected.key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_body = deterministic_object_body(object_size_bytes, *payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {key}");
|
||||
if evidence_run.is_some()
|
||||
&& let Some(expected) = expected_manifests
|
||||
.iter()
|
||||
.find(|manifest| manifest.key.as_str() == key.as_str())
|
||||
{
|
||||
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": expected.key, "version_id": expected.shard_census.version_id,
|
||||
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
|
||||
@@ -1803,18 +1684,32 @@ mod tests {
|
||||
if let Some(evidence_context) = evidence_run {
|
||||
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
|
||||
assert_ne!(target_pid, restarted_pid, "target must be a new process");
|
||||
evidence_context.write(
|
||||
&server_binary,
|
||||
RestartObservation {
|
||||
nodes: cluster.nodes.len(),
|
||||
drives_per_node: cluster.nodes[0].data_dirs.len(),
|
||||
pid_before: target_pid,
|
||||
pid_after: restarted_pid,
|
||||
unclean_shutdown_marker: unclean_shutdown_marker_observed.ok_or("missing shutdown marker observation")?,
|
||||
objects: evidence_objects,
|
||||
node_listings,
|
||||
},
|
||||
)?;
|
||||
assert_eq!(
|
||||
file_sha256(&server_binary)?,
|
||||
evidence_context.run.binary.sha256,
|
||||
"server build changed during restart"
|
||||
);
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": evidence_context.case.id, "evidence": evidence_context.case.evidence,
|
||||
"run_id": evidence_context.run.run_id, "source_revision": evidence_context.run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": evidence_context.run.binary.sha256,
|
||||
"test_binary_sha256": evidence_context.run.test_binary.sha256,
|
||||
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
}
|
||||
let mut output = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(evidence_context.directory.join(evidence_context.case.oracle))?;
|
||||
output.write_all(&data)?;
|
||||
output.sync_all()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
//! One S3 GET can select readers on multiple EC nodes, so the counter tracks
|
||||
//! distributed reader selection rather than HTTP request count.
|
||||
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, signal_process};
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
@@ -2207,33 +2207,6 @@ async fn four_node_manual_transition_job_status_survives_node_restart() -> TestR
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct SuspendedTransitionTarget<'a> {
|
||||
// Keep the owned child borrowed until it is resumed so its PID cannot be reused.
|
||||
child: &'a std::process::Child,
|
||||
suspended: bool,
|
||||
}
|
||||
|
||||
impl<'a> SuspendedTransitionTarget<'a> {
|
||||
fn suspend(child: &'a std::process::Child) -> TestResult<Self> {
|
||||
signal_process(child.id(), "STOP")?;
|
||||
Ok(Self { child, suspended: true })
|
||||
}
|
||||
|
||||
fn resume(&mut self) -> TestResult {
|
||||
signal_process(self.child.id(), "CONT")?;
|
||||
self.suspended = false;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SuspendedTransitionTarget<'_> {
|
||||
fn drop(&mut self) {
|
||||
if self.suspended {
|
||||
let _ = signal_process(self.child.id(), "CONT");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_manual_transition_distributed_admission_conflict_reports_status_and_backpressure() -> TestResult {
|
||||
init_logging();
|
||||
@@ -2271,21 +2244,7 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
// Lifecycle PUT starts its own backfill. Keep its first page on a separate
|
||||
// node and stop it at queue backpressure before it reaches the tested prefix:
|
||||
// one active worker, one queued item, then the first rejected item.
|
||||
for index in 0u8..3 {
|
||||
hot_client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(format!("transition/automatic-admission/object-{index:02}.bin"))
|
||||
.body(ByteStream::from(payload(KIB, index)))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
let mut suspended_cold = SuspendedTransitionTarget::suspend(cold.process.as_ref().ok_or("cold-tier process missing")?)?;
|
||||
let lifecycle_client = hot.create_s3_client(2)?;
|
||||
put_lifecycle_with_transition_retry(&lifecycle_client, &bucket, &tier_name).await?;
|
||||
put_lifecycle_with_transition_retry(&hot_client, &bucket, &tier_name).await?;
|
||||
|
||||
let (node0, node1) = tokio::join!(
|
||||
start_manual_transition_job_on_node(&hot, 0, &bucket, prefix, &tier_name, false, 64),
|
||||
@@ -2345,31 +2304,6 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
assert_eq!(status["job_id"].as_str(), Some(job_id));
|
||||
assert_eq!(status["status_endpoint"].as_str(), Some(status_endpoint));
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let status = read_manual_transition_job_status_endpoint(&hot, accepted.0, status_endpoint).await?;
|
||||
assert_eq!(
|
||||
status["status"].as_str(),
|
||||
Some("running"),
|
||||
"blocked cold tier must keep the admitted job running: {status}"
|
||||
);
|
||||
if status["report"]["skipped_queue_full"].as_u64().is_some_and(|count| count > 0) {
|
||||
assert!(
|
||||
status["report"]["enqueued"].as_u64().is_some_and(|count| count > 0),
|
||||
"the job must own pending transitions while the cold tier is suspended: {status}"
|
||||
);
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"manual transition job did not reach queue backpressure while the cold tier was suspended: {status}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
suspended_cold.resume()?;
|
||||
let terminal = wait_for_manual_transition_job_terminal(&hot, conflict.0, job_id, false).await?;
|
||||
assert_eq!(terminal["job_id"].as_str(), Some(job_id));
|
||||
assert_eq!(terminal["bucket"].as_str(), Some(bucket.as_str()));
|
||||
|
||||
@@ -23,9 +23,6 @@ pub mod common;
|
||||
#[cfg(test)]
|
||||
pub mod chaos;
|
||||
|
||||
#[cfg(test)]
|
||||
mod scanner_heal_evidence;
|
||||
|
||||
// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8)
|
||||
// and on-demand-migration source scenarios (backlog#2151).
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -25,6 +25,190 @@ const KEY: &str = "thumb/79/concurrent-overwrite.jpg";
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
async fn assert_quorum_object_body(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let body = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(body.as_ref(), expected, "quorum read returned incorrect contents for {key}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> TestResult {
|
||||
// SIGKILL can orphan a granted lease. Wait for shared metadata-lock
|
||||
// admission before asserting the stable quorum boundary; cold bodies
|
||||
// remain unread throughout this readiness probe.
|
||||
let deadline =
|
||||
tokio::time::Instant::now() + rustfs_lock::fast_lock::DEFAULT_LOCK_TIMEOUT + std::time::Duration::from_secs(15);
|
||||
loop {
|
||||
let mut ready = true;
|
||||
for client in clients {
|
||||
for key in ["warm-small", "warm-large"] {
|
||||
match client.head_object().bucket(bucket).key(key).send().await {
|
||||
Ok(_) => {}
|
||||
Err(error) if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) => {
|
||||
ready = false;
|
||||
break;
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
if !ready {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ready {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("read quorum did not become available after lease convergence for {bucket}").into());
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
for (node_count, parity) in [(4, 2), (6, 3), (6, 2)] {
|
||||
let read_quorum = node_count - parity;
|
||||
let write_quorum = read_quorum + usize::from(read_quorum == parity);
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(node_count).await?;
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{parity}"));
|
||||
// Wait for every seed fanout before removing any physical shard.
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
cluster.set_env("RUST_LOG", "warn,rustfs_lock=debug");
|
||||
cluster.start().await?;
|
||||
|
||||
let clients = cluster
|
||||
.create_all_clients()?
|
||||
.into_iter()
|
||||
.map(|client| {
|
||||
Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let bucket = format!("read-quorum-{node_count}-{parity}");
|
||||
clients[0].create_bucket().bucket(&bucket).send().await?;
|
||||
let small = b"read quorum is derived from the erasure layout".to_vec();
|
||||
let large = (0..1_048_576)
|
||||
.map(|index| u8::try_from(index % 251).expect("bounded payload byte"))
|
||||
.collect::<Vec<_>>();
|
||||
for (key, body) in [
|
||||
("warm-small", &small),
|
||||
("warm-large", &large),
|
||||
("cold-small", &small),
|
||||
("cold-large", &large),
|
||||
("below-quorum", &large),
|
||||
] {
|
||||
clients[node_count - 1]
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
for node in &cluster.nodes {
|
||||
for key in ["warm-small", "warm-large", "cold-small", "cold-large", "below-quorum"] {
|
||||
let census =
|
||||
crate::chaos::census_object_version_on_disk(std::path::Path::new(&node.data_dir), &bucket, key, None)?;
|
||||
assert!(census.is_complete(), "seed shard must be complete before fault injection: {census:?}");
|
||||
assert_eq!(census.data_blocks, Some(read_quorum));
|
||||
assert_eq!(census.parity_blocks, Some(parity));
|
||||
}
|
||||
}
|
||||
for client in &clients {
|
||||
assert_quorum_object_body(client, &bucket, "warm-small", &small).await?;
|
||||
assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
|
||||
}
|
||||
|
||||
for offline_node in (read_quorum..node_count).rev() {
|
||||
cluster.stop_node(offline_node)?;
|
||||
wait_for_quorum_read_admission(&clients[..offline_node], &bucket).await?;
|
||||
for client in clients.iter().take(offline_node) {
|
||||
client.head_bucket().bucket(&bucket).send().await?;
|
||||
assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise more than the five-second positive bucket-validation TTL.
|
||||
// Every sample must succeed; polling must not hide a transient failure.
|
||||
let validation_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(6);
|
||||
loop {
|
||||
for client in clients.iter().take(read_quorum) {
|
||||
assert_quorum_object_body(client, &bucket, "warm-small", &small).await?;
|
||||
assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
for key in ["warm-small", "warm-large", "cold-small", "cold-large", "below-quorum"] {
|
||||
assert!(listing.contents().iter().any(|entry| entry.key() == Some(key)), "listing omitted {key}");
|
||||
}
|
||||
}
|
||||
if tokio::time::Instant::now() >= validation_deadline {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
}
|
||||
for client in clients.iter().take(read_quorum) {
|
||||
assert_quorum_object_body(client, &bucket, "cold-small", &small).await?;
|
||||
assert_quorum_object_body(client, &bucket, "cold-large", &large).await?;
|
||||
}
|
||||
|
||||
let write = clients[0]
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("quorum-write")
|
||||
.body(Bytes::copy_from_slice(&small).into())
|
||||
.send()
|
||||
.await;
|
||||
if read_quorum >= write_quorum {
|
||||
write?;
|
||||
} else {
|
||||
let error = write.expect_err("a read quorum must not authorize a write that needs more votes");
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
}
|
||||
|
||||
cluster.stop_node(read_quorum - 1)?;
|
||||
for client in clients.iter().take(read_quorum - 1) {
|
||||
match client.get_object().bucket(&bucket).key("below-quorum").send().await {
|
||||
Ok(response) => assert!(
|
||||
response.body.collect().await.is_err(),
|
||||
"fewer than {read_quorum} valid fragments must not reconstruct an uncached object"
|
||||
),
|
||||
Err(error) => assert_eq!(
|
||||
error.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable"),
|
||||
"a quorum loss must not be mistaken for a missing object"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
for node in 0..read_quorum - 1 {
|
||||
cluster.stop_node(node)?;
|
||||
}
|
||||
cluster.start().await?;
|
||||
for client in &clients {
|
||||
assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
|
||||
assert_quorum_object_body(client, &bucket, "below-quorum", &large).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_object(client: Client, payload: Vec<u8>, writer_id: usize) -> Result<(), String> {
|
||||
client
|
||||
.put_object()
|
||||
|
||||
@@ -1137,7 +1137,12 @@ async fn test_odm_admin_config_is_redacted_and_status_counts_match_the_source()
|
||||
let miss = env.raw_get(bucket, miss_key).await?;
|
||||
assert_eq!(miss.status, 404, "{}", String::from_utf8_lossy(&miss.body));
|
||||
}
|
||||
assert!(env.wait_local_listed(bucket, hit_key, SETTLE).await?);
|
||||
let (listed, _, _) = tokio::try_join!(
|
||||
env.wait_local_listed(bucket, hit_key, SETTLE),
|
||||
env.wait_for_status_counter(bucket, "/counters/pulled_objects_total/inline", 1, SETTLE),
|
||||
env.wait_for_status_counter(bucket, "/counters/pulled_bytes_total", body.len() as u64, SETTLE),
|
||||
)?;
|
||||
assert!(listed);
|
||||
|
||||
let status = env.status_json(bucket).await?;
|
||||
assert_eq!(status.pointer("/configured").and_then(Value::as_bool), Some(true), "{status}");
|
||||
|
||||
@@ -79,11 +79,6 @@ pub async fn test_sftp_compliance_suite() -> Result<()> {
|
||||
.await
|
||||
.map_err(|e| anyhow!("{}", e))?;
|
||||
|
||||
// Protocol listeners can accept connections before IAM is initialized.
|
||||
// A signed S3 request establishes readiness before the first SFTP login.
|
||||
let s3 = build_test_s3_client(&format!("http://{COMPLIANCE_RW_S3_ADDRESS}"));
|
||||
wait_for_s3_ready(&s3, 30).await?;
|
||||
|
||||
let (session, sftp) = connect_sftp_to(COMPLIANCE_RW_SFTP_ADDRESS).await?;
|
||||
|
||||
cmptst_01::run_medium_binary_round_trip(&sftp).await?;
|
||||
@@ -106,6 +101,8 @@ pub async fn test_sftp_compliance_suite() -> Result<()> {
|
||||
// reach the finalised object as x-amz-meta-* user metadata
|
||||
// through the CreateMultipartUpload input field. The S3 client
|
||||
// connects to the same rustfs process this suite already drives.
|
||||
let s3 = build_test_s3_client(&format!("http://{COMPLIANCE_RW_S3_ADDRESS}"));
|
||||
wait_for_s3_ready(&s3, 30).await?;
|
||||
cmptst_34::run_open_attrs_round_trip_multipart(&sftp, &s3).await?;
|
||||
|
||||
drop(sftp);
|
||||
|
||||
@@ -168,10 +168,6 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
.await
|
||||
.map_err(|e| anyhow!("{}", e))?;
|
||||
|
||||
// Protocol listeners can accept connections before IAM is initialized.
|
||||
let s3 = build_test_s3_client(S3_ENDPOINT);
|
||||
wait_for_s3_ready(&s3, S3_READY_ATTEMPTS).await?;
|
||||
|
||||
let (session, sftp) = connect_sftp().await?;
|
||||
|
||||
// --- 1. Subsystem canary: SFTP session reachable after password auth ---
|
||||
@@ -352,6 +348,16 @@ pub async fn test_sftp_core_operations() -> Result<()> {
|
||||
let _ = bad_session.disconnect(russh::Disconnect::ByApplication, "", "en").await;
|
||||
info!("PASS: bad-password authentication rejected");
|
||||
|
||||
// --- Cross-protocol setup: aws-sdk-s3 client against the same server ---
|
||||
// The rustfs binary spawned for this suite serves both SFTP on port
|
||||
// 9022 and S3 on port 9000. The S3 stack may need a moment to finish
|
||||
// initialising after TCP is listening, so list_buckets is polled
|
||||
// until it succeeds before any cross-protocol assertion runs.
|
||||
info!("Testing SFTP: prepare aws-sdk-s3 client and wait for S3 readiness");
|
||||
let s3 = build_test_s3_client(S3_ENDPOINT);
|
||||
wait_for_s3_ready(&s3, S3_READY_ATTEMPTS).await?;
|
||||
info!("PASS: S3 endpoint reachable from cross-protocol client");
|
||||
|
||||
// --- SFTP write, S3 read: SHA256 round-trip ---
|
||||
// SFTP creates the object, then assert_cross_protocol_sha_match
|
||||
// fetches it via both S3 GetObject and SFTP READ and compares
|
||||
@@ -516,9 +522,6 @@ pub async fn test_sftp_idle_timeout_disconnects() -> Result<()> {
|
||||
.await
|
||||
.map_err(|e| anyhow!("{}", e))?;
|
||||
|
||||
let s3 = build_test_s3_client(&format!("http://{IDLE_S3_ADDRESS}"));
|
||||
wait_for_s3_ready(&s3, S3_READY_ATTEMPTS).await?;
|
||||
|
||||
let (session, sftp) = connect_sftp_to(IDLE_SFTP_ADDRESS).await?;
|
||||
|
||||
// Confirm the session is live before the wait so a failure in the
|
||||
|
||||
@@ -4235,6 +4235,16 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<Destination><Bucket>{target_b_arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>matrix-and-tags</ID>
|
||||
<Priority>135</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><And><Prefix>and-tags/</Prefix><Tag><Key>env</Key><Value>prod</Value></Tag><Tag><Key>tier</Key><Value>gold</Value></Tag></And></Filter>
|
||||
<DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication>
|
||||
<DeleteReplication><Status>Enabled</Status></DeleteReplication>
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<Destination><Bucket>{target_b_arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>matrix-disabled</ID>
|
||||
<Priority>140</Priority>
|
||||
@@ -4289,6 +4299,7 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
|
||||
"matrix-prefix",
|
||||
"matrix-tag",
|
||||
"matrix-disabled",
|
||||
"matrix-and-tags",
|
||||
"matrix-priority-high",
|
||||
"Priority>200",
|
||||
"<Status>Disabled</Status>",
|
||||
@@ -4409,6 +4420,30 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
|
||||
put_single_tag_current(&source_client, source_bucket, "tagged/no-match.txt", "route", "tagged").await?;
|
||||
assert_replication_key_absent(&target_client_b, target_bucket_b, "tagged/no-match.txt", Duration::from_secs(3)).await?;
|
||||
|
||||
// S3 and MinIO both read `And.Tags` as AND: an object carrying only one of
|
||||
// the required tags is not admitted. Matching any single tag would push
|
||||
// data to a destination the rule never selected (backlog#2366 P1-1), and
|
||||
// the two-tag rule is the shape `mc replicate add --tags "k1=v1&k2=v2"`
|
||||
// writes, so a single-tag rule passing is not evidence for this.
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("and-tags/partial.txt")
|
||||
.tagging("env=prod")
|
||||
.body(ByteStream::from_static(b"one of two tags"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_replication_key_absent(&target_client_b, target_bucket_b, "and-tags/partial.txt", Duration::from_secs(3)).await?;
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("and-tags/full.txt")
|
||||
.tagging("env=prod&tier=gold")
|
||||
.body(ByteStream::from_static(b"both tags"))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_user_get_object(&target_client_b, target_bucket_b, "and-tags/full.txt").await?;
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
|
||||
@@ -1,181 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Build-bound evidence for scanner and heal restart tests.
|
||||
|
||||
use crate::common::ClusterTopology;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RestartEvidenceRun {
|
||||
schema: u32,
|
||||
run_id: String,
|
||||
source_revision: String,
|
||||
test_build: serde_json::Value,
|
||||
binary: EvidenceBuild,
|
||||
test_binary: EvidenceBuild,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct ScannerHealEvidenceCase {
|
||||
pub(crate) id: &'static str,
|
||||
pub(crate) oracle: &'static str,
|
||||
pub(crate) evidence: &'static str,
|
||||
pub(crate) unclean_shutdown_marker: bool,
|
||||
pub(crate) topology: EvidenceTopology,
|
||||
pub(crate) storage_class_standard: Option<&'static str>,
|
||||
pub(crate) erasure_set_drive_count: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct EvidenceTopology {
|
||||
pub(crate) nodes: usize,
|
||||
pub(crate) drives_per_node: usize,
|
||||
}
|
||||
|
||||
impl EvidenceTopology {
|
||||
pub(crate) const fn new(nodes: usize, drives_per_node: usize) -> Self {
|
||||
Self { nodes, drives_per_node }
|
||||
}
|
||||
|
||||
pub(crate) fn total_drives(self) -> usize {
|
||||
self.nodes * self.drives_per_node
|
||||
}
|
||||
|
||||
pub(crate) fn cluster_topology(self) -> ClusterTopology {
|
||||
ClusterTopology::single_pool_multidrive(self.nodes, self.drives_per_node)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RestartEvidenceContext {
|
||||
directory: PathBuf,
|
||||
run: RestartEvidenceRun,
|
||||
case: ScannerHealEvidenceCase,
|
||||
}
|
||||
|
||||
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file.read(&mut buffer)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..read]);
|
||||
}
|
||||
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||
}
|
||||
|
||||
pub(crate) fn restart_evidence_run(
|
||||
binary: &Path,
|
||||
case: ScannerHealEvidenceCase,
|
||||
) -> Result<Option<RestartEvidenceContext>, Box<dyn Error + Send + Sync>> {
|
||||
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||
return Ok(None);
|
||||
};
|
||||
if case.id.is_empty()
|
||||
|| case.oracle.is_empty()
|
||||
|| !case.oracle.ends_with(".json")
|
||||
|| case.oracle.contains('/')
|
||||
|| case.oracle.contains('\\')
|
||||
|| case.oracle.contains("..")
|
||||
|| !matches!(case.evidence, "process-restart" | "process-crash-restart")
|
||||
|| (case.evidence == "process-crash-restart") != case.unclean_shutdown_marker
|
||||
{
|
||||
return Err("invalid scanner/heal evidence case".into());
|
||||
}
|
||||
let directory = PathBuf::from(directory);
|
||||
let receipt = directory.join("run.json");
|
||||
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||
return Err("oversized scanner/heal execution receipt".into());
|
||||
}
|
||||
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
|
||||
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
|
||||
return Err("invalid scanner/heal execution identity".into());
|
||||
}
|
||||
let built = compiled_test_identity();
|
||||
for key in ["source_revision", "dirty", "lock_blob", "features"] {
|
||||
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
|
||||
}
|
||||
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
|
||||
assert_eq!(
|
||||
file_sha256(&std::env::current_exe()?)?,
|
||||
run.test_binary.sha256,
|
||||
"test executable must match the run receipt"
|
||||
);
|
||||
if directory.join(case.oracle).exists() {
|
||||
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||
}
|
||||
Ok(Some(RestartEvidenceContext { directory, run, case }))
|
||||
}
|
||||
|
||||
fn compiled_test_identity() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
|
||||
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
|
||||
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
|
||||
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
|
||||
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
|
||||
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
|
||||
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) struct RestartObservation {
|
||||
pub(crate) nodes: usize,
|
||||
pub(crate) drives_per_node: usize,
|
||||
pub(crate) pid_before: u32,
|
||||
pub(crate) pid_after: u32,
|
||||
pub(crate) unclean_shutdown_marker: bool,
|
||||
pub(crate) objects: Vec<serde_json::Value>,
|
||||
pub(crate) node_listings: Vec<Vec<String>>,
|
||||
}
|
||||
|
||||
impl RestartEvidenceContext {
|
||||
pub(crate) fn write(self, binary: &Path, observed: RestartObservation) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
assert_ne!(observed.pid_before, observed.pid_after, "target must be a new process");
|
||||
assert_eq!(file_sha256(binary)?, self.run.binary.sha256, "server build changed during restart");
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": self.case.id, "evidence": self.case.evidence,
|
||||
"run_id": self.run.run_id, "source_revision": self.run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": self.run.binary.sha256,
|
||||
"test_binary_sha256": self.run.test_binary.sha256,
|
||||
"topology": {"nodes": observed.nodes, "drives_per_node": observed.drives_per_node},
|
||||
"pid_before": observed.pid_before, "pid_after": observed.pid_after,
|
||||
"unclean_shutdown_marker": observed.unclean_shutdown_marker,
|
||||
"objects": observed.objects, "node_listings": observed.node_listings,
|
||||
});
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
}
|
||||
let mut output = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(self.directory.join(self.case.oracle))?;
|
||||
output.write_all(&data)?;
|
||||
output.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -563,7 +563,7 @@ pub mod store_list {
|
||||
}
|
||||
|
||||
pub mod storage {
|
||||
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
|
||||
@@ -285,20 +285,6 @@ fn peer_replay_state(audience: &str) -> PeerReplayState {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) fn clear_peer_replay_state_for_addr(addr: &str) -> std::io::Result<()> {
|
||||
let uri = addr
|
||||
.parse::<Uri>()
|
||||
.map_err(|_| std::io::Error::other("Invalid gRPC peer URI"))?;
|
||||
let audience = uri
|
||||
.authority()
|
||||
.map(|authority| normalize_tonic_rpc_audience(authority.as_str()))
|
||||
.ok_or_else(|| std::io::Error::other("Missing gRPC peer authority"))??;
|
||||
if let Ok(mut states) = PEER_REPLAY_STATES.lock() {
|
||||
states.remove(&audience);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn apply_peer_replay_response(
|
||||
audience: String,
|
||||
sent_state: PeerReplayState,
|
||||
@@ -633,13 +619,6 @@ mod tests {
|
||||
.remove(audience);
|
||||
}
|
||||
|
||||
fn set_peer_capability(audience: &str, state: PeerReplayState) {
|
||||
PEER_REPLAY_STATES
|
||||
.lock()
|
||||
.expect("peer capability cache lock must not be poisoned")
|
||||
.insert(audience.to_string(), state);
|
||||
}
|
||||
|
||||
fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> {
|
||||
let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest {
|
||||
args: "canonical mutation request".to_string(),
|
||||
@@ -1111,23 +1090,6 @@ mod tests {
|
||||
clear_peer_capability(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_peer_replay_state_for_addr_removes_normalized_audience() {
|
||||
let audience = "clear-peer-replay-state-test:9000";
|
||||
let boot_epoch = Uuid::new_v4();
|
||||
set_peer_capability(
|
||||
audience,
|
||||
PeerReplayState {
|
||||
boot_epoch: Some(boot_epoch),
|
||||
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
|
||||
},
|
||||
);
|
||||
|
||||
clear_peer_replay_state_for_addr("http://clear-peer-replay-state-test:9000").expect("peer URI should clear replay state");
|
||||
|
||||
assert_eq!(peer_replay_state(audience), PeerReplayState::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -13,9 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, clear_peer_replay_state_for_addr, embedded_tonic_status,
|
||||
gen_tonic_signature_interceptor, heal_control_time_out_client, is_network_like_status, message_has_network_needle,
|
||||
node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
|
||||
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -545,16 +544,6 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
|
||||
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
|
||||
}
|
||||
|
||||
fn heal_control_auth_may_need_replay_scope_refresh(err: &Error) -> bool {
|
||||
matches!(
|
||||
err,
|
||||
Error::Io(io_err)
|
||||
if embedded_tonic_status(io_err).is_some_and(|status| {
|
||||
status.code() == tonic::Code::Unauthenticated && status.message() == "No valid auth token"
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
|
||||
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
|
||||
if topology_member != expected_member {
|
||||
@@ -1731,72 +1720,45 @@ impl PeerRestClient {
|
||||
return Err(Error::other("heal control command exceeds size limit"));
|
||||
}
|
||||
let capability_probe = rustfs_protos::is_heal_control_capability_probe(&command);
|
||||
let result = self
|
||||
.heal_control_once(version, &topology_fingerprint, &command, capability_probe)
|
||||
.await;
|
||||
if result
|
||||
.as_ref()
|
||||
.err()
|
||||
.is_some_and(heal_control_auth_may_need_replay_scope_refresh)
|
||||
{
|
||||
self.prepare_heal_control_auth_retry().await;
|
||||
return self
|
||||
.finalize_result(
|
||||
self.heal_control_once(version, &topology_fingerprint, &command, capability_probe)
|
||||
.await,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.finalize_result(result).await
|
||||
}
|
||||
|
||||
async fn prepare_heal_control_auth_retry(&self) {
|
||||
if let Err(err) = clear_peer_replay_state_for_addr(&self.grid_host) {
|
||||
debug!(
|
||||
peer = %self.grid_host,
|
||||
error = %err,
|
||||
"could not clear heal control replay state before retry"
|
||||
);
|
||||
}
|
||||
self.evict_connection().await;
|
||||
}
|
||||
|
||||
async fn heal_control_once(
|
||||
&self,
|
||||
version: u32,
|
||||
topology_fingerprint: &str,
|
||||
command: &[u8],
|
||||
capability_probe: bool,
|
||||
) -> Result<Vec<u8>> {
|
||||
let mut client = self
|
||||
.get_heal_control_client()
|
||||
.await?
|
||||
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
|
||||
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
|
||||
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, topology_fingerprint, command)
|
||||
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
|
||||
let mut request = Request::new(HealControlRequest {
|
||||
version,
|
||||
topology_fingerprint: topology_fingerprint.to_string(),
|
||||
command: command.to_vec().into(),
|
||||
});
|
||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
||||
let response = client.heal_control(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer heal control failed without an error".to_string()),
|
||||
));
|
||||
}
|
||||
if !capability_probe {
|
||||
let canonical_response =
|
||||
rustfs_protos::canonical_heal_control_response_body(version, topology_fingerprint, command, &response.result)
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self
|
||||
.get_heal_control_client()
|
||||
.await?
|
||||
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
|
||||
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE);
|
||||
let canonical_body = rustfs_protos::canonical_heal_control_request_body(version, &topology_fingerprint, &command)
|
||||
.map_err(|_| Error::other("heal control request length cannot be represented"))?;
|
||||
let mut request = Request::new(HealControlRequest {
|
||||
version,
|
||||
topology_fingerprint: topology_fingerprint.clone(),
|
||||
command: command.clone().into(),
|
||||
});
|
||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical_body)?;
|
||||
let response = client.heal_control(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(Error::other(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer heal control failed without an error".to_string()),
|
||||
));
|
||||
}
|
||||
if !capability_probe {
|
||||
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
version,
|
||||
&topology_fingerprint,
|
||||
&command,
|
||||
&response.result,
|
||||
)
|
||||
.map_err(|_| Error::other("heal control response length cannot be represented"))?;
|
||||
validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
|
||||
}
|
||||
Ok(response.result.to_vec())
|
||||
validate_heal_control_response_proof(&canonical_response, &response.response_proof)?;
|
||||
}
|
||||
Ok(response.result.to_vec())
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Confirms that a peer supports the current heal-control coordination
|
||||
@@ -3766,22 +3728,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_control_auth_retry_is_limited_to_transport_auth_rejection() {
|
||||
assert!(heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||
tonic::Status::unauthenticated("No valid auth token")
|
||||
)));
|
||||
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||
tonic::Status::permission_denied("bad signature")
|
||||
)));
|
||||
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::from(
|
||||
tonic::Status::unauthenticated("application rejected heal control")
|
||||
)));
|
||||
assert!(!heal_control_auth_may_need_replay_scope_refresh(&Error::other(
|
||||
"Io error: code: 'Unauthenticated', message: \"No valid auth token\""
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_network_classifier_keeps_slow_peers_online() {
|
||||
// The per-RPC channel deadline (RUSTFS_INTERNODE_RPC_TIMEOUT, 30s)
|
||||
|
||||
@@ -3463,6 +3463,11 @@ impl PoolRebalanceActivationFence {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
pub(crate) static REBALANCE_ACTIVATION_LOCK_ATTEMPT: Arc<tokio::sync::Notify>;
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_pool_rebalance_activation_locks<S>(
|
||||
pool: Arc<S>,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
@@ -3473,17 +3478,21 @@ where
|
||||
NamespaceLock = rustfs_lock::NamespaceLockWrapper,
|
||||
>,
|
||||
{
|
||||
// Activation lock order is always pool.bin -> rebalance.bin.
|
||||
// Match entry admission: rebalance.bin -> pool.bin. An entry retains its
|
||||
// run read fence while target mutations acquire the pool metadata fence;
|
||||
// activation must not hold pool.bin while waiting for that entry to drain.
|
||||
let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
|
||||
#[cfg(test)]
|
||||
let _ = REBALANCE_ACTIVATION_LOCK_ATTEMPT.try_with(|attempted| attempted.notify_one());
|
||||
let rebalance_meta_guard = rebalance_meta_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(activation_rebalance_meta_lock_error)?;
|
||||
let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
|
||||
let pool_meta_guard = pool_meta_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(activation_pool_meta_lock_error)?;
|
||||
let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
|
||||
let rebalance_meta_guard = rebalance_meta_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(activation_rebalance_meta_lock_error)?;
|
||||
|
||||
Ok(PoolRebalanceActivationFence {
|
||||
pool_meta_guard,
|
||||
@@ -22094,7 +22103,7 @@ mod pools_tests {
|
||||
.resources
|
||||
.lock()
|
||||
.expect("activation lock recorder should not be poisoned"),
|
||||
vec![POOL_META_NAME.to_string(), REBAL_META_NAME.to_string()]
|
||||
vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()]
|
||||
);
|
||||
|
||||
let mut second_acquire = Box::pin(acquire_pool_rebalance_activation_locks(second.clone(), None));
|
||||
@@ -22110,10 +22119,50 @@ mod pools_tests {
|
||||
.resources
|
||||
.lock()
|
||||
.expect("activation lock recorder should not be poisoned"),
|
||||
vec![POOL_META_NAME.to_string(), REBAL_META_NAME.to_string()]
|
||||
vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_activation_cancellation_releases_rebalance_fence_while_pool_fence_is_contended() {
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
let pool = Arc::new(ActivationLockRecorder {
|
||||
lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
|
||||
owner: "activation-cancellation",
|
||||
resources: StdMutex::new(Vec::new()),
|
||||
});
|
||||
let pool_lock = pool
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("pool lock should be created");
|
||||
let pool_reader = pool_lock
|
||||
.get_read_lock(std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("ordinary mutation should hold the pool read fence");
|
||||
pool.resources.lock().expect("recorder should not be poisoned").clear();
|
||||
let mut activation = Box::pin(acquire_pool_rebalance_activation_locks(Arc::clone(&pool), None));
|
||||
assert!(matches!(futures::poll!(&mut activation), Poll::Pending));
|
||||
assert_eq!(
|
||||
*pool.resources.lock().expect("recorder should not be poisoned"),
|
||||
vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()],
|
||||
"activation must hold the run fence before waiting for the pool fence",
|
||||
);
|
||||
drop(activation);
|
||||
let rebalance_lock = pool
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
|
||||
.await
|
||||
.expect("run lock should be created");
|
||||
let run_writer = rebalance_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("cancelling activation must release its already-acquired run fence");
|
||||
assert!(
|
||||
!pool_reader.is_released(),
|
||||
"cancelling activation must not release another caller's pool fence"
|
||||
);
|
||||
assert!(!run_writer.is_lock_lost());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_receipt_run_token_changes_with_persisted_start_time() {
|
||||
let first = OffsetDateTime::from_unix_timestamp(1_000).expect("first run timestamp should be valid");
|
||||
|
||||
@@ -627,7 +627,13 @@ impl From<tokio::task::JoinError> for DiskError {
|
||||
impl Clone for DiskError {
|
||||
fn clone(&self) -> Self {
|
||||
match self {
|
||||
DiskError::Io(io_error) => DiskError::Io(std::io::Error::new(io_error.kind(), io_error.to_string())),
|
||||
DiskError::Io(io_error) => DiskError::Io(
|
||||
rustfs_rio::clone_internode_http_io_error(io_error)
|
||||
.and_then(std::io::Error::into_inner)
|
||||
// The helper derives a kind from the source; Clone must retain the original outer kind.
|
||||
.map(|source| std::io::Error::new(io_error.kind(), source))
|
||||
.unwrap_or_else(|| std::io::Error::new(io_error.kind(), io_error.to_string())),
|
||||
),
|
||||
DiskError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
|
||||
DiskError::Unexpected => DiskError::Unexpected,
|
||||
DiskError::CorruptedFormat => DiskError::CorruptedFormat,
|
||||
@@ -1265,6 +1271,49 @@ mod tests {
|
||||
assert!(!bad_request.is_retryable_internode_write_failure());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_http_clone_preserves_retryability_status_and_context() {
|
||||
use http::StatusCode;
|
||||
use rustfs_rio::InternodeHttpErrorKind::{ConnectionRefused, ConnectionReset, HttpStatus, Unknown};
|
||||
|
||||
for (kind, retryable) in [
|
||||
(ConnectionRefused, true),
|
||||
(ConnectionReset, true),
|
||||
(HttpStatus(StatusCode::TOO_MANY_REQUESTS), true),
|
||||
(HttpStatus(StatusCode::SERVICE_UNAVAILABLE), true),
|
||||
(HttpStatus(StatusCode::CONFLICT), true),
|
||||
(Unknown, false),
|
||||
(HttpStatus(StatusCode::BAD_REQUEST), false),
|
||||
(HttpStatus(StatusCode::INTERNAL_SERVER_ERROR), false),
|
||||
] {
|
||||
let original = DiskError::from(rustfs_rio::new_test_internode_http_io_error(kind));
|
||||
assert_eq!(original.internode_http_error_kind(), Some(kind));
|
||||
assert_eq!(original.is_retryable_internode_write_failure(), retryable);
|
||||
|
||||
let cloned = original.clone();
|
||||
assert_eq!(cloned, original, "clone must preserve the error bucket for {kind:?}");
|
||||
assert_eq!(
|
||||
cloned.is_retryable_internode_write_failure(),
|
||||
retryable,
|
||||
"clone changed retryability for {kind:?}"
|
||||
);
|
||||
assert_eq!(cloned.internode_http_error_kind(), Some(kind));
|
||||
if let HttpStatus(status) = kind {
|
||||
assert!(cloned.is_internode_http_status(status.as_u16()));
|
||||
}
|
||||
let DiskError::Io(io_error) = &cloned else {
|
||||
panic!("unmarked internode error must remain Io: {cloned:?}");
|
||||
};
|
||||
let source = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
.expect("clone must retain the structured internode error");
|
||||
assert_eq!(source.context().method(), "PUT");
|
||||
assert_eq!(source.context().target(), "/rustfs/rpc/put_file_stream");
|
||||
assert_eq!(source.context().operation(), Some(INTERNODE_OPERATION_PUT_FILE_STREAM));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_stream_conflict_is_not_a_retryable_put_file_failure() {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
@@ -1309,11 +1358,57 @@ mod tests {
|
||||
!error.is_retryable_internode_write_failure(),
|
||||
"read-operation 409 must not trigger put-file retry"
|
||||
);
|
||||
let cloned = error.clone();
|
||||
let reduced = crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(error)], &[], 1)
|
||||
.expect("the read conflict must remain the dominant error");
|
||||
for preserved in [&cloned, &reduced] {
|
||||
assert!(
|
||||
!preserved.is_retryable_internode_write_failure(),
|
||||
"cloning or reducing a read conflict must not turn it into a PUT retry"
|
||||
);
|
||||
assert!(preserved.is_internode_http_status(409));
|
||||
let DiskError::Io(io_error) = preserved else {
|
||||
panic!("read conflict must remain Io: {preserved:?}");
|
||||
};
|
||||
let source = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
.expect("read conflict must retain its request context");
|
||||
assert_eq!(source.context().method(), "GET");
|
||||
assert_eq!(source.context().target(), "/rustfs/rpc/read_file_stream");
|
||||
assert_eq!(
|
||||
source.context().operation(),
|
||||
Some(rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_READ_FILE_STREAM)
|
||||
);
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("isolated read-conflict test must finish within its budget");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_http_clone_preserves_outer_io_kind_and_message() {
|
||||
let source = rustfs_rio::new_test_internode_http_io_error(InternodeHttpErrorKind::ConnectionReset)
|
||||
.into_inner()
|
||||
.expect("the internode helper must provide a typed source");
|
||||
let original_io = io::Error::new(io::ErrorKind::InvalidData, source);
|
||||
let message = original_io.to_string();
|
||||
let original = DiskError::from(original_io);
|
||||
assert_eq!(original.internode_http_error_kind(), Some(InternodeHttpErrorKind::ConnectionReset));
|
||||
assert!(original.is_retryable_internode_write_failure());
|
||||
|
||||
let cloned = original.clone();
|
||||
let reduced = crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(original)], &[], 1)
|
||||
.expect("the wrapped internode error must remain the dominant error");
|
||||
for preserved in [&cloned, &reduced] {
|
||||
let DiskError::Io(io_error) = preserved else {
|
||||
panic!("the wrapped error must remain Io: {preserved:?}");
|
||||
};
|
||||
assert_eq!(io_error.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(io_error.to_string(), message);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_missing_errors_preserve_disk_error_types() {
|
||||
let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
|
||||
@@ -1325,6 +1420,17 @@ mod tests {
|
||||
assert_eq!(file_missing, DiskError::FileNotFound);
|
||||
assert_eq!(volume_missing, DiskError::VolumeNotFound);
|
||||
assert!(matches!(unmarked_server_error, DiskError::Io(_)));
|
||||
for missing in [file_missing, volume_missing] {
|
||||
assert_eq!(missing.clone(), missing);
|
||||
assert_eq!(
|
||||
crate::disk::error_reduce::reduce_write_quorum_errs(
|
||||
&[Some(missing.clone()), Some(missing.clone()), None],
|
||||
&[],
|
||||
2
|
||||
),
|
||||
Some(missing)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -226,6 +226,78 @@ mod tests {
|
||||
assert_eq!(res, Some(quorum_err));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_quorum_reduction_preserves_internode_http_identity() {
|
||||
use http::StatusCode;
|
||||
use rustfs_rio::InternodeHttpErrorKind::{ConnectionRefused, HttpStatus, Unknown};
|
||||
|
||||
for (kind, retryable) in [
|
||||
(ConnectionRefused, true),
|
||||
(HttpStatus(StatusCode::SERVICE_UNAVAILABLE), true),
|
||||
(HttpStatus(StatusCode::CONFLICT), true),
|
||||
(Unknown, false),
|
||||
(HttpStatus(StatusCode::BAD_REQUEST), false),
|
||||
] {
|
||||
// Construct both producer errors independently: the reducer owns the first clone.
|
||||
let first = Error::from(rustfs_rio::new_test_internode_http_io_error(kind));
|
||||
let second = Error::from(rustfs_rio::new_test_internode_http_io_error(kind));
|
||||
assert_eq!(first.internode_http_error_kind(), Some(kind));
|
||||
assert_eq!(second.internode_http_error_kind(), Some(kind));
|
||||
assert_eq!(first.is_retryable_internode_write_failure(), retryable);
|
||||
let errors = [Some(first), Some(second), None];
|
||||
let reduced = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, 2)
|
||||
.expect("two equal producer errors must dominate one successful write");
|
||||
|
||||
assert_eq!(Some(&reduced), errors[0].as_ref());
|
||||
assert_eq!(
|
||||
reduced.is_retryable_internode_write_failure(),
|
||||
retryable,
|
||||
"quorum reduction changed retryability for {kind:?}"
|
||||
);
|
||||
assert_eq!(reduced.internode_http_error_kind(), Some(kind));
|
||||
if let HttpStatus(status) = kind {
|
||||
assert!(reduced.is_internode_http_status(status.as_u16()));
|
||||
}
|
||||
let Error::Io(io_error) = &reduced else {
|
||||
panic!("the dominant error must remain Io: {reduced:?}");
|
||||
};
|
||||
let source = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<rustfs_rio::InternodeHttpError>())
|
||||
.expect("quorum reduction must retain the structured internode error");
|
||||
assert_eq!(source.context().method(), "PUT");
|
||||
assert_eq!(source.context().target(), "/rustfs/rpc/put_file_stream");
|
||||
assert_eq!(
|
||||
source.context().operation(),
|
||||
Some(rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clone_and_write_quorum_do_not_promote_non_retryable_errors() {
|
||||
use http::StatusCode;
|
||||
use rustfs_rio::InternodeHttpErrorKind::{HttpStatus, Unknown};
|
||||
|
||||
for original in [
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(Unknown)),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::BAD_REQUEST))),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::FORBIDDEN))),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::NOT_FOUND))),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
))),
|
||||
err_io("internode connection reset: PUT /rustfs/rpc/put_file_stream"),
|
||||
] {
|
||||
assert!(!original.is_retryable_internode_write_failure());
|
||||
let cloned = original.clone();
|
||||
let reduced =
|
||||
reduce_write_quorum_errs(&[Some(original)], &[], 1).expect("a non-retryable error must remain an error");
|
||||
assert!(!cloned.is_retryable_internode_write_failure());
|
||||
assert!(!reduced.is_retryable_internode_write_failure());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_count_errs() {
|
||||
let e1 = err_io("a");
|
||||
|
||||
@@ -22463,86 +22463,6 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn conditional_mrf_manifest_dir_fsync_failure_keeps_recovery_anchors() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
const MRF_COMMIT_MANIFEST_SLOT_0: &str = ".heal-mrf-commit.0.bin";
|
||||
const MRF_COMMIT_MANIFEST_SLOT_1: &str = ".heal-mrf-commit.1.bin";
|
||||
const MRF_SCOPED_JOURNAL_PATH: &str = "buckets/.heal/mrf/journal-scoped.bin";
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Relaxed);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let previous_manifest = Bytes::from_static(b"mrf-committed-manifest-v1");
|
||||
let successor_manifest = Bytes::from_static(b"mrf-committed-manifest-v2");
|
||||
let legacy_journal = Bytes::from_static(b"legacy-mrf-journal-records");
|
||||
|
||||
assert_eq!(
|
||||
disk.compare_and_update_file(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0, None, Some(previous_manifest.clone()),)
|
||||
.await
|
||||
.expect("previous MRF manifest should commit"),
|
||||
ConditionalFileUpdate::Updated
|
||||
);
|
||||
disk.write_all(RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH, legacy_journal.clone())
|
||||
.await
|
||||
.expect("legacy MRF journal should be retained");
|
||||
|
||||
let manifest_path = disk
|
||||
.get_object_path(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0)
|
||||
.expect("MRF manifest path should resolve");
|
||||
let parent = manifest_path.parent().expect("MRF manifest path should have a parent");
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(parent),
|
||||
"system metadata MRF manifest publication must fsync the metadata directory even under relaxed durability"
|
||||
);
|
||||
os::fsync_dir_recorder::set_failure(parent, ErrorKind::Other);
|
||||
|
||||
let err = disk
|
||||
.compare_and_update_file(
|
||||
RUSTFS_META_BUCKET,
|
||||
MRF_COMMIT_MANIFEST_SLOT_0,
|
||||
Some(previous_manifest.clone()),
|
||||
Some(successor_manifest),
|
||||
)
|
||||
.await
|
||||
.expect_err("directory fsync failure must fail the MRF manifest successor commit");
|
||||
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::Other));
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_0)
|
||||
.await
|
||||
.expect("previous committed MRF manifest should remain readable after rollback"),
|
||||
previous_manifest
|
||||
);
|
||||
|
||||
os::fsync_dir_recorder::set_failure(parent, ErrorKind::Other);
|
||||
let err = disk
|
||||
.compare_and_update_file(
|
||||
RUSTFS_META_BUCKET,
|
||||
MRF_COMMIT_MANIFEST_SLOT_1,
|
||||
None,
|
||||
Some(Bytes::from_static(b"first-successor-manifest")),
|
||||
)
|
||||
.await
|
||||
.expect_err("directory fsync failure must fail first MRF manifest commit");
|
||||
assert!(matches!(err, DiskError::Io(ref err) if err.kind() == ErrorKind::Other));
|
||||
assert!(
|
||||
matches!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, MRF_COMMIT_MANIFEST_SLOT_1).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"uncommitted first MRF manifest must be removed when no committed anchor exists"
|
||||
);
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, MRF_SCOPED_JOURNAL_PATH)
|
||||
.await
|
||||
.expect("legacy MRF journal should remain readable after failed manifest publication"),
|
||||
legacy_journal
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn conditional_file_update_dir_fsync_failure_removes_new_file_without_anchor() {
|
||||
|
||||
@@ -570,25 +570,11 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
pub(crate) static TEST_TIER_CONFIG_MGR: Arc<tokio::sync::RwLock<TierConfigMgr>>;
|
||||
}
|
||||
|
||||
pub(crate) async fn init_tier_config_mgr(store: Arc<ECStore>) -> Result<()> {
|
||||
#[cfg(not(test))]
|
||||
let handle = get_global_tier_config_mgr();
|
||||
#[cfg(test)]
|
||||
let handle = TEST_TIER_CONFIG_MGR
|
||||
.try_with(Arc::clone)
|
||||
.unwrap_or_else(|_| get_global_tier_config_mgr());
|
||||
let initial_reload = TierConfigMgr::reload_handle(&handle, store.clone()).await;
|
||||
if initial_reload.is_err() {
|
||||
// Keep local recovery active when the initial snapshot cannot be loaded.
|
||||
TierConfigMgr::request_committed_mutation_refresh(&handle).await;
|
||||
}
|
||||
TierConfigMgr::reload_handle(&handle, store.clone()).await?;
|
||||
tokio::spawn(TierConfigMgr::refresh_tier_config_handle(handle, store));
|
||||
initial_reload.map_err(Error::from)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -572,7 +572,7 @@ impl ECStore {
|
||||
where
|
||||
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
// Lock order: pool_meta_save_gate -> pool.bin -> rebalance.bin.
|
||||
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin.
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
|
||||
// Classify the durable rebalance record while holding both namespace
|
||||
|
||||
@@ -50,6 +50,11 @@ fn ensure_rebalance_entry_active(cancel: &CancellationToken) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static REBALANCE_ENTRY_RUN_FENCE_BARRIER: (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct RebalanceEntryTarget {
|
||||
bucket: String,
|
||||
@@ -256,9 +261,15 @@ impl ECStore {
|
||||
.sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
|
||||
|
||||
// Entry lock order is bucket incarnation -> activation_gate -> rebalance.bin -> movement gate.
|
||||
// Target capacity admission can then acquire pool.bin under the run fence.
|
||||
// Stop waits for in-flight entries through cleanup, but not for entries admitted later.
|
||||
ensure_rebalance_entry_active(&cancel)?;
|
||||
let run_guard = self.rebalance_run_guard(rebalance_id.as_ref(), "rebalance entry").await?;
|
||||
#[cfg(test)]
|
||||
if let Ok((arrived, release)) = REBALANCE_ENTRY_RUN_FENCE_BARRIER.try_with(Clone::clone) {
|
||||
arrived.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
let lock_lost_signal = run_guard.lock_lost_signal();
|
||||
#[cfg(test)]
|
||||
let _run_signal_test_fence = lock_lost_signal
|
||||
@@ -1237,6 +1248,130 @@ mod tests {
|
||||
assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn real_rebalance_entry_progresses_while_peer_activation_waits_for_run_fence() {
|
||||
const REBALANCE_ID: &str = "rebalance-peer-activation-lock-order";
|
||||
let (_temp_dirs, store, peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(
|
||||
active_rebalance_meta(REBALANCE_ID),
|
||||
))
|
||||
.await;
|
||||
assert!(!Arc::ptr_eq(&store.ctx, &peer.ctx), "node-local movement gates must be independent");
|
||||
{
|
||||
let mut meta = peer.rebalance_meta.write().await;
|
||||
let meta = meta.as_mut().expect("peer should know the durable run");
|
||||
meta.activation_gate = Arc::default();
|
||||
meta.cancel = None;
|
||||
}
|
||||
let bucket = crate::disk::RUSTFS_META_BUCKET;
|
||||
let object = "rebalance-peer-activation-object";
|
||||
let version_id = uuid::Uuid::new_v4();
|
||||
let payload = b"entry must drain before peer activation takes the pool fence".repeat(1024);
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let target_set = store.pools[1].get_disks_by_key(object);
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let mut writer = PutObjReader::from_vec(payload.clone());
|
||||
let source_before = source_set
|
||||
.put_object(bucket, object, &mut writer, &opts)
|
||||
.await
|
||||
.expect("source version should be written");
|
||||
let entry = metacache_entry_from_source(&source_set, bucket, object).await;
|
||||
let arrived = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
// JoinSet aborts both scoped tasks if an assertion or timeout fails.
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
let entry_store = Arc::clone(&store);
|
||||
tasks.spawn(
|
||||
REBALANCE_ENTRY_RUN_FENCE_BARRIER.scope((Arc::clone(&arrived), Arc::clone(&release)), async move {
|
||||
entry_store
|
||||
.rebalance_entry(
|
||||
RebalanceEntryTarget {
|
||||
bucket: bucket.to_string(),
|
||||
pool_index: 0,
|
||||
},
|
||||
entry,
|
||||
source_set,
|
||||
Arc::new(RebalanceBucketConfigs::default()),
|
||||
Arc::from(REBALANCE_ID),
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
}),
|
||||
);
|
||||
tokio::time::timeout(StdDuration::from_secs(30), arrived.notified())
|
||||
.await
|
||||
.expect("real entry must acquire its persisted run read fence");
|
||||
|
||||
let attempted = Arc::new(tokio::sync::Notify::new());
|
||||
let peer_pool = Arc::clone(&peer.pools[0]);
|
||||
let (activation_done, activation_result) = tokio::sync::oneshot::channel();
|
||||
tasks.spawn(
|
||||
crate::core::pools::REBALANCE_ACTIVATION_LOCK_ATTEMPT.scope(Arc::clone(&attempted), async move {
|
||||
let result = peer.fence_rebalance_worker_activation(peer_pool, REBALANCE_ID).await;
|
||||
let result = result.map(|fence| match fence {
|
||||
super::super::control::RebalanceWorkerActivationFence::Ready(fence) => {
|
||||
fence.ensure_held().expect("peer activation must retain both fences");
|
||||
}
|
||||
super::super::control::RebalanceWorkerActivationFence::NotStartedTerminal => {
|
||||
panic!("the paused entry's run must still require activation");
|
||||
}
|
||||
});
|
||||
activation_done.send(result).expect("activation receiver should remain alive");
|
||||
Ok(RebalanceEntryOutcome::Completed)
|
||||
}),
|
||||
);
|
||||
tokio::time::timeout(StdDuration::from_secs(30), attempted.notified())
|
||||
.await
|
||||
.expect("peer activation must attempt the persisted rebalance write fence");
|
||||
release.notify_one();
|
||||
|
||||
tokio::time::timeout(StdDuration::from_secs(30), async {
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
assert!(matches!(
|
||||
result
|
||||
.expect("scoped task must not panic")
|
||||
.expect("entry must not fail or defer"),
|
||||
RebalanceEntryOutcome::Completed
|
||||
));
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("entry and peer activation must both make progress");
|
||||
activation_result
|
||||
.await
|
||||
.expect("peer activation result should be sent")
|
||||
.expect("peer activation must not time out behind the entry it blocks");
|
||||
|
||||
let mut reader = target_set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("the exact target version must be readable");
|
||||
let mut actual = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("target body should drain completely");
|
||||
assert_eq!(actual, payload);
|
||||
assert_eq!(reader.object_info.version_id, source_before.version_id);
|
||||
assert_eq!(reader.object_info.etag, source_before.etag);
|
||||
assert_eq!(reader.object_info.mod_time, source_before.mod_time);
|
||||
let source_error = store.pools[0]
|
||||
.get_object_info(bucket, object, &opts)
|
||||
.await
|
||||
.expect_err("completed entry must clean up the source version");
|
||||
assert!(crate::error::is_err_object_not_found(&source_error) || crate::error::is_err_version_not_found(&source_error));
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let stats = &meta.as_ref().expect("local run must remain installed").pool_stats[0];
|
||||
assert_eq!(stats.num_objects, 1);
|
||||
assert_eq!(stats.num_versions, 1);
|
||||
assert_eq!(stats.cleanup_warnings.count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn real_rebalance_run_fence_loss_before_target_commit_preserves_target_and_source() {
|
||||
|
||||
@@ -1907,6 +1907,124 @@ fn test_is_transient_rebalance_error_accepts_wrapped_disk_timeout() {
|
||||
assert!(is_transient_rebalance_error(&Error::Io(std::io::Error::other(DiskError::Timeout))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rebalance_stage_wrapped_transient_errors_remain_retryable() {
|
||||
let cases = [
|
||||
Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
|
||||
Error::Lock(rustfs_lock::LockError::network(
|
||||
"peer unavailable",
|
||||
std::io::Error::from(std::io::ErrorKind::ConnectionReset),
|
||||
)),
|
||||
Error::SlowDown,
|
||||
Error::ErasureReadQuorum,
|
||||
Error::ErasureWriteQuorum,
|
||||
Error::Io(std::io::Error::other(DiskError::Timeout)),
|
||||
Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
|
||||
];
|
||||
for mut error in cases {
|
||||
for depth in 0..=3 {
|
||||
assert!(is_transient_rebalance_error(&error), "transient source lost at depth {depth}: {error:?}");
|
||||
assert!(
|
||||
should_defer_rebalance_entry_failure(&error),
|
||||
"exhausted transient entries must be deferred"
|
||||
);
|
||||
assert!(should_retry_rebalance_listing(&error, 0, 3));
|
||||
assert!(
|
||||
!should_retry_rebalance_listing(&error, 2, 3),
|
||||
"wrapping must not bypass the attempt limit"
|
||||
);
|
||||
error = data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"baseline/00042.bin",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rebalance_stage_wrapped_terminal_errors_remain_terminal() {
|
||||
let cases = [
|
||||
Error::FileAccessDenied,
|
||||
Error::FileCorrupt,
|
||||
Error::OperationCanceled,
|
||||
Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()),
|
||||
Error::Lock(rustfs_lock::LockError::already_locked("bucket/object", "owner")),
|
||||
Error::other("permission denied"),
|
||||
];
|
||||
for mut error in cases {
|
||||
for depth in 0..=3 {
|
||||
assert!(
|
||||
!is_transient_rebalance_error(&error),
|
||||
"terminal source must survive depth {depth}: {error:?}"
|
||||
);
|
||||
assert!(!should_defer_rebalance_entry_failure(&error));
|
||||
// Object names are untrusted context, not evidence of a transient failure.
|
||||
error = data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"remote lock rpc timed out",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rebalance_stage_wrapped_lock_timeout_retries_real_migration_loop() {
|
||||
for succeeds_on_retry in [true, false] {
|
||||
let backend = MigrationBackendSpy::new(None, None);
|
||||
let attempts = AtomicUsize::new(0);
|
||||
let waits = AtomicUsize::new(0);
|
||||
let mut transfer = |_, _, _| {
|
||||
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async move {
|
||||
if succeeds_on_retry && attempt > 0 {
|
||||
return Ok(());
|
||||
}
|
||||
Err(data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"baseline/00042.bin",
|
||||
Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
|
||||
))
|
||||
}
|
||||
};
|
||||
let version = version_normal();
|
||||
let result = migrate_entry_version_with_retry_wait(
|
||||
&backend,
|
||||
"bucket".to_string(),
|
||||
0,
|
||||
&version,
|
||||
None,
|
||||
3,
|
||||
false,
|
||||
&mut transfer,
|
||||
|_: String, _: String, _: ObjectOptions| async { Ok::<_, Error>(ObjectInfo::default()) },
|
||||
|_| {
|
||||
waits.fetch_add(1, Ordering::SeqCst);
|
||||
std::future::ready(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.moved, succeeds_on_retry);
|
||||
assert_eq!(result.failed, !succeeds_on_retry);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), if succeeds_on_retry { 2 } else { 3 });
|
||||
assert_eq!(backend.get_calls(), attempts.load(Ordering::SeqCst));
|
||||
assert_eq!(waits.load(Ordering::SeqCst), attempts.load(Ordering::SeqCst) - 1);
|
||||
if !succeeds_on_retry {
|
||||
assert_eq!(result.stage, Some("write_target"));
|
||||
assert!(should_defer_rebalance_entry_failure(
|
||||
result.error.as_ref().expect("exhaustion must retain its source error")
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_transient_rebalance_error_accepts_io_timeout_message() {
|
||||
assert!(is_transient_rebalance_error(&Error::Io(std::io::Error::other("timeout"))));
|
||||
|
||||
@@ -244,6 +244,7 @@ pub(super) fn resolve_rebalance_bucket_result(
|
||||
}
|
||||
|
||||
pub(super) fn is_transient_rebalance_error(err: &Error) -> bool {
|
||||
let err = rebalance_error_source(err);
|
||||
match err {
|
||||
Error::SlowDown
|
||||
| Error::ErasureReadQuorum
|
||||
@@ -256,6 +257,15 @@ pub(super) fn is_transient_rebalance_error(err: &Error) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn rebalance_error_source(mut err: &Error) -> &Error {
|
||||
// Stage context contains object names, so classify the preserved source,
|
||||
// not timeout-like text supplied by an object name. Iterate nested stages.
|
||||
while let Some(source) = crate::data_movement::data_movement_stage_source(err) {
|
||||
err = source;
|
||||
}
|
||||
err
|
||||
}
|
||||
|
||||
fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
|
||||
match err {
|
||||
rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::Network { .. } => true,
|
||||
@@ -309,6 +319,7 @@ pub(super) fn rebalance_listing_retry_delay(attempt: usize) -> Duration {
|
||||
}
|
||||
|
||||
fn is_rebalance_lock_or_rpc_timeout(err: &Error) -> bool {
|
||||
let err = rebalance_error_source(err);
|
||||
match err {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. }) | Error::Lock(rustfs_lock::LockError::Network { .. }) => true,
|
||||
Error::Io(io_err) => is_rebalance_lock_or_rpc_timeout_message(&io_err.to_string()),
|
||||
@@ -585,3 +596,48 @@ impl SetDisks {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod error_source_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn stage_wrapped_errors_select_the_source_backoff_policy() {
|
||||
let cases = [
|
||||
(
|
||||
Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
|
||||
true,
|
||||
),
|
||||
(
|
||||
Error::Lock(rustfs_lock::LockError::network(
|
||||
"peer unavailable",
|
||||
std::io::Error::from(std::io::ErrorKind::ConnectionReset),
|
||||
)),
|
||||
true,
|
||||
),
|
||||
(Error::other("remote lock rpc timed out"), true),
|
||||
(Error::SlowDown, false),
|
||||
(Error::Io(std::io::Error::other(DiskError::Timeout)), false),
|
||||
(Error::FileAccessDenied, false),
|
||||
];
|
||||
for (mut error, lock_backoff) in cases {
|
||||
for depth in 0..=3 {
|
||||
assert_eq!(
|
||||
is_rebalance_lock_or_rpc_timeout(&error),
|
||||
lock_backoff,
|
||||
"wrong backoff at depth {depth}: {error:?}"
|
||||
);
|
||||
if !lock_backoff {
|
||||
assert_eq!(rebalance_migration_retry_delay(1, &error), REBALANCE_MIGRATION_RETRY_BASE_DELAY * 2);
|
||||
}
|
||||
error = crate::data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"remote lock rpc timed out",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -326,14 +326,15 @@ impl SetDisks {
|
||||
let parity_blocks = Self::common_parity(&parities, default_parity_count as i32);
|
||||
|
||||
if parity_blocks < 0 {
|
||||
// No parity value reached read quorum. Distinguish two cases:
|
||||
// enough disks answered with valid-looking metadata that simply
|
||||
// cannot be reconciled (corrupt/foreign entries — retrying cannot
|
||||
// help, and heal should see Corrupt, rustfs#5801) versus too few
|
||||
// healthy answers (a genuine quorum condition where retry may
|
||||
// succeed once disks recover).
|
||||
// A consistent layout can require more replies than the initial
|
||||
// half-set probe. Reaching that probe alone is not corruption;
|
||||
// only invalid or conflicting healthy replies establish that.
|
||||
let healthy_replies = errs.iter().filter(|err| err.is_none()).count();
|
||||
if healthy_replies >= expected_rquorum {
|
||||
let consistent_parity = parities
|
||||
.iter()
|
||||
.find(|&&parity| parity >= 0)
|
||||
.filter(|&&parity| parities.iter().filter(|&&candidate| candidate == parity).count() == healthy_replies);
|
||||
if healthy_replies >= expected_rquorum && consistent_parity.is_none() {
|
||||
error!(
|
||||
"object_quorum_from_meta: irreconcilable parity across {healthy_replies} healthy replies (corrupt metadata), errs={errs:?}"
|
||||
);
|
||||
@@ -1652,6 +1653,40 @@ mod tests {
|
||||
assert_eq!(err, DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consistent_parity_below_its_data_shard_quorum_is_not_corruption() {
|
||||
for (drive_count, parity) in [(6, 2), (8, 2), (12, 4)] {
|
||||
let data = drive_count - parity;
|
||||
let mut metas = (1..=drive_count)
|
||||
.map(|index| {
|
||||
let mut info = FileInfo::new("bucket/object", data, parity);
|
||||
info.size = 1024;
|
||||
info.erasure.index = index;
|
||||
info
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut errs = vec![Some(DiskError::DiskNotFound); drive_count];
|
||||
errs[..data].fill(None);
|
||||
assert_eq!(
|
||||
SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect("exact data quorum should resolve"),
|
||||
(data as i32, data as i32)
|
||||
);
|
||||
|
||||
errs[data - 1] = Some(DiskError::DiskNotFound);
|
||||
assert_eq!(
|
||||
SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect_err("one fewer shard cannot resolve"),
|
||||
DiskError::ErasureReadQuorum,
|
||||
"layout {drive_count}/{parity} has consistent metadata but insufficient shards"
|
||||
);
|
||||
|
||||
metas[0].erasure.parity_blocks = usize::MAX;
|
||||
assert_eq!(
|
||||
SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect_err("corrupt healthy replies must be rejected"),
|
||||
DiskError::FileCorrupt
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Too few healthy replies remains a genuine quorum condition where a
|
||||
/// retry may succeed once disks recover.
|
||||
#[test]
|
||||
|
||||
@@ -865,6 +865,7 @@ pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, ren
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
pub(crate) use ops::bucket::BucketInfoQuorum;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::hermetic_set_disks_isolated;
|
||||
|
||||
@@ -21,12 +21,72 @@
|
||||
|
||||
use super::super::{
|
||||
BUCKET_OP_IGNORED_ERRS, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DiskError, Error, HashMap,
|
||||
MakeBucketOptions, Result, SetDisks, is_reserved_or_invalid_bucket, join_all, reduce_write_quorum_errs,
|
||||
MakeBucketOptions, Result, SetDisks, is_reserved_or_invalid_bucket, join_all, reduce_read_quorum_errs,
|
||||
reduce_write_quorum_errs,
|
||||
};
|
||||
use crate::api::bucket::metadata_sys;
|
||||
use crate::disk::DiskAPI;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum BucketInfoQuorum {
|
||||
Read,
|
||||
Write,
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(crate) async fn stat_bucket_with_quorum(&self, bucket: &str, quorum: BucketInfoQuorum) -> Result<BucketInfo> {
|
||||
let disks = self.disk_inventory().await;
|
||||
let disk_count = disks.len();
|
||||
let mut futures = Vec::with_capacity(disk_count);
|
||||
for disk in disks {
|
||||
let bucket = bucket.to_string();
|
||||
futures.push(async move {
|
||||
match disk {
|
||||
Some(disk) => disk.stat_volume(&bucket).await,
|
||||
None => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
let mut infos = Vec::with_capacity(results.len());
|
||||
let mut errs = Vec::with_capacity(results.len());
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(info) => {
|
||||
infos.push(Some(info));
|
||||
errs.push(None);
|
||||
}
|
||||
Err(err) => {
|
||||
infos.push(None);
|
||||
errs.push(Some(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let error = match quorum {
|
||||
// Bucket mutations use a majority regardless of object storage
|
||||
// class. A namespace read must intersect that majority; object
|
||||
// readers still enforce the persisted layout's data-shard quorum.
|
||||
BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count.div_ceil(2).max(1)),
|
||||
BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count / 2 + 1),
|
||||
};
|
||||
if let Some(err) = error {
|
||||
return Err(err.into());
|
||||
}
|
||||
|
||||
infos
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.next()
|
||||
.map(|info| BucketInfo {
|
||||
name: info.name,
|
||||
created: info.created,
|
||||
..Default::default()
|
||||
})
|
||||
.ok_or(Error::VolumeNotFound)
|
||||
}
|
||||
|
||||
pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec<BucketInfo>, bool)> {
|
||||
let disks = self.disk_inventory().await;
|
||||
let write_quorum = (disks.len() / 2) + 1;
|
||||
@@ -131,59 +191,12 @@ impl BucketOperations for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn get_bucket_info(&self, bucket: &str, _opts: &BucketOptions) -> Result<BucketInfo> {
|
||||
let disks = self.disk_inventory().await;
|
||||
let write_quorum = (disks.len() / 2) + 1;
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
let bucket = bucket.to_string();
|
||||
futures.push(async move {
|
||||
match disk {
|
||||
Some(disk) => disk.stat_volume(&bucket).await,
|
||||
None => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
let mut infos = Vec::with_capacity(results.len());
|
||||
let mut errs = Vec::with_capacity(results.len());
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(info) => {
|
||||
infos.push(Some(info));
|
||||
errs.push(None);
|
||||
}
|
||||
Err(err) => {
|
||||
infos.push(None);
|
||||
errs.push(Some(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(err.into());
|
||||
}
|
||||
|
||||
let mut versioning = false;
|
||||
let mut object_locking = false;
|
||||
let mut info = self.stat_bucket_with_quorum(bucket, BucketInfoQuorum::Write).await?;
|
||||
if let Ok(sys) = metadata_sys::get(bucket).await {
|
||||
versioning = sys.versioning();
|
||||
object_locking = sys.object_locking();
|
||||
info.versioning = sys.versioning();
|
||||
info.object_locking = sys.object_locking();
|
||||
}
|
||||
|
||||
infos
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.next()
|
||||
.map(|info| BucketInfo {
|
||||
name: info.name,
|
||||
created: info.created,
|
||||
versioning,
|
||||
object_locking,
|
||||
..Default::default()
|
||||
})
|
||||
.ok_or(Error::VolumeNotFound)
|
||||
Ok(info)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::bucket::{
|
||||
};
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use futures::stream::{self, StreamExt};
|
||||
@@ -772,17 +772,30 @@ impl ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(crate) async fn get_bucket_info_from_sets(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
|
||||
self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Write)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_bucket_info_from_sets_with_quorum(
|
||||
&self,
|
||||
bucket: &str,
|
||||
opts: &BucketOptions,
|
||||
quorum: BucketInfoQuorum,
|
||||
) -> Result<BucketInfo> {
|
||||
// One host may participate in several pools after expansion. Resolve the
|
||||
// namespace against each erasure set so disks from different pools can
|
||||
// never be combined into one bucket quorum.
|
||||
// Bucket validation is request-path IO. Keep the previous peer fanout's
|
||||
// latency shape by probing every set concurrently; scanner listings use
|
||||
// a separate bounded path below because they run continuously.
|
||||
let mut scoped_results =
|
||||
futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
|
||||
(pool_index, set_index, set.get_bucket_info(bucket, opts).await)
|
||||
}))
|
||||
.await;
|
||||
let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
|
||||
let result = match quorum {
|
||||
BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
|
||||
BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
|
||||
};
|
||||
(pool_index, set_index, result)
|
||||
}))
|
||||
.await;
|
||||
scoped_results.sort_unstable_by_key(|(pool_index, set_index, _)| (*pool_index, *set_index));
|
||||
|
||||
let mut first_info = None;
|
||||
@@ -806,7 +819,11 @@ impl ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
|
||||
let mut info = self.get_bucket_info_from_sets(bucket, opts).await?;
|
||||
let mut info = match self.get_bucket_info_from_sets(bucket, opts).await {
|
||||
Ok(info) => info,
|
||||
Err(Error::ErasureWriteQuorum) => return self.get_bucket_info_at_read_quorum(bucket, opts).await,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
if let Ok(sys) = metadata_sys::get_in(&self.ctx, bucket).await {
|
||||
if should_override_created_from_metadata(sys.created) {
|
||||
@@ -819,6 +836,35 @@ impl ECStore {
|
||||
Ok(info)
|
||||
}
|
||||
|
||||
async fn get_bucket_info_at_read_quorum(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
|
||||
// Lock order: bucket lifecycle -> internal metadata object read locks.
|
||||
// Keep create/delete from changing the namespace while a read quorum
|
||||
// confirms both physical presence and persisted bucket metadata.
|
||||
let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
await_bucket_namespace_operation(Some(&guard), bucket, "bucket read quorum validation", async {
|
||||
let mut info = self
|
||||
.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
|
||||
.await?;
|
||||
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
if !persisted {
|
||||
// A minority of directories left by failed creation is not an
|
||||
// authoritative bucket. Never turn fabricated defaults into
|
||||
// permission to serve degraded reads.
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
if metadata.name != bucket {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
if should_override_created_from_metadata(metadata.created) {
|
||||
info.created = Some(metadata.created);
|
||||
}
|
||||
info.versioning = metadata.versioning();
|
||||
info.object_locking = metadata.object_locking();
|
||||
Ok(info)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
|
||||
// TODO(backlog): support cached bucket listing via opts.cached
|
||||
@@ -1049,7 +1095,7 @@ mod tests {
|
||||
run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget,
|
||||
should_override_created_from_metadata, validate_table_bucket_delete_allowed,
|
||||
};
|
||||
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
|
||||
use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix};
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
|
||||
use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
|
||||
@@ -1076,6 +1122,7 @@ mod tests {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Notify, OnceCell};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
@@ -1359,11 +1406,18 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn setup_multi_pool_bucket_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
setup_bucket_quorum_test_env(&[4, 4], None).await
|
||||
}
|
||||
|
||||
async fn setup_bucket_quorum_test_env(
|
||||
drives_per_pool: &[usize],
|
||||
standard_parity: Option<usize>,
|
||||
) -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool bucket test directory should be created");
|
||||
let mut pools = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
for (pool_index, &drive_count) in drives_per_pool.iter().enumerate() {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
for disk_index in 0..drive_count {
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
@@ -1378,7 +1432,7 @@ mod tests {
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
drives_per_set: drive_count,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("bucket-test-pool-{pool_index}"),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
@@ -1399,9 +1453,12 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("multi-pool ECStore should initialize");
|
||||
let storage_class =
|
||||
crate::config::storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4, 4])
|
||||
.expect("multi-pool storage class should match both four-disk pools");
|
||||
let mut storage_class_kvs = rustfs_config::server_config::KVS::new();
|
||||
if let Some(parity) = standard_parity {
|
||||
storage_class_kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), format!("EC:{parity}"));
|
||||
}
|
||||
let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(&storage_class_kvs, drives_per_pool)
|
||||
.expect("storage class should match every test erasure set");
|
||||
for pool in &ecstore.pools {
|
||||
for set in &pool.disk_set {
|
||||
set.set_test_storage_class_config(storage_class.clone());
|
||||
@@ -2067,6 +2124,218 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_info_read_quorum_tracks_erasure_layout() {
|
||||
for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (5, 2), (6, 3), (8, 4), (6, 2), (12, 6)] {
|
||||
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[drive_count], Some(parity)).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("read-quorum-{drive_count}-{parity}");
|
||||
let object = "uncached-object";
|
||||
let body = b"erasure read quorum must follow the persisted layout".repeat(32_768);
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("healthy namespace should accept bucket creation");
|
||||
store
|
||||
.put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("healthy erasure set should accept the seed object");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let lock = set
|
||||
.new_ns_lock(&bucket, object)
|
||||
.await
|
||||
.expect("seed namespace lock should resolve");
|
||||
drop(
|
||||
lock.get_write_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("seed physical fanout must finish before taking disks offline"),
|
||||
);
|
||||
if (drive_count, parity) == (6, 3) {
|
||||
let mut kvs = rustfs_config::server_config::KVS::new();
|
||||
kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
|
||||
set.set_test_storage_class_config(
|
||||
crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &[drive_count])
|
||||
.expect("a later storage-class change must not raise old objects' read quorum"),
|
||||
);
|
||||
}
|
||||
|
||||
let offline_indexes = (0..parity).collect::<Vec<_>>();
|
||||
let offline = take_set_disks_offline(&store, set, &offline_indexes).await;
|
||||
let info = store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect("bucket validation must admit the object's exact read quorum");
|
||||
assert_eq!(info.name, bucket);
|
||||
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the persisted layout should remain readable at its exact data-shard quorum");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("quorum read should reconstruct the body");
|
||||
assert_eq!(restored, body, "layout {drive_count}/{parity} must retain exact object contents");
|
||||
drop(reader);
|
||||
|
||||
if drive_count - parity == drive_count / 2 {
|
||||
let error = store
|
||||
.get_bucket_info_from_sets(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect_err("bucket mutations must retain their majority namespace check");
|
||||
assert_eq!(error, StorageError::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
let below_quorum = take_set_disks_offline(&store, set, &[parity]).await;
|
||||
let read = store
|
||||
.get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
|
||||
.await;
|
||||
match read {
|
||||
Ok(mut reader) => assert!(
|
||||
reader.stream.read_to_end(&mut Vec::new()).await.is_err(),
|
||||
"layout {drive_count}/{parity} must reject fewer than its data-shard quorum"
|
||||
),
|
||||
Err(error) => assert!(
|
||||
matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
|
||||
"a missing shard must report read quorum loss, got {error}"
|
||||
),
|
||||
}
|
||||
restore_set_disks(&store, set, below_quorum).await;
|
||||
restore_set_disks(&store, set, offline).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_info_read_quorum_is_scoped_to_each_erasure_set() {
|
||||
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4, 6], None).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "read-quorum-mixed-pools";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("healthy pools should accept bucket creation");
|
||||
|
||||
let first_set = &store.pools[0].disk_set[0];
|
||||
let second_set = &store.pools[1].disk_set[0];
|
||||
let first_offline = take_set_disks_offline(&store, first_set, &[0, 1]).await;
|
||||
let second_offline = take_set_disks_offline(&store, second_set, &[0, 1, 2]).await;
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect("each set independently satisfies its namespace read quorum");
|
||||
|
||||
for (set, extra_disk) in [(first_set, 2), (second_set, 3)] {
|
||||
let extra_offline = take_set_disks_offline(&store, set, &[extra_disk]).await;
|
||||
assert_eq!(
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect_err("another pool must not subsidize a set below its read quorum"),
|
||||
StorageError::ErasureReadQuorum
|
||||
);
|
||||
restore_set_disks(&store, set, extra_offline).await;
|
||||
}
|
||||
restore_set_disks(&store, first_set, first_offline).await;
|
||||
restore_set_disks(&store, second_set, second_offline).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_info_read_quorum_requires_authoritative_metadata() {
|
||||
for state in ["missing", "corrupt", "foreign", "incarnation"] {
|
||||
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("read-quorum-{state}-metadata");
|
||||
let mut metadata = if state == "missing" {
|
||||
store
|
||||
.make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("simulate directories left before bucket metadata is published");
|
||||
BucketMetadata::new(&bucket)
|
||||
} else {
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("healthy bucket should publish metadata");
|
||||
metadata_sys::get_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("seed metadata should be cached")
|
||||
.as_ref()
|
||||
.clone()
|
||||
};
|
||||
let path = metadata.save_file_path();
|
||||
match state {
|
||||
"corrupt" => crate::config::com::save_config(store.clone(), &path, b"corrupt".to_vec())
|
||||
.await
|
||||
.expect("persist corrupt metadata while the cached copy remains valid"),
|
||||
"foreign" => {
|
||||
metadata.name = "different-bucket".to_string();
|
||||
let mut encoded = vec![1, 0, 1, 0];
|
||||
encoded.extend(metadata.marshal_msg().expect("foreign metadata should encode"));
|
||||
crate::config::com::save_config(store.clone(), &path, encoded)
|
||||
.await
|
||||
.expect("persist metadata for a different bucket at the requested path");
|
||||
}
|
||||
"incarnation" => crate::bucket::metadata::save_bucket_incarnation(store.clone(), &bucket, Uuid::new_v4())
|
||||
.await
|
||||
.expect("persist a different bucket generation"),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
|
||||
let error = store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect_err("read admission must not trust residual directories or cached metadata");
|
||||
match state {
|
||||
"missing" => assert_eq!(error, StorageError::ErasureReadQuorum),
|
||||
"foreign" => assert_eq!(error, StorageError::FileCorrupt),
|
||||
"incarnation" => assert!(error.to_string().contains("sidecar does not match bucket metadata")),
|
||||
"corrupt" => assert!(error.to_string().contains("format invalid"), "unexpected corruption error: {error}"),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
restore_set_disks(&store, set, offline).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_info_read_quorum_accepts_persisted_legacy_metadata() {
|
||||
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "interop";
|
||||
store
|
||||
.make_bucket_on_sets(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("legacy bucket directories should exist");
|
||||
let hex = include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")
|
||||
.split_whitespace()
|
||||
.collect::<String>();
|
||||
let body = (0..hex.len())
|
||||
.step_by(2)
|
||||
.map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("pinned MinIO metadata fixture"))
|
||||
.collect();
|
||||
crate::config::com::save_config(store.clone(), &BucketMetadata::new(bucket).save_file_path(), body)
|
||||
.await
|
||||
.expect("legacy metadata should be persisted without an incarnation sidecar");
|
||||
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
|
||||
let info = store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect("persisted MinIO metadata should authorize reads at the namespace read quorum");
|
||||
assert_eq!(info.name, bucket);
|
||||
assert!(info.versioning);
|
||||
assert!(info.object_locking);
|
||||
restore_set_disks(&store, set, offline).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_namespace_reads_report_missing_when_every_set_is_absent() {
|
||||
@@ -2090,6 +2359,7 @@ mod tests {
|
||||
#[serial]
|
||||
async fn bucket_namespace_reads_fail_closed_when_any_set_loses_quorum() {
|
||||
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
|
||||
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
|
||||
let bucket = format!("degraded-expansion-{}", Uuid::new_v4().simple());
|
||||
ecstore.pools[0].disk_set[0]
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
@@ -2097,6 +2367,7 @@ mod tests {
|
||||
.expect("bucket should be created in the original pool only");
|
||||
ecstore.pools[1].disk_set[0].disks.write().await[0] = None;
|
||||
ecstore.pools[1].disk_set[0].disks.write().await[1] = None;
|
||||
ecstore.pools[1].disk_set[0].disks.write().await[2] = None;
|
||||
|
||||
let list_err = ecstore
|
||||
.list_bucket(&BucketOptions::default())
|
||||
@@ -2108,7 +2379,7 @@ mod tests {
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect_err("bucket validation must fail when an expansion pool is unavailable");
|
||||
assert_eq!(info_err, StorageError::ErasureWriteQuorum);
|
||||
assert_eq!(info_err, StorageError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -18,7 +18,6 @@ use crate::services::rebalance::{REBAL_META_NAME, RebalStatus};
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use rustfs_heal_contracts::heal_channel::DriveState;
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use std::collections::BTreeSet;
|
||||
use tracing::trace;
|
||||
@@ -379,79 +378,6 @@ impl ECStore {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Heal every pool metadata owner in the selected scope without allowing
|
||||
/// one healthy pool to hide another pool's failed repair.
|
||||
pub async fn heal_pool_metadata(&self, opts: &HealOpts) -> Result<Vec<HealResultItem>> {
|
||||
let scopes = self.heal_erasure_set_scopes(opts).await?;
|
||||
let mut results = Vec::new();
|
||||
for (pool_index, set_index) in scopes {
|
||||
if !self.replacement_pool_metadata_applies(pool_index, set_index)? {
|
||||
continue;
|
||||
}
|
||||
let set = &self.pools[pool_index].disk_set[set_index];
|
||||
let targets = set.set_endpoints.iter().map(ToString::to_string).collect::<Vec<_>>();
|
||||
if targets.is_empty()
|
||||
|| targets.len() != set.set_drive_count
|
||||
|| targets.iter().collect::<BTreeSet<_>>().len() != targets.len()
|
||||
{
|
||||
return Err(Error::SlowDown);
|
||||
}
|
||||
// Administrative remove/no-lock options apply to user objects,
|
||||
// never to the cluster's authoritative metadata transaction.
|
||||
let metadata_opts = HealOpts {
|
||||
dry_run: opts.dry_run,
|
||||
recreate: opts.recreate,
|
||||
scan_mode: opts.scan_mode,
|
||||
pool: Some(pool_index),
|
||||
set: Some(set_index),
|
||||
..Default::default()
|
||||
};
|
||||
let (result, error) = self
|
||||
.handle_heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, "", &metadata_opts)
|
||||
.await?;
|
||||
if let Some(error) = error {
|
||||
return Err(error);
|
||||
}
|
||||
if !opts.dry_run {
|
||||
let ok_state = DriveState::Ok.to_string();
|
||||
let complete = result.after.drives.len() == targets.len()
|
||||
&& targets.iter().all(|target| {
|
||||
let mut outcomes = result.after.drives.iter().filter(|drive| drive.endpoint == *target);
|
||||
outcomes.next().is_some_and(|drive| drive.state == ok_state) && outcomes.next().is_none()
|
||||
});
|
||||
if !complete
|
||||
|| !set
|
||||
.replacement_targets_have_version(RUSTFS_META_BUCKET, POOL_META_NAME, "", &targets)
|
||||
.await?
|
||||
{
|
||||
return Err(Error::SlowDown);
|
||||
}
|
||||
}
|
||||
results.push(result);
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Whether this replacement set owns the pool's metadata replica.
|
||||
///
|
||||
/// Pool metadata follows normal object placement within each pool. A valid
|
||||
/// non-owner set has no replica to repair; missing metadata on the owner
|
||||
/// set still requires healing and target-specific readback.
|
||||
pub fn replacement_pool_metadata_applies(&self, pool_index: usize, set_index: usize) -> Result<bool> {
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
|
||||
let selected = pool.get_disks_for_heal_object(
|
||||
POOL_META_NAME,
|
||||
&HealOpts {
|
||||
set: Some(set_index),
|
||||
..Default::default()
|
||||
},
|
||||
)?;
|
||||
Ok(Arc::ptr_eq(&selected, &pool.get_disks_by_key(POOL_META_NAME)))
|
||||
}
|
||||
|
||||
#[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
|
||||
pub async fn replacement_targets_have_version(
|
||||
&self,
|
||||
@@ -903,249 +829,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_pool_metadata_applies_to_the_written_replica_in_each_pool() {
|
||||
let mut store = minimal_heal_store().await;
|
||||
for pool_index in 0..store.pools.len() {
|
||||
assert!(
|
||||
store
|
||||
.replacement_pool_metadata_applies(pool_index, 0)
|
||||
.expect("a valid single-set pool should have a metadata owner")
|
||||
);
|
||||
}
|
||||
store.ctx = Arc::new(InstanceContext::new());
|
||||
for algorithm in [
|
||||
crate::disk::format::DistributionAlgoVersion::V1,
|
||||
crate::disk::format::DistributionAlgoVersion::V2,
|
||||
crate::disk::format::DistributionAlgoVersion::V3,
|
||||
] {
|
||||
let mut temp_dirs = Vec::new();
|
||||
for pool_index in 0..store.pools.len() {
|
||||
let (dirs, mut pool) =
|
||||
crate::core::sets::make_local_two_set_sets_for_pool_with_ctx(Arc::clone(&store.ctx), pool_index).await;
|
||||
temp_dirs.extend(dirs);
|
||||
Arc::get_mut(&mut pool)
|
||||
.expect("fixture pool should have one owner")
|
||||
.distribution_algo = algorithm.clone();
|
||||
store.pools[pool_index] = pool;
|
||||
}
|
||||
for (pool_index, pool) in store.pools.iter().enumerate() {
|
||||
let mut required_sets = 0;
|
||||
for set_index in 0..pool.disk_set.len() {
|
||||
required_sets += usize::from(
|
||||
store
|
||||
.replacement_pool_metadata_applies(pool_index, set_index)
|
||||
.expect("valid replacement topology should be classified before metadata exists"),
|
||||
);
|
||||
}
|
||||
assert_eq!(required_sets, 1, "missing metadata cannot exempt the owner set");
|
||||
save_config(pool.clone(), POOL_META_NAME, b"pool metadata placement".to_vec())
|
||||
.await
|
||||
.expect("normal config writes should persist one metadata replica per pool");
|
||||
for (set_index, set) in pool.disk_set.iter().enumerate() {
|
||||
let applies = store
|
||||
.replacement_pool_metadata_applies(pool_index, set_index)
|
||||
.expect("valid replacement topology should be classified");
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().flatten() {
|
||||
let replica = disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await;
|
||||
if applies {
|
||||
replica.expect("the metadata owner must match actual persisted shards");
|
||||
} else {
|
||||
assert!(
|
||||
matches!(replica, Err(crate::disk::error::DiskError::FileNotFound)),
|
||||
"non-owner sets must have no persisted metadata shard; observed error: {:?}",
|
||||
replica.as_ref().err()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.replacement_pool_metadata_applies(pool_index, pool.disk_set.len())
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(store.replacement_pool_metadata_applies(store.pools.len(), 0).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn ordinary_pool_metadata_heal_repairs_each_owner_and_preserves_dry_run() {
|
||||
let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
|
||||
let first_missing = remove_pool_meta_shard(&store, 0).await;
|
||||
let second_missing = remove_pool_meta_shard(&store, 1).await;
|
||||
let destructive_options = HealOpts {
|
||||
remove: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let results = store
|
||||
.heal_pool_metadata(&HealOpts {
|
||||
dry_run: true,
|
||||
..destructive_options
|
||||
})
|
||||
.await
|
||||
.expect("dry-run should inspect both metadata owners without requiring a commit");
|
||||
assert_eq!(results.len(), 2);
|
||||
assert!(
|
||||
first_missing
|
||||
.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
second_missing
|
||||
.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let lock = store.pools[0]
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("metadata namespace lock should be available");
|
||||
let guard = lock
|
||||
.get_read_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("a metadata reader should hold the shared fence");
|
||||
let error = temp_env::async_with_vars(
|
||||
[(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))],
|
||||
store.heal_pool_metadata(&destructive_options),
|
||||
)
|
||||
.await
|
||||
.expect_err("administrative no-lock cannot bypass the metadata write fence");
|
||||
assert!(matches!(error, Error::Lock(rustfs_lock::LockError::Timeout { .. })));
|
||||
drop(guard);
|
||||
let results = store
|
||||
.heal_pool_metadata(&destructive_options)
|
||||
.await
|
||||
.expect("every metadata owner should be repaired");
|
||||
assert_eq!(results.len(), 2);
|
||||
assert!(first_missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok());
|
||||
assert!(
|
||||
second_missing
|
||||
.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false)
|
||||
.await
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn ordinary_pool_metadata_heal_does_not_hide_missing_later_pool() {
|
||||
let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
|
||||
delete_config(store.pools[1].clone(), POOL_META_NAME)
|
||||
.await
|
||||
.expect("the second pool metadata replica should be removed");
|
||||
|
||||
let error = store
|
||||
.heal_pool_metadata(&HealOpts::default())
|
||||
.await
|
||||
.expect_err("the healthy first pool must not hide the second owner's missing replica");
|
||||
|
||||
assert!(!matches!(error, Error::NoHealRequired));
|
||||
let second_set = store.pools[1].get_disks_by_key(POOL_META_NAME);
|
||||
for disk in second_set.disks.read().await.iter().flatten() {
|
||||
assert!(disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_pool_metadata_heal_skips_only_valid_non_owner_sets() {
|
||||
let mut store = minimal_heal_store().await;
|
||||
store.ctx = Arc::new(InstanceContext::new());
|
||||
for algorithm in [
|
||||
crate::disk::format::DistributionAlgoVersion::V1,
|
||||
crate::disk::format::DistributionAlgoVersion::V2,
|
||||
crate::disk::format::DistributionAlgoVersion::V3,
|
||||
] {
|
||||
let mut temp_dirs = Vec::new();
|
||||
for pool_index in 0..store.pools.len() {
|
||||
let (dirs, mut pool) =
|
||||
crate::core::sets::make_local_two_set_sets_for_pool_with_ctx(Arc::clone(&store.ctx), pool_index).await;
|
||||
temp_dirs.extend(dirs);
|
||||
Arc::get_mut(&mut pool)
|
||||
.expect("fixture pool should have one owner")
|
||||
.distribution_algo = algorithm.clone();
|
||||
store.pools[pool_index] = pool;
|
||||
}
|
||||
for pool_index in 0..store.pools.len() {
|
||||
let owner = (0..store.pools[pool_index].disk_set.len())
|
||||
.find(|set_index| {
|
||||
store
|
||||
.replacement_pool_metadata_applies(pool_index, *set_index)
|
||||
.expect("valid metadata placement")
|
||||
})
|
||||
.expect("every pool must have one metadata owner");
|
||||
let non_owner = 1 - owner;
|
||||
assert!(
|
||||
store
|
||||
.heal_pool_metadata(&HealOpts {
|
||||
pool: Some(pool_index),
|
||||
set: Some(non_owner),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("valid non-owner should need no metadata write")
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.heal_pool_metadata(&HealOpts {
|
||||
pool: Some(pool_index),
|
||||
set: Some(owner),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.is_err(),
|
||||
"an owner with no authoritative metadata must fail"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.heal_pool_metadata(&HealOpts {
|
||||
pool: Some(pool_index),
|
||||
set: Some(2),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.is_err(),
|
||||
"invalid sets cannot claim the non-owner exemption"
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.heal_pool_metadata(&HealOpts {
|
||||
pool: Some(2),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn ordinary_pool_metadata_heal_requires_every_owner_endpoint() {
|
||||
let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
|
||||
let owner = store.pools[0].get_disks_by_key(POOL_META_NAME);
|
||||
let offline_disk = owner.disks.write().await[0]
|
||||
.take()
|
||||
.expect("fixture owner disk should start online");
|
||||
|
||||
let result = store
|
||||
.heal_pool_metadata(&HealOpts {
|
||||
pool: Some(0),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(result.is_err(), "a surviving metadata shard must not hide an offline owner endpoint");
|
||||
owner.disks.write().await[0] = Some(offline_disk);
|
||||
}
|
||||
|
||||
async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
|
||||
let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
|
||||
let missing_disk = target_set.disks.read().await[0]
|
||||
|
||||
@@ -18932,124 +18932,6 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn tier_config_init_recovers_after_initial_reload_failure_without_another_mutation() {
|
||||
run_large_stack_async_test("tier-config-init-recovery", || async {
|
||||
use crate::services::tier::tier::{TIER_DRIVER_TEST_FACTORY, TierDriverTestFactory};
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create tier startup recovery store dir");
|
||||
let (ctx, store, _shutdown) = build_isolated_test_store(temp_dir.path(), "tier-startup-recovery", &[4]).await;
|
||||
let manager = ctx.tier_config_mgr();
|
||||
let candidate = TierConfigMgr::new();
|
||||
let tier_name = "STARTUP-RECOVERY";
|
||||
let backend = register_mock_tier(&candidate, tier_name).await;
|
||||
let backend_identity = TierConfigMgr::acquire_operation_lease(&candidate, tier_name)
|
||||
.await
|
||||
.expect("candidate backend identity should resolve")
|
||||
.backend_identity();
|
||||
let candidate_digest = {
|
||||
let candidate = candidate.read().await;
|
||||
candidate
|
||||
.save_tiering_config(store.clone())
|
||||
.await
|
||||
.expect("committed tier config should persist");
|
||||
tier_config_candidate_digest(&candidate).expect("committed candidate digest should build")
|
||||
};
|
||||
let config_path = format!("{}/{}", com::CONFIG_PREFIX, TIER_CONFIG_FILE);
|
||||
let config_bytes = com::read_config(store.clone(), &config_path)
|
||||
.await
|
||||
.expect("committed tier config bytes should load");
|
||||
let config_etag = store
|
||||
.get_object_info(RUSTFS_META_BUCKET, &config_path, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("committed tier config metadata should load")
|
||||
.etag
|
||||
.expect("committed tier config should have an ETag");
|
||||
let mutation_id = uuid::Uuid::new_v4();
|
||||
let intent = TierMutationIntent {
|
||||
mutation_id,
|
||||
revision: 2,
|
||||
kind: TierMutationIntentKind::Add,
|
||||
state: TierMutationIntentState::Committed,
|
||||
old_config_etag: None,
|
||||
committed_config_etag: Some(config_etag),
|
||||
candidate_digest,
|
||||
affected_targets: vec![TierMutationIntentTarget {
|
||||
tier_name: tier_name.to_string(),
|
||||
old_backend_identity: None,
|
||||
new_backend_identity: Some(backend_identity),
|
||||
}],
|
||||
expires_at_unix_nanos: 1,
|
||||
};
|
||||
// Persist the restart state directly: no peer Commit or runtime block
|
||||
// installation may leave a notification that masks a missing startup wakeup.
|
||||
save_tier_mutation_intent_record(store.clone(), &intent)
|
||||
.await
|
||||
.expect("committed restart intent should persist without notifying the manager");
|
||||
com::save_config(store.clone(), &config_path, vec![0])
|
||||
.await
|
||||
.expect("controlled invalid config should persist");
|
||||
|
||||
runtime_sources::TEST_TIER_CONFIG_MGR
|
||||
.scope(manager.clone(), async {
|
||||
let err = runtime_sources::init_tier_config_mgr(store.clone())
|
||||
.await
|
||||
.expect_err("initial reload must report the controlled config parse failure");
|
||||
assert!(err.to_string().contains("tierConfigInit: no data"), "unexpected startup failure: {err}");
|
||||
})
|
||||
.await;
|
||||
assert!(manager.read().await.tiers.is_empty(), "failed startup must not publish the candidate");
|
||||
let blocked = match TierConfigMgr::acquire_operation_lease(&manager, tier_name).await {
|
||||
Ok(_) => panic!("failed startup must retain the recovered committed fence"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked));
|
||||
|
||||
// Restore only the durable bytes. Recovery must come from the worker
|
||||
// started by init, after the test-only handle scope has already ended.
|
||||
com::save_config(store.clone(), &config_path, config_bytes)
|
||||
.await
|
||||
.expect("restoring committed config bytes should remove the startup failure");
|
||||
// Reload only publishes this Add into an empty manager, so it has
|
||||
// no replaced backend to construct or probe. Only the lease check
|
||||
// below needs a driver; keep its factory on the observing task.
|
||||
let driver_factory: TierDriverTestFactory = Arc::new(move |_| Ok(Box::new(backend.clone())));
|
||||
TIER_DRIVER_TEST_FACTORY
|
||||
.scope(driver_factory, async {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
match TierConfigMgr::acquire_operation_lease(&manager, tier_name).await {
|
||||
Ok(lease) => {
|
||||
drop(lease);
|
||||
match load_tier_mutation_intent_record(store.clone(), mutation_id).await {
|
||||
Err(Error::ConfigNotFound) => break,
|
||||
Ok(retained) => assert_eq!(retained, intent),
|
||||
Err(err) => panic!("committed recovery intent should remain readable: {err}"),
|
||||
}
|
||||
}
|
||||
Err(err) => assert!(
|
||||
TierConfigMgr::operation_lease_blocked_by_mutation(&err),
|
||||
"recovery must retain the fence until the tier becomes available: {err}"
|
||||
),
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("startup worker must publish the tier and clean its fence without another mutation");
|
||||
})
|
||||
.await;
|
||||
let recovered = manager.read().await;
|
||||
assert_eq!(
|
||||
tier_config_candidate_digest(&recovered).expect("recovered config digest should build"),
|
||||
candidate_digest,
|
||||
"startup recovery must publish the committed configuration"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
|
||||
@@ -34,7 +34,7 @@ use std::time::{Duration, UNIX_EPOCH};
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::{DiskStore, EcstoreError, POOL_META_NAME, RUSTFS_META_BUCKET};
|
||||
use super::{DiskStore, EcstoreError};
|
||||
|
||||
/// Outcome of classifying an error returned by [`HealStorageAPI::heal_object`].
|
||||
enum HealObjectOutcome {
|
||||
@@ -68,13 +68,6 @@ struct PageConcurrencyGuard {
|
||||
set_label: String,
|
||||
}
|
||||
|
||||
struct ErasureSetPassCounters<'a> {
|
||||
processed_objects: &'a mut u64,
|
||||
successful_objects: &'a mut u64,
|
||||
failed_objects: &'a mut u64,
|
||||
skipped_objects: &'a mut u64,
|
||||
}
|
||||
|
||||
impl PageConcurrencyGuard {
|
||||
fn new(in_flight: Arc<AtomicUsize>, set_label: String) -> Self {
|
||||
let current = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
@@ -113,7 +106,6 @@ pub struct ErasureSetHealer {
|
||||
heal_opts: HealOpts,
|
||||
source: HealRequestSource,
|
||||
target_endpoints: Arc<[String]>,
|
||||
pool_metadata_target_endpoints: Arc<[String]>,
|
||||
replacement_task_id: Option<String>,
|
||||
replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
|
||||
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
|
||||
@@ -363,7 +355,6 @@ impl ErasureSetHealer {
|
||||
heal_opts,
|
||||
source,
|
||||
target_endpoints: Vec::new().into(),
|
||||
pool_metadata_target_endpoints: Vec::new().into(),
|
||||
replacement_task_id: None,
|
||||
replacement_target_identities: None,
|
||||
mainline_pacer: None,
|
||||
@@ -387,13 +378,6 @@ impl ErasureSetHealer {
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_pool_metadata_targets(mut self, mut target_endpoints: Vec<String>) -> Self {
|
||||
target_endpoints.sort_unstable();
|
||||
target_endpoints.dedup();
|
||||
self.pool_metadata_target_endpoints = target_endpoints.into();
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_replacement_identity_fence(
|
||||
mut self,
|
||||
replacement_target_identities: Option<Vec<ReplacementTargetIdentity>>,
|
||||
@@ -850,21 +834,6 @@ impl ErasureSetHealer {
|
||||
current_object_index = 0;
|
||||
}
|
||||
|
||||
if failed_objects == 0 && skipped_objects == 0 && failed_buckets == 0 {
|
||||
self.heal_pool_metadata(
|
||||
set_disk_id,
|
||||
&mut ErasureSetPassCounters {
|
||||
processed_objects: &mut processed_objects,
|
||||
successful_objects: &mut successful_objects,
|
||||
failed_objects: &mut failed_objects,
|
||||
skipped_objects: &mut skipped_objects,
|
||||
},
|
||||
resume_manager,
|
||||
checkpoint_manager,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// 5. finalize. Only declare the set healed when nothing failed AND
|
||||
// nothing was transiently skipped — otherwise the resume/checkpoint
|
||||
// state must survive so the failed/skipped versions are retried instead
|
||||
@@ -950,251 +919,6 @@ impl ErasureSetHealer {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn heal_pool_metadata(
|
||||
&self,
|
||||
set_disk_id: &str,
|
||||
counters: &mut ErasureSetPassCounters<'_>,
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
) -> Result<()> {
|
||||
let mut metadata_opts = self.heal_opts;
|
||||
metadata_opts.remove = false;
|
||||
metadata_opts.no_lock = false;
|
||||
if self.replacement_task_id.is_none() {
|
||||
let (pool_index, set_index) = crate::heal::utils::parse_set_disk_id(set_disk_id)?;
|
||||
if metadata_opts.pool.is_some_and(|pool| pool != pool_index) || metadata_opts.set.is_some_and(|set| set != set_index)
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Pool metadata scope does not match resumed set {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
metadata_opts.pool = Some(pool_index);
|
||||
metadata_opts.set = Some(set_index);
|
||||
}
|
||||
let target_endpoints = if self.replacement_task_id.is_some() || self.pool_metadata_target_endpoints.is_empty() {
|
||||
self.target_endpoints.as_ref()
|
||||
} else {
|
||||
self.pool_metadata_target_endpoints.as_ref()
|
||||
};
|
||||
let target_scoped_recreate = !metadata_opts.dry_run && metadata_opts.recreate && !target_endpoints.is_empty();
|
||||
let ordinary_heal = self.replacement_task_id.is_none() && !target_scoped_recreate;
|
||||
if !ordinary_heal {
|
||||
if target_endpoints.is_empty() {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Replacement pool metadata heal requires target endpoints".to_string(),
|
||||
});
|
||||
}
|
||||
if !self.storage.replacement_pool_metadata_applies(&metadata_opts).await? {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
|
||||
let checkpoint_key = compose_key(&object_key, None);
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
if checkpoint.processed_objects.contains(&checkpoint_key)
|
||||
|| checkpoint.failed_objects.contains(&checkpoint_key)
|
||||
|| checkpoint.skipped_objects.contains(&checkpoint_key)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
drop(checkpoint);
|
||||
|
||||
self.verify_replacement_identity_fence("pool metadata").await?;
|
||||
resume_manager
|
||||
.set_current_item(Some(RUSTFS_META_BUCKET.to_string()), Some(POOL_META_NAME.to_string()))
|
||||
.await?;
|
||||
|
||||
let result = if ordinary_heal {
|
||||
match self.storage.heal_pool_metadata(&metadata_opts).await {
|
||||
Ok(results) if results.is_empty() => return Ok(()),
|
||||
Ok(results) => {
|
||||
let [result] = results.as_slice() else {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Pool metadata returned multiple replicas for set {set_disk_id}"),
|
||||
});
|
||||
};
|
||||
(result_object_size_u64(result), Ok(()))
|
||||
}
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(err) => match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent | HealObjectOutcome::Transient => {
|
||||
(0, Err(Error::transient_skip(format!("Pool metadata heal must be retried: {err}"))))
|
||||
}
|
||||
HealObjectOutcome::Failed => (0, Err(err)),
|
||||
},
|
||||
}
|
||||
} else {
|
||||
match self
|
||||
.storage
|
||||
.heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, None, &metadata_opts)
|
||||
.await
|
||||
{
|
||||
Ok((result, None)) if target_outcomes_complete(&result, target_endpoints) => {
|
||||
let object_size = result_object_size_u64(&result);
|
||||
match self
|
||||
.storage
|
||||
.replacement_targets_have_version(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
None,
|
||||
&metadata_opts,
|
||||
target_endpoints,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(true) => (object_size, Ok(())),
|
||||
Ok(false) => (
|
||||
object_size,
|
||||
Err(Error::transient_skip(
|
||||
"Skipped replacement pool metadata heal because target readback did not confirm the committed version",
|
||||
)),
|
||||
),
|
||||
Err(err) => (
|
||||
object_size,
|
||||
Err(Error::transient_skip(format!(
|
||||
"Skipped replacement pool metadata heal because target readback failed: {err}"
|
||||
))),
|
||||
),
|
||||
}
|
||||
}
|
||||
Ok((result, None)) => (
|
||||
result_object_size_u64(&result),
|
||||
Err(Error::transient_skip(
|
||||
"Skipped replacement pool metadata heal because a replacement target was not committed",
|
||||
)),
|
||||
),
|
||||
Ok((result, Some(err))) => {
|
||||
let object_size = result_object_size_u64(&result);
|
||||
match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent | HealObjectOutcome::Transient => (
|
||||
object_size,
|
||||
Err(Error::transient_skip(format!(
|
||||
"Skipped replacement pool metadata heal due to transient error: {err}"
|
||||
))),
|
||||
),
|
||||
HealObjectOutcome::Failed => (object_size, Err(err)),
|
||||
}
|
||||
}
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(err) => match Self::classify_heal_object_error(&err) {
|
||||
HealObjectOutcome::Absent | HealObjectOutcome::Transient => (
|
||||
0,
|
||||
Err(Error::transient_skip(format!(
|
||||
"Skipped replacement pool metadata heal due to transient error: {err}"
|
||||
))),
|
||||
),
|
||||
HealObjectOutcome::Failed => (0, Err(err)),
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
let (object_size, result) = result;
|
||||
let mut bytes_processed = self.progress.read().await.bytes_processed;
|
||||
let mut telemetry_unknown = false;
|
||||
let checkpoint_outcome = match result {
|
||||
Ok(()) => {
|
||||
telemetry_unknown |= !increment_counter(counters.successful_objects);
|
||||
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket = RUSTFS_META_BUCKET,
|
||||
object = POOL_META_NAME,
|
||||
state = "healed",
|
||||
"Pool metadata healed"
|
||||
);
|
||||
CheckpointObjectOutcome::Processed
|
||||
}
|
||||
Err(Error::TransientSkip { message }) => {
|
||||
telemetry_unknown |= !increment_counter(counters.skipped_objects);
|
||||
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
|
||||
warn!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket = RUSTFS_META_BUCKET,
|
||||
object = POOL_META_NAME,
|
||||
state = "transient_skip",
|
||||
error = %message,
|
||||
"Pool metadata heal skipped due to transient error"
|
||||
);
|
||||
CheckpointObjectOutcome::Skipped
|
||||
}
|
||||
Err(err) => {
|
||||
telemetry_unknown |= !increment_counter(counters.failed_objects);
|
||||
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
|
||||
warn!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
bucket = RUSTFS_META_BUCKET,
|
||||
object = POOL_META_NAME,
|
||||
state = "failed",
|
||||
error = %err,
|
||||
"Pool metadata heal failed"
|
||||
);
|
||||
CheckpointObjectOutcome::Failed
|
||||
}
|
||||
};
|
||||
|
||||
telemetry_unknown |= !increment_counter(counters.processed_objects);
|
||||
let (outcome_record, counter_unknown, skipped_new_versions, skipped_ilm_expired) = {
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(object_key.clone()));
|
||||
progress.update_object_progress(
|
||||
*counters.processed_objects,
|
||||
*counters.successful_objects,
|
||||
*counters.failed_objects,
|
||||
*counters.skipped_objects,
|
||||
bytes_processed,
|
||||
);
|
||||
if telemetry_unknown {
|
||||
progress.mark_unknown();
|
||||
}
|
||||
(
|
||||
CheckpointObjectOutcomeRecord {
|
||||
object: checkpoint_key,
|
||||
outcome: checkpoint_outcome,
|
||||
successful: progress.objects_healed,
|
||||
failed: progress.objects_failed,
|
||||
skipped: progress.skipped_objects,
|
||||
bytes: progress.bytes_processed,
|
||||
skipped_new_versions: progress.skipped_new_versions,
|
||||
skipped_ilm_expired: progress.skipped_ilm_expired,
|
||||
counter_unknown: progress.counter_unknown,
|
||||
},
|
||||
progress.counter_unknown,
|
||||
progress.skipped_new_versions,
|
||||
progress.skipped_ilm_expired,
|
||||
)
|
||||
};
|
||||
checkpoint_manager.record_object_outcome(outcome_record).await?;
|
||||
resume_manager
|
||||
.update_progress_with_bytes(
|
||||
*counters.processed_objects,
|
||||
*counters.successful_objects,
|
||||
*counters.failed_objects,
|
||||
*counters.skipped_objects,
|
||||
bytes_processed,
|
||||
)
|
||||
.await?;
|
||||
resume_manager
|
||||
.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired)
|
||||
.await?;
|
||||
if counter_unknown {
|
||||
resume_manager.mark_counter_unknown().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// heal single bucket with resume
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
|
||||
@@ -1580,9 +1304,7 @@ impl ErasureSetHealer {
|
||||
);
|
||||
CheckpointObjectOutcome::Processed
|
||||
}
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => {
|
||||
return Err(err);
|
||||
}
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(Error::TransientSkip { message }) => {
|
||||
telemetry_unknown |= !increment_counter(skipped_objects);
|
||||
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
|
||||
@@ -1957,8 +1679,7 @@ mod resume_loop_tests {
|
||||
use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage_api::status::BucketInfo;
|
||||
use crate::heal::{
|
||||
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, POOL_META_NAME, RUSTFS_META_BUCKET,
|
||||
new_disk,
|
||||
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk,
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource};
|
||||
@@ -2052,28 +1773,6 @@ mod resume_loop_tests {
|
||||
assert!(!target_outcomes_complete(&duplicate, &["replacement-a".to_string()]));
|
||||
}
|
||||
|
||||
fn replacement_target_ok_result(endpoint: &str, object: &str) -> HealResultItem {
|
||||
HealResultItem {
|
||||
object: object.to_string(),
|
||||
object_size: 1024,
|
||||
before: Infos {
|
||||
drives: vec![HealDriveInfo {
|
||||
endpoint: endpoint.to_string(),
|
||||
state: "missing".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
},
|
||||
after: Infos {
|
||||
drives: vec![HealDriveInfo {
|
||||
endpoint: endpoint.to_string(),
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
},
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Page {
|
||||
items: Vec<HealListItem>,
|
||||
@@ -2084,8 +1783,6 @@ mod resume_loop_tests {
|
||||
#[derive(Clone)]
|
||||
enum HealOutcome {
|
||||
Ok,
|
||||
/// The object has no metadata on any disk in the selected set.
|
||||
FileNotFound,
|
||||
/// The version vanished before heal ran (deleted mid-heal).
|
||||
VersionNotFound,
|
||||
/// A transient infrastructure condition (offline disk / unmet quorum):
|
||||
@@ -2111,10 +1808,6 @@ mod resume_loop_tests {
|
||||
/// Target-specific physical readback evidence per `compose_key`; the
|
||||
/// fake models a healthy backend unless a test explicitly revokes it.
|
||||
replacement_commit_evidence: Mutex<HashMap<String, ReplacementCommitEvidence>>,
|
||||
ordinary_pool_metadata_required: AtomicBool,
|
||||
pool_metadata_opts: Mutex<Vec<HealOpts>>,
|
||||
pool_metadata_not_applicable: AtomicBool,
|
||||
fail_pool_metadata_scope: AtomicBool,
|
||||
lifecycle_expired: Mutex<HashSet<String>>,
|
||||
/// every heal_object call recorded as (name, version_id)
|
||||
heal_calls: Mutex<Vec<(String, Option<String>)>>,
|
||||
@@ -2203,14 +1896,11 @@ mod resume_loop_tests {
|
||||
}
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
_bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
opts: &HealOpts,
|
||||
_opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
if bucket == RUSTFS_META_BUCKET && object == POOL_META_NAME {
|
||||
self.pool_metadata_opts.lock().expect("metadata options").push(*opts);
|
||||
}
|
||||
self.heal_calls
|
||||
.lock()
|
||||
.unwrap()
|
||||
@@ -2219,7 +1909,6 @@ mod resume_loop_tests {
|
||||
let outcome = self.outcomes.lock().unwrap().get(&key).cloned().unwrap_or(HealOutcome::Ok);
|
||||
match outcome {
|
||||
HealOutcome::Ok => Ok((self.results.lock().unwrap().get(&key).cloned().unwrap_or_default(), None)),
|
||||
HealOutcome::FileNotFound => Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::FileNotFound)))),
|
||||
HealOutcome::VersionNotFound => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::FileVersionNotFound))))
|
||||
}
|
||||
@@ -2233,30 +1922,6 @@ mod resume_loop_tests {
|
||||
async fn heal_format(&self, _dry: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
Ok((HealResultItem::default(), None))
|
||||
}
|
||||
async fn heal_pool_metadata(&self, opts: &HealOpts) -> Result<Vec<HealResultItem>> {
|
||||
if !self.ordinary_pool_metadata_required.load(Ordering::SeqCst) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if !self.replacement_pool_metadata_applies(opts).await? {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let (result, error) = self.heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, None, opts).await?;
|
||||
if let Some(error) = error {
|
||||
return Err(error);
|
||||
}
|
||||
Ok(vec![result])
|
||||
}
|
||||
async fn replacement_pool_metadata_applies(&self, opts: &HealOpts) -> Result<bool> {
|
||||
if self.fail_pool_metadata_scope.load(Ordering::SeqCst) {
|
||||
return Err(Error::other("injected pool metadata scope failure"));
|
||||
}
|
||||
if self.pool_metadata_not_applicable.load(Ordering::SeqCst) {
|
||||
assert_eq!(opts.pool, Some(0));
|
||||
assert_eq!(opts.set, Some(1));
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -2748,115 +2413,6 @@ mod resume_loop_tests {
|
||||
assert!(state.completed, "successful data heal must be persisted before cleanup is attempted");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_set_heals_pool_metadata_without_replacement_generation_or_targets() {
|
||||
let env = make_env().await;
|
||||
env.storage.ordinary_pool_metadata_required.store(true, Ordering::SeqCst);
|
||||
env.storage
|
||||
.set_result(POOL_META_NAME, None, replacement_target_ok_result("metadata-disk", POOL_META_NAME));
|
||||
assert!(env.healer.replacement_task_id.is_none());
|
||||
assert!(env.healer.target_endpoints.is_empty());
|
||||
|
||||
env.healer
|
||||
.execute_heal_with_resume(&[], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect("ordinary set recovery should repair metadata even without user buckets");
|
||||
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
{
|
||||
let opts = env.storage.pool_metadata_opts.lock().expect("metadata options");
|
||||
assert_eq!(opts.len(), 1);
|
||||
assert_eq!((opts[0].pool, opts[0].set), (Some(0), Some(0)));
|
||||
}
|
||||
let state = env.resume.get_state().await;
|
||||
assert!(state.completed);
|
||||
assert_eq!(state.successful_objects, 1, "metadata must enter durable completion counters");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_set_pool_metadata_respects_non_owner_and_dry_run() {
|
||||
let mut env = make_env().await;
|
||||
env.storage.ordinary_pool_metadata_required.store(true, Ordering::SeqCst);
|
||||
env.storage.pool_metadata_not_applicable.store(true, Ordering::SeqCst);
|
||||
env.healer.heal_opts.pool = Some(0);
|
||||
env.healer.heal_opts.set = Some(1);
|
||||
env.healer
|
||||
.execute_heal_with_resume(&[], "pool_0_set_1", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect("a valid non-owner set must not invent a metadata replica");
|
||||
assert!(env.storage.calls().is_empty());
|
||||
assert_eq!(env.resume.get_state().await.successful_objects, 0);
|
||||
|
||||
let mut env = make_env().await;
|
||||
env.storage.ordinary_pool_metadata_required.store(true, Ordering::SeqCst);
|
||||
env.healer.heal_opts.dry_run = true;
|
||||
env.healer.heal_opts.remove = true;
|
||||
env.healer.heal_opts.no_lock = true;
|
||||
env.storage.set_replacement_commit_evidence(POOL_META_NAME, None, false);
|
||||
env.healer
|
||||
.execute_heal_with_resume(&[], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect("ordinary dry-run metadata work must not require a replacement commit");
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
let opts = env.storage.pool_metadata_opts.lock().expect("metadata options");
|
||||
assert_eq!(opts.len(), 1);
|
||||
assert!(opts[0].dry_run);
|
||||
assert!(!opts[0].remove);
|
||||
assert!(!opts[0].no_lock);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_set_missing_pool_metadata_preserves_retry_state() {
|
||||
let env = make_env().await;
|
||||
env.storage.ordinary_pool_metadata_required.store(true, Ordering::SeqCst);
|
||||
env.storage.set_outcome(POOL_META_NAME, None, HealOutcome::FileNotFound);
|
||||
|
||||
let error = env
|
||||
.healer
|
||||
.execute_heal_with_resume(&[], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect_err("missing required pool metadata must prevent ordinary set completion");
|
||||
|
||||
assert!(matches!(error, Error::TransientSkip { .. }));
|
||||
let state = env.resume.get_state().await;
|
||||
assert!(!state.completed);
|
||||
assert_eq!(state.retry_count, 1);
|
||||
assert!(CheckpointManager::has_checkpoint(&env.healer.disk, &env.task_id).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_set_pool_metadata_timeout_keeps_control_error() {
|
||||
let env = make_env().await;
|
||||
env.storage.ordinary_pool_metadata_required.store(true, Ordering::SeqCst);
|
||||
env.storage.set_outcome(POOL_META_NAME, None, HealOutcome::Timeout);
|
||||
|
||||
let error = env
|
||||
.healer
|
||||
.execute_heal_with_resume(&[], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect_err("metadata timeout must abort the ordinary set pass");
|
||||
|
||||
assert!(matches!(error, Error::TaskTimeout));
|
||||
assert!(!env.resume.get_state().await.completed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ordinary_set_pool_metadata_rejects_mismatched_explicit_scope() {
|
||||
let mut env = make_env().await;
|
||||
env.storage.ordinary_pool_metadata_required.store(true, Ordering::SeqCst);
|
||||
env.healer.heal_opts.pool = Some(1);
|
||||
|
||||
let error = env
|
||||
.healer
|
||||
.execute_heal_with_resume(&[], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect_err("explicit metadata scope must agree with the resumed set");
|
||||
|
||||
assert!(matches!(error, Error::TaskExecutionFailed { .. }));
|
||||
assert!(env.storage.calls().is_empty(), "scope mismatch must fail before metadata mutation");
|
||||
assert!(!env.resume.get_state().await.completed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_completion_keeps_resume_artifacts_until_marker_cleanup() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
@@ -2888,8 +2444,6 @@ mod resume_loop_tests {
|
||||
HealRequestSource::AutoHeal,
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
||||
env.storage
|
||||
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||
|
||||
healer
|
||||
.heal_erasure_set(&["b".to_string()], "pool_0_set_0")
|
||||
@@ -2907,263 +2461,9 @@ mod resume_loop_tests {
|
||||
CheckpointManager::has_checkpoint(&env.healer.disk, &replacement_task_id).await,
|
||||
"the checkpoint must survive until the caller clears the healing marker"
|
||||
);
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
drop(checkpoint);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_pool_metadata_non_owner_completes_but_missing_owner_retries() {
|
||||
for owns_pool_metadata in [false, true] {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let replacement_task_id = ResumeUtils::generate_task_id();
|
||||
let set_index = usize::from(!owns_pool_metadata);
|
||||
let set_disk_id = format!("pool_0_set_{set_index}");
|
||||
ResumeManager::new_replacement_intent(
|
||||
env.healer.disk.clone(),
|
||||
replacement_task_id.clone(),
|
||||
set_disk_id.clone(),
|
||||
vec!["b".to_string()],
|
||||
vec!["replacement-a".to_string()],
|
||||
vec![crate::heal::resume::ReplacementTargetIdentity {
|
||||
endpoint: "replacement-a".to_string(),
|
||||
canonical_path: "/mnt/replacement-a".to_string(),
|
||||
physical_device_ids: vec!["device-a".to_string()],
|
||||
filesystem_identity: "1:2:3".to_string(),
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.expect("replacement intent should persist");
|
||||
env.storage
|
||||
.pool_metadata_not_applicable
|
||||
.store(!owns_pool_metadata, Ordering::SeqCst);
|
||||
env.storage.set_outcome(POOL_META_NAME, None, HealOutcome::FileNotFound);
|
||||
let healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
CancellationToken::new(),
|
||||
env.healer.disk.clone(),
|
||||
HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(set_index),
|
||||
..Default::default()
|
||||
},
|
||||
HealRequestSource::AutoHeal,
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
||||
|
||||
let result = healer.heal_erasure_set(&["b".to_string()], &set_disk_id).await;
|
||||
let state = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id)
|
||||
.await
|
||||
.expect("replacement state must remain until marker cleanup")
|
||||
.get_state()
|
||||
.await;
|
||||
if owns_pool_metadata {
|
||||
let error = result.expect_err("missing metadata in the owner set must keep replacement incomplete");
|
||||
assert!(error.to_string().contains("Replacement erasure set heal incomplete"));
|
||||
assert!(!state.completed);
|
||||
assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Intent);
|
||||
assert_eq!(state.retry_count, 1);
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
} else {
|
||||
result.expect("a non-owner set must complete without a pool metadata replica");
|
||||
assert!(state.completed);
|
||||
assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Verified);
|
||||
assert_eq!(state.retry_count, 0);
|
||||
assert!(env.storage.calls().is_empty(), "non-owner sets must not attempt pool metadata repair");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_pool_metadata_unknown_scope_cannot_complete() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
CancellationToken::new(),
|
||||
env.healer.disk.clone(),
|
||||
HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
HealRequestSource::AutoHeal,
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some("generation-a".to_string()));
|
||||
env.storage.fail_pool_metadata_scope.store(true, Ordering::SeqCst);
|
||||
env.storage
|
||||
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||
let mut processed_objects = 0;
|
||||
let mut successful_objects = 0;
|
||||
let mut failed_objects = 0;
|
||||
let mut skipped_objects = 0;
|
||||
let error = healer
|
||||
.heal_pool_metadata(
|
||||
"pool_0_set_0",
|
||||
&mut super::ErasureSetPassCounters {
|
||||
processed_objects: &mut processed_objects,
|
||||
successful_objects: &mut successful_objects,
|
||||
failed_objects: &mut failed_objects,
|
||||
skipped_objects: &mut skipped_objects,
|
||||
},
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
)
|
||||
.await
|
||||
.expect_err("unknown metadata placement must keep replacement incomplete");
|
||||
assert!(error.to_string().contains("injected pool metadata scope failure"));
|
||||
assert!(env.storage.calls().is_empty());
|
||||
assert_eq!((processed_objects, successful_objects, failed_objects, skipped_objects), (0, 0, 0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_pool_metadata_readback_failure_schedules_retry() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let replacement_task_id = ResumeUtils::generate_task_id();
|
||||
ResumeManager::new_replacement_intent(
|
||||
env.healer.disk.clone(),
|
||||
replacement_task_id.clone(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["b".to_string()],
|
||||
vec!["replacement-a".to_string()],
|
||||
vec![crate::heal::resume::ReplacementTargetIdentity {
|
||||
endpoint: "replacement-a".to_string(),
|
||||
canonical_path: "/mnt/replacement-a".to_string(),
|
||||
physical_device_ids: vec!["device-a".to_string()],
|
||||
filesystem_identity: "1:2:3".to_string(),
|
||||
}],
|
||||
)
|
||||
.await
|
||||
.expect("replacement intent should persist");
|
||||
env.storage
|
||||
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||
env.storage.set_replacement_commit_evidence(POOL_META_NAME, None, false);
|
||||
let healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
CancellationToken::new(),
|
||||
env.healer.disk.clone(),
|
||||
HealOpts::default(),
|
||||
HealRequestSource::AutoHeal,
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some(replacement_task_id.clone()));
|
||||
|
||||
let error = healer
|
||||
.heal_erasure_set(&["b".to_string()], "pool_0_set_0")
|
||||
.await
|
||||
.expect_err("unconfirmed pool metadata readback must keep the replacement incomplete");
|
||||
|
||||
assert!(error.to_string().contains("Replacement erasure set heal incomplete"));
|
||||
let state = ResumeManager::load_replacement_intent(env.healer.disk.clone(), &replacement_task_id)
|
||||
.await
|
||||
.expect("replacement retry state must remain")
|
||||
.get_state()
|
||||
.await;
|
||||
assert!(!state.completed);
|
||||
assert_eq!(state.replacement_phase, crate::heal::resume::ReplacementPhase::Intent);
|
||||
assert_eq!(state.retry_count, 1);
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_recreate_target_heals_pool_metadata_before_completion() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
CancellationToken::new(),
|
||||
env.healer.disk.clone(),
|
||||
HealOpts {
|
||||
recreate: true,
|
||||
remove: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
HealRequestSource::Admin,
|
||||
)
|
||||
.with_pool_metadata_targets(vec!["replacement-a".to_string()]);
|
||||
env.storage
|
||||
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||
|
||||
healer
|
||||
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect("admin recreate should heal and verify pool metadata");
|
||||
|
||||
assert!(env.resume.get_state().await.completed);
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
let opts = env.storage.pool_metadata_opts.lock().expect("metadata options");
|
||||
assert_eq!(opts.len(), 1);
|
||||
assert_eq!((opts[0].pool, opts[0].set), (Some(0), Some(0)));
|
||||
assert!(opts[0].recreate);
|
||||
assert!(!opts[0].remove);
|
||||
assert!(!opts[0].no_lock);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_recreate_pool_metadata_validates_owner_scope_before_io() {
|
||||
for (non_owner, unknown_scope, pool) in [(true, false, None), (false, true, None), (false, false, Some(1))] {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
env.storage.pool_metadata_not_applicable.store(non_owner, Ordering::SeqCst);
|
||||
env.storage.fail_pool_metadata_scope.store(unknown_scope, Ordering::SeqCst);
|
||||
let healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
CancellationToken::new(),
|
||||
env.healer.disk.clone(),
|
||||
HealOpts {
|
||||
recreate: true,
|
||||
pool,
|
||||
..Default::default()
|
||||
},
|
||||
HealRequestSource::Admin,
|
||||
)
|
||||
.with_pool_metadata_targets(vec!["replacement-a".to_string()]);
|
||||
|
||||
let set_disk_id = if non_owner { "pool_0_set_1" } else { "pool_0_set_0" };
|
||||
let result = healer
|
||||
.execute_heal_with_resume(&[], set_disk_id, &env.resume, &env.checkpoint)
|
||||
.await;
|
||||
|
||||
assert_eq!(result.is_ok(), non_owner, "only a known non-owner may skip metadata: {result:?}");
|
||||
assert!(env.storage.calls().is_empty(), "scope validation must precede metadata I/O");
|
||||
assert_eq!(env.resume.get_state().await.completed, non_owner);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_recreate_pool_metadata_readback_failure_keeps_resume_state() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
CancellationToken::new(),
|
||||
env.healer.disk.clone(),
|
||||
HealOpts {
|
||||
recreate: true,
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
HealRequestSource::Admin,
|
||||
)
|
||||
.with_pool_metadata_targets(vec!["replacement-a".to_string()]);
|
||||
env.storage
|
||||
.set_result(POOL_META_NAME, None, replacement_target_ok_result("replacement-a", POOL_META_NAME));
|
||||
env.storage.set_replacement_commit_evidence(POOL_META_NAME, None, false);
|
||||
|
||||
let error = healer
|
||||
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect_err("unconfirmed admin recreate pool metadata must keep the set incomplete");
|
||||
|
||||
assert!(error.to_string().contains("Erasure set heal incomplete"));
|
||||
let state = env.resume.get_state().await;
|
||||
assert!(!state.completed);
|
||||
assert_eq!(state.retry_count, 1);
|
||||
assert_eq!(env.storage.calls(), vec![(POOL_META_NAME.to_string(), None)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retry_exhaustion_keeps_resume_artifacts_for_recovery() {
|
||||
let env = make_env().await;
|
||||
|
||||
@@ -506,25 +506,6 @@ fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key:
|
||||
.map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
|
||||
}
|
||||
|
||||
fn request_matches_task(request: &HealRequest, task: &HealTask) -> bool {
|
||||
request.heal_type == task.heal_type
|
||||
&& request.options == task.options
|
||||
&& request.priority == task.priority
|
||||
&& request.source == task.source
|
||||
&& request.retry_attempts == task.retry_attempts
|
||||
&& request.heal_endpoints == task.heal_endpoints
|
||||
}
|
||||
|
||||
fn request_matches_request(request: &HealRequest, existing: &HealRequest) -> bool {
|
||||
request.heal_type == existing.heal_type
|
||||
&& request.options == existing.options
|
||||
&& request.priority == existing.priority
|
||||
&& request.source == existing.source
|
||||
&& request.force_start == existing.force_start
|
||||
&& request.retry_attempts == existing.retry_attempts
|
||||
&& request.heal_endpoints == existing.heal_endpoints
|
||||
}
|
||||
|
||||
fn retrying_heal_for_dedup_key(retrying_heals: &HashMap<String, RetryingHeal>, key: &str) -> Option<(String, HealType)> {
|
||||
retrying_heals
|
||||
.iter()
|
||||
@@ -1527,7 +1508,7 @@ impl HealManager {
|
||||
request: HealRequest,
|
||||
preserve_alias: bool,
|
||||
) -> Result<HealAdmissionReceipt> {
|
||||
self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, preserve_alias, true, None)
|
||||
self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, preserve_alias, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1563,27 +1544,14 @@ impl HealManager {
|
||||
request: HealRequest,
|
||||
mrf_notice_target: MrfRepairNoticeTarget,
|
||||
) -> Result<HealAdmissionReceipt> {
|
||||
self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, true, true, Some(mrf_notice_target))
|
||||
self.submit_heal_request_with_receipt_alias_and_mrf_notice(request, true, Some(mrf_notice_target))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn durable_mrf_repair_anchor(
|
||||
&self,
|
||||
intent: &rustfs_common::mrf_channel::MrfIntent,
|
||||
) -> Option<rustfs_common::mrf_channel::MrfDurableRepairAnchor> {
|
||||
match self.storage.mrf_bucket_incarnation_id(intent.bucket.as_ref()).await {
|
||||
Ok(Some(bucket_incarnation_id)) => {
|
||||
rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(intent, bucket_incarnation_id)
|
||||
}
|
||||
Ok(None) | Err(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn submit_heal_request_with_receipt_alias_and_mrf_notice(
|
||||
&self,
|
||||
request: HealRequest,
|
||||
preserve_alias: bool,
|
||||
accept_same_request_id_replay: bool,
|
||||
mrf_notice_target: Option<MrfRepairNoticeTarget>,
|
||||
) -> Result<HealAdmissionReceipt> {
|
||||
let admission_start = Instant::now();
|
||||
@@ -1645,68 +1613,6 @@ impl HealManager {
|
||||
pause_duplicate_admission_after_active_lock(&request.id).await;
|
||||
let mut queue = self.heal_queue.lock().await;
|
||||
let retrying_heals = self.retrying_heals.lock().await;
|
||||
|
||||
let request_id_admission = active_heals
|
||||
.get(&request.id)
|
||||
.map(|task| (request_matches_task(&request, task), "active"))
|
||||
.or_else(|| {
|
||||
queue
|
||||
.requests()
|
||||
.find(|queued| queued.id == request.id)
|
||||
.map(|queued| (request_matches_request(&request, queued), "queued"))
|
||||
})
|
||||
.or_else(|| {
|
||||
retrying_heals
|
||||
.get(&request.id)
|
||||
.map(|retrying| (request_matches_request(&request, &retrying.request), "retrying"))
|
||||
});
|
||||
if let Some((matches_existing, duplicate_state)) = request_id_admission {
|
||||
let admission = if matches_existing {
|
||||
if accept_same_request_id_replay {
|
||||
HealAdmissionResult::Accepted
|
||||
} else {
|
||||
Self::duplicate_admission_for_request(&request, &config)
|
||||
}
|
||||
} else {
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
};
|
||||
if matches!(admission, HealAdmissionResult::Accepted | HealAdmissionResult::Merged)
|
||||
&& let Some(target) = mrf_notice_target
|
||||
{
|
||||
let mut targets = lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets);
|
||||
Self::insert_mrf_repair_notice_target(&mut targets, &request.id, target);
|
||||
}
|
||||
drop(retrying_heals);
|
||||
drop(queue);
|
||||
drop(active_heals);
|
||||
let lock_phase = lock_phase_start.elapsed();
|
||||
Self::record_admission_metric(request.source, admission, "duplicate");
|
||||
self.record_admission_observation(HealAdmissionObservation {
|
||||
source,
|
||||
result: admission,
|
||||
context: "duplicate",
|
||||
force_start,
|
||||
displaced: false,
|
||||
start_duration: admission_start.elapsed(),
|
||||
lock_phase,
|
||||
});
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
duplicate_state,
|
||||
result = admission.result_label(),
|
||||
reason = admission.reason_label(),
|
||||
"Heal queue admission reused an existing request id"
|
||||
);
|
||||
return Ok(HealAdmissionReceipt {
|
||||
result: admission,
|
||||
task_id: request.id,
|
||||
});
|
||||
}
|
||||
|
||||
let duplicate = (!request.force_start).then(|| {
|
||||
active_heal_for_dedup_key(&active_heals, &dedup_key)
|
||||
.map(|(task_id, _)| (task_id, "active"))
|
||||
@@ -1913,10 +1819,7 @@ impl HealManager {
|
||||
|
||||
/// Submit heal request.
|
||||
pub async fn submit_heal_request(&self, request: HealRequest) -> Result<HealAdmissionResult> {
|
||||
Ok(self
|
||||
.submit_heal_request_with_receipt_alias_and_mrf_notice(request, true, false, None)
|
||||
.await?
|
||||
.result)
|
||||
Ok(self.submit_heal_request_with_receipt_and_alias(request, true).await?.result)
|
||||
}
|
||||
|
||||
/// Get task status
|
||||
|
||||
@@ -733,30 +733,23 @@ pub(super) fn mrf_verified_repair_event_for_target(
|
||||
HealObjectDisposition::AuthoritativelyAbsent => MrfVerifiedRepairDisposition::AuthoritativelyAbsent,
|
||||
_ => return None,
|
||||
};
|
||||
let expected_kind = match target.kind {
|
||||
MrfKind::DecodeFailure => HealObjectKind::Decode,
|
||||
MrfKind::MetadataCorruption => HealObjectKind::Metadata,
|
||||
MrfKind::PartialWrite => HealObjectKind::Object,
|
||||
};
|
||||
if target.kind != MrfKind::PartialWrite {
|
||||
return None;
|
||||
}
|
||||
let expected_kind = HealObjectKind::Object;
|
||||
if outcome.identity.kind != expected_kind
|
||||
|| outcome.identity.bucket.as_str() != target.bucket.as_ref()
|
||||
|| outcome.identity.object.as_str() != target.object.as_ref()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let version_id = (!matches!(target.kind, MrfKind::MetadataCorruption))
|
||||
.then_some(target.version_id)
|
||||
.flatten()
|
||||
.filter(|bytes| *bytes != [0; 16]);
|
||||
let version_id = target.version_id.filter(|bytes| *bytes != [0; 16]);
|
||||
let expected_version = version_id.map(|bytes| uuid::Uuid::from_bytes(bytes).to_string());
|
||||
if outcome.identity.version_id != expected_version {
|
||||
return None;
|
||||
}
|
||||
let scope = (!matches!(target.kind, MrfKind::MetadataCorruption))
|
||||
.then_some(target.scope)
|
||||
.flatten();
|
||||
let expected_pool = scope.and_then(|scope| usize::try_from(scope.pool_index).ok());
|
||||
let expected_set = scope.and_then(|scope| usize::try_from(scope.set_index).ok());
|
||||
let expected_pool = target.scope.and_then(|scope| usize::try_from(scope.pool_index).ok());
|
||||
let expected_set = target.scope.and_then(|scope| usize::try_from(scope.set_index).ok());
|
||||
if outcome.identity.pool_index != expected_pool || outcome.identity.set_index != expected_set {
|
||||
return None;
|
||||
}
|
||||
@@ -766,7 +759,7 @@ pub(super) fn mrf_verified_repair_event_for_target(
|
||||
bucket: target.bucket.clone(),
|
||||
object: target.object.clone(),
|
||||
version_id,
|
||||
scope,
|
||||
scope: target.scope,
|
||||
lease: target.lease,
|
||||
bucket_incarnation_id,
|
||||
disposition,
|
||||
|
||||
@@ -555,10 +555,6 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl HealStorageAPI for MockStorage {
|
||||
async fn heal_pool_metadata(&self, _opts: &HealOpts) -> Result<Vec<HealResultItem>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
@@ -1146,48 +1142,18 @@ fn mrf_verified_repair_event_requires_positive_exact_identity() {
|
||||
assert_eq!(event.bucket_incarnation_id, incarnation);
|
||||
assert_eq!(event.disposition, MrfVerifiedRepairDisposition::Repaired);
|
||||
|
||||
let decode_target = MrfRepairNoticeTarget {
|
||||
kind: MrfKind::DecodeFailure,
|
||||
..target.clone()
|
||||
};
|
||||
let decode_outcome = HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Decode,
|
||||
..matching.identity.clone()
|
||||
},
|
||||
..matching.clone()
|
||||
};
|
||||
let decode_event =
|
||||
mrf_verified_repair_event_for_target(&decode_target, &decode_outcome).expect("decode repairs publish exact proofs");
|
||||
assert_eq!(decode_event.kind, MrfKind::DecodeFailure);
|
||||
assert_eq!(
|
||||
decode_event.scope,
|
||||
Some(MrfScope {
|
||||
pool_index: 1,
|
||||
set_index: 2
|
||||
})
|
||||
assert!(
|
||||
mrf_verified_repair_event_for_target(
|
||||
&MrfRepairNoticeTarget {
|
||||
kind: MrfKind::DecodeFailure,
|
||||
..target.clone()
|
||||
},
|
||||
&matching
|
||||
)
|
||||
.is_none(),
|
||||
"only receipt-producing partial-write object heals can publish verified events today"
|
||||
);
|
||||
|
||||
let metadata_target = MrfRepairNoticeTarget {
|
||||
kind: MrfKind::MetadataCorruption,
|
||||
..target.clone()
|
||||
};
|
||||
let metadata_outcome = HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Metadata,
|
||||
version_id: None,
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
..matching.identity.clone()
|
||||
},
|
||||
..matching.clone()
|
||||
};
|
||||
let metadata_event =
|
||||
mrf_verified_repair_event_for_target(&metadata_target, &metadata_outcome).expect("metadata repairs publish exact proofs");
|
||||
assert_eq!(metadata_event.kind, MrfKind::MetadataCorruption);
|
||||
assert_eq!(metadata_event.version_id, None);
|
||||
assert_eq!(metadata_event.scope, None);
|
||||
|
||||
for rejected in [
|
||||
HealObjectOutcome {
|
||||
disposition: HealObjectDisposition::Unknown,
|
||||
@@ -1873,7 +1839,7 @@ async fn test_submit_heal_request_returns_merged_for_duplicate() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let mut request = HealRequest::new(
|
||||
let request = HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
@@ -1890,7 +1856,6 @@ async fn test_submit_heal_request_returns_merged_for_duplicate() {
|
||||
.expect("first request should be accepted"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
request.id = uuid::Uuid::new_v4().to_string();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
@@ -3730,7 +3695,7 @@ async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
|
||||
}),
|
||||
);
|
||||
|
||||
let mut request = HealRequest::new(
|
||||
let request = HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
@@ -3747,7 +3712,6 @@ async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
|
||||
.expect("first request should be accepted"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
request.id = uuid::Uuid::new_v4().to_string();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
@@ -4573,48 +4537,6 @@ async fn test_force_start_marks_dedup_key_for_future_duplicates() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn same_request_id_replay_reuses_existing_task_without_force_start_duplication() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let mut original = admin_prefix_request("bucket", "logs/");
|
||||
original.force_start = true;
|
||||
let original_id = original.id.clone();
|
||||
let accepted = manager
|
||||
.submit_heal_request_with_receipt(original.clone())
|
||||
.await
|
||||
.expect("original forceStart request should queue");
|
||||
assert_eq!(accepted.result, HealAdmissionResult::Accepted);
|
||||
assert_eq!(accepted.task_id, original_id);
|
||||
|
||||
let replayed = manager
|
||||
.submit_heal_request_with_receipt(original.clone())
|
||||
.await
|
||||
.expect("same request id and payload should reuse the existing task");
|
||||
assert_eq!(replayed.result, HealAdmissionResult::Accepted);
|
||||
assert_eq!(replayed.task_id, original_id);
|
||||
assert_eq!(
|
||||
manager.get_queue_length().await,
|
||||
1,
|
||||
"exact forceStart replay must not create a second queued task"
|
||||
);
|
||||
|
||||
let mut changed = original;
|
||||
changed.options.remove_corrupted = true;
|
||||
let changed = manager
|
||||
.submit_heal_request_with_receipt(changed)
|
||||
.await
|
||||
.expect("same request id with a changed payload should fail closed");
|
||||
assert_eq!(changed.result, HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning));
|
||||
assert_eq!(changed.task_id, original_id);
|
||||
assert_eq!(
|
||||
manager.get_queue_length().await,
|
||||
1,
|
||||
"same-id conflict must not displace or duplicate the original task"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_running_heal_set_counts_groups_set_scoped_tasks() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
|
||||
@@ -27,10 +27,10 @@ pub mod task;
|
||||
pub mod utils;
|
||||
|
||||
use storage_api::owner::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME,
|
||||
ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes,
|
||||
EcstoreDiskError, EcstoreDiskOption, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType,
|
||||
EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations, ecstore_local_disk_map_read, ecstore_new_disk,
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskOption,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO,
|
||||
ObjectOperations, ecstore_local_disk_map_read, ecstore_new_disk,
|
||||
};
|
||||
|
||||
pub use erasure_healer::ErasureSetHealer;
|
||||
@@ -41,7 +41,6 @@ pub use task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};
|
||||
pub(crate) const DATA_USAGE_CACHE_NAME: &str = ECSTORE_DATA_USAGE_CACHE_NAME;
|
||||
pub(crate) const BUCKET_META_PREFIX: &str = ECSTORE_BUCKET_META_PREFIX;
|
||||
pub(crate) const RUSTFS_META_BUCKET: &str = ECSTORE_RUSTFS_META_BUCKET;
|
||||
pub(crate) const POOL_META_NAME: &str = ECSTORE_POOL_META_NAME;
|
||||
|
||||
/// Marker written to every local disk while the process runs; removed by
|
||||
/// [`clear_unclean_shutdown_markers`] on graceful shutdown. Finding it at
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -110,8 +110,7 @@ impl HealObjectReceipt {
|
||||
&& self.identity.version_id == expected.version_id
|
||||
&& self.identity.pool_index == expected.pool_index
|
||||
&& self.identity.set_index == expected.set_index
|
||||
&& self.identity.bucket_incarnation_id == expected.bucket_incarnation_id
|
||||
&& expected.bucket_incarnation_id.is_some()
|
||||
&& self.identity.bucket_incarnation_id.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -497,31 +496,21 @@ mod canonical_outcome_tests {
|
||||
|
||||
#[test]
|
||||
fn positive_receipt_requires_exact_identity_and_bucket_incarnation() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let expected = HealObjectIdentity {
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
..item(HealObjectDisposition::Unknown).identity
|
||||
};
|
||||
let expected = item(HealObjectDisposition::Unknown).identity;
|
||||
let mut receipt = HealObjectReceipt {
|
||||
identity: expected.clone(),
|
||||
disposition: HealObjectDisposition::Repaired,
|
||||
};
|
||||
|
||||
receipt.identity.bucket_incarnation_id = None;
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
"a positive storage receipt without bucket incarnation must remain untrusted"
|
||||
);
|
||||
|
||||
let incarnation = Uuid::new_v4();
|
||||
receipt.identity.bucket_incarnation_id = Some(incarnation);
|
||||
assert!(receipt.verified_for(&expected));
|
||||
|
||||
receipt.identity.bucket_incarnation_id = Some(Uuid::new_v4());
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
"a storage receipt for a different bucket incarnation must not clear the requested responsibility"
|
||||
);
|
||||
|
||||
receipt.identity.version_id = Some("older-version".to_string());
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
|
||||
@@ -20,7 +20,6 @@ use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::outcome::{HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectReceipt};
|
||||
use super::progress::stable_generation;
|
||||
@@ -356,14 +355,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
/// Get bucket info
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>>;
|
||||
|
||||
/// Return the current bucket incarnation for exact MRF durable proof
|
||||
/// matching. Alternate backends that cannot expose this must return
|
||||
/// `None`, leaving replay anchors retained instead of acknowledged with an
|
||||
/// incomplete identity.
|
||||
async fn mrf_bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Aggregate usage-cache baselines for the requested buckets.
|
||||
async fn erasure_set_usage_baseline(&self, _buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
Ok(None)
|
||||
@@ -392,11 +383,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
/// Check object exists
|
||||
async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool>;
|
||||
|
||||
/// Stable bucket incarnation observed before an object heal starts.
|
||||
async fn bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Heal object using ecstore
|
||||
async fn heal_object(
|
||||
&self,
|
||||
@@ -436,23 +422,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Err(Error::other("target-scoped replacement format is unsupported"))
|
||||
}
|
||||
|
||||
/// Heal each pool metadata replica owned by the selected live scope.
|
||||
///
|
||||
/// A successful result requires every applicable owner to finish; an empty
|
||||
/// result is valid only for a known scope with no metadata replica. Backends
|
||||
/// without pool metadata must explicitly implement that empty result.
|
||||
async fn heal_pool_metadata(&self, _opts: &HealOpts) -> Result<Vec<HealResultItem>> {
|
||||
Err(Error::other("pool metadata healing is unsupported"))
|
||||
}
|
||||
|
||||
/// Whether the selected replacement set owns the pool metadata replica.
|
||||
///
|
||||
/// Only a topology-aware backend may exempt a valid non-owner set. The
|
||||
/// conservative default requires the existing repair and readback checks.
|
||||
async fn replacement_pool_metadata_applies(&self, _opts: &HealOpts) -> Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Read target-specific physical evidence for one replacement version.
|
||||
///
|
||||
/// This is only used by automatic replacement healing after the normal
|
||||
@@ -840,14 +809,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn mrf_bucket_incarnation_id(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||
self.ecstore
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<Option<HealBucketUsageBaseline>> {
|
||||
if buckets.is_empty() {
|
||||
return Ok(None);
|
||||
@@ -1067,14 +1028,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Option<Uuid>> {
|
||||
self.ecstore
|
||||
.bucket_incarnation_id(bucket)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1285,22 +1238,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn heal_pool_metadata(&self, opts: &HealOpts) -> Result<Vec<HealResultItem>> {
|
||||
self.ecstore.heal_pool_metadata(opts).await.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn replacement_pool_metadata_applies(&self, opts: &HealOpts) -> Result<bool> {
|
||||
let pool_index = opts
|
||||
.pool
|
||||
.ok_or_else(|| Error::other("replacement pool metadata is missing pool scope"))?;
|
||||
let set_index = opts
|
||||
.set
|
||||
.ok_or_else(|| Error::other("replacement pool metadata is missing set scope"))?;
|
||||
self.ecstore
|
||||
.replacement_pool_metadata_applies(pool_index, set_index)
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn replacement_targets_have_version(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -29,7 +29,6 @@ pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageEr
|
||||
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext,
|
||||
POOL_META_NAME as ECSTORE_POOL_META_NAME,
|
||||
};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
@@ -37,11 +36,10 @@ pub(crate) mod owner {
|
||||
pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations};
|
||||
|
||||
pub(crate) use super::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME,
|
||||
ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes,
|
||||
EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType,
|
||||
EcstoreHealLifecycleExpiryContext, EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached,
|
||||
ecstore_local_disk_map_read,
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError,
|
||||
EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType, EcstoreHealLifecycleExpiryContext,
|
||||
EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
|
||||
};
|
||||
|
||||
pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk};
|
||||
|
||||
@@ -106,7 +106,7 @@ const EVENT_HEAL_ERASURE_SET_STAGE: &str = "heal_erasure_set_stage";
|
||||
const EVENT_HEAL_ERASURE_SET_RESULT: &str = "heal_erasure_set_result";
|
||||
|
||||
/// Heal type
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum HealType {
|
||||
/// Cluster heal
|
||||
Cluster,
|
||||
@@ -209,7 +209,7 @@ impl HealPriority {
|
||||
}
|
||||
|
||||
/// Heal options
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HealOptions {
|
||||
/// Scan mode
|
||||
pub scan_mode: HealScanMode,
|
||||
@@ -574,18 +574,6 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn outcome_bucket_incarnation_id(&self, bucket: &str, dry_run: bool) -> Result<Option<Uuid>> {
|
||||
if dry_run {
|
||||
return Ok(None);
|
||||
}
|
||||
match self.await_with_control(self.storage.bucket_incarnation_id(bucket)).await {
|
||||
Ok(incarnation_id) => Ok(incarnation_id),
|
||||
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||
Err(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn single_object_identity(&self) -> Option<HealObjectIdentity> {
|
||||
let (bucket, object, version) = match &self.heal_type {
|
||||
HealType::Object {
|
||||
@@ -609,9 +597,6 @@ impl HealTask {
|
||||
expected: HealObjectIdentity,
|
||||
receipt: Option<HealObjectReceipt>,
|
||||
) -> bool {
|
||||
if self.options.dry_run || self.cancel_token.is_cancelled() {
|
||||
return false;
|
||||
}
|
||||
let Some(receipt) = receipt else {
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -328,23 +328,6 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
if !self.options.dry_run {
|
||||
let metadata_opts = HealOpts {
|
||||
dry_run: self.options.dry_run,
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: self.options.scan_mode,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
..Default::default()
|
||||
};
|
||||
for result in self
|
||||
.await_with_control(self.storage.heal_pool_metadata(&metadata_opts))
|
||||
.await?
|
||||
{
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
let failure = BatchHealFailure {
|
||||
scope: "cluster".to_string(),
|
||||
@@ -420,7 +403,6 @@ impl HealTask {
|
||||
};
|
||||
|
||||
for (set_disk_id, heal_opts) in listing_scopes {
|
||||
let bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, heal_opts.dry_run).await?;
|
||||
let mut continuation_token: Option<String> = None;
|
||||
let mut deferred = DeferredWindow::default();
|
||||
let mut inline_retry: Option<DeferredObject> = None;
|
||||
@@ -519,15 +501,13 @@ impl HealTask {
|
||||
let retry_attempt = item.attempt;
|
||||
let mut telemetry_unknown = false;
|
||||
let object = item.name.as_str();
|
||||
let mut identity =
|
||||
let identity =
|
||||
self.outcome_identity(bucket, object, item.version_id.as_deref(), heal_opts.pool, heal_opts.set);
|
||||
identity.bucket_incarnation_id = bucket_incarnation_id;
|
||||
let mut disposition = if heal_opts.dry_run {
|
||||
HealObjectDisposition::DryRunObserved
|
||||
} else {
|
||||
HealObjectDisposition::Unknown
|
||||
};
|
||||
let mut recorded_authoritative_outcome = false;
|
||||
let mut detail = None;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
@@ -540,37 +520,23 @@ impl HealTask {
|
||||
Some(Error::other("heal object retry age exhausted"))
|
||||
} else {
|
||||
match self
|
||||
.await_with_control(self.storage.heal_object_with_receipt(
|
||||
bucket,
|
||||
object,
|
||||
item.version_id.as_deref(),
|
||||
&heal_opts,
|
||||
))
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(storage_result) if storage_result.error.is_none() => {
|
||||
match unavailable_recreate_error(&storage_result.item, &heal_opts) {
|
||||
Some(error) => Some(error),
|
||||
None => {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
telemetry_unknown |= !add_bytes(
|
||||
&mut bytes,
|
||||
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
|
||||
);
|
||||
recorded_authoritative_outcome = self
|
||||
.record_verified_storage_receipt(identity.clone(), storage_result.receipt)
|
||||
.await;
|
||||
self.record_result_item(storage_result.item).await;
|
||||
None
|
||||
}
|
||||
Ok((result, None)) => match unavailable_recreate_error(&result, &heal_opts) {
|
||||
Some(error) => Some(error),
|
||||
None => {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
telemetry_unknown |=
|
||||
!add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX));
|
||||
self.record_result_item(result).await;
|
||||
None
|
||||
}
|
||||
}
|
||||
Ok(storage_result)
|
||||
if storage_result
|
||||
.error
|
||||
.as_ref()
|
||||
.is_some_and(|err| is_missing_object_dir_heal_result(object, err)) =>
|
||||
{
|
||||
},
|
||||
Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
@@ -585,8 +551,7 @@ impl HealTask {
|
||||
);
|
||||
None
|
||||
}
|
||||
Ok(storage_result) => storage_result.error,
|
||||
Err(err) => Some(err),
|
||||
Ok((_, Some(err))) | Err(err) => Some(err),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -709,13 +674,11 @@ impl HealTask {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !recorded_authoritative_outcome {
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
});
|
||||
}
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
});
|
||||
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(
|
||||
|
||||
@@ -451,11 +451,6 @@ impl HealTask {
|
||||
self.source,
|
||||
)
|
||||
.with_replacement_targets(replacement_targets, is_auto_replacement.then(|| self.id.clone()))
|
||||
.with_pool_metadata_targets(if self.options.recreate_missing && !self.options.dry_run {
|
||||
self.heal_endpoints.clone()
|
||||
} else {
|
||||
Vec::new()
|
||||
})
|
||||
.with_replacement_identity_fence(replacement_target_identities.clone())
|
||||
.with_mainline_pacer(self.mainline_pacer.clone());
|
||||
|
||||
|
||||
@@ -162,9 +162,6 @@ impl HealTask {
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
let mut expected_identity =
|
||||
self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
|
||||
expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?;
|
||||
|
||||
let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts);
|
||||
let heal_result = if self.source == HealRequestSource::ReadRepair {
|
||||
@@ -269,6 +266,8 @@ impl HealTask {
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(1, 1, 0, 0, object_size);
|
||||
}
|
||||
let expected_identity =
|
||||
self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
|
||||
self.record_verified_storage_receipt(expected_identity, storage_result.receipt)
|
||||
.await;
|
||||
self.record_result_item(result).await;
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
|
||||
use super::*;
|
||||
use crate::heal::POOL_META_NAME;
|
||||
use crate::heal::storage::HealStorageObjectResult;
|
||||
|
||||
mod deferred_retry;
|
||||
@@ -43,7 +42,6 @@ mod canonical_outcome {
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cluster_retries_only_the_failed_listing_page() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
recoverable_second_page_failures: Mutex::new(Some(1)),
|
||||
..Default::default()
|
||||
});
|
||||
@@ -68,7 +66,7 @@ mod canonical_outcome {
|
||||
assert_eq!(task.get_progress().await.objects_scanned, 2);
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||
["object-a", "object-b", POOL_META_NAME]
|
||||
["object-a", "object-b"]
|
||||
);
|
||||
assert_eq!(
|
||||
storage.listing_tokens.lock().expect("listing tokens").as_slice(),
|
||||
@@ -156,126 +154,6 @@ mod canonical_outcome {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_heal_records_matching_positive_storage_receipts() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([
|
||||
(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||
),
|
||||
(
|
||||
"object-b".to_string(),
|
||||
VecDeque::from([object_receipt("object-b", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||
),
|
||||
])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage);
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("bucket heal should record verified object receipts");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.healed, 2);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
assert_eq!(outcome.objects.len(), 2);
|
||||
assert!(outcome.objects.iter().all(|item| {
|
||||
item.identity.bucket_incarnation_id == Some(incarnation) && item.disposition == HealObjectDisposition::Repaired
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_heal_keeps_repairing_when_bucket_incarnation_is_unavailable() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
None,
|
||||
HealObjectDisposition::Repaired,
|
||||
Uuid::new_v4(),
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_unavailable: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage.clone());
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("bucket heal should continue when only proof ownership is unavailable");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unknown, 2);
|
||||
assert!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.all(|item| item.disposition == HealObjectDisposition::Unknown)
|
||||
);
|
||||
assert_eq!(storage.healed_objects.lock().expect("healed objects").len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_heal_rejects_stale_receipts_without_double_recording() {
|
||||
let expected_incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([
|
||||
(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
None,
|
||||
HealObjectDisposition::Repaired,
|
||||
Uuid::new_v4(),
|
||||
)]),
|
||||
),
|
||||
(
|
||||
"object-b".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-b",
|
||||
None,
|
||||
HealObjectDisposition::Repaired,
|
||||
expected_incarnation,
|
||||
)]),
|
||||
),
|
||||
])),
|
||||
bucket_incarnation_id: Mutex::new(Some(expected_incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage);
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("stale bucket receipt should not fail the legacy heal");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.healed, 1);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.objects.len(), 2);
|
||||
let object_a = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-a")
|
||||
.expect("stale receipt object outcome");
|
||||
assert_eq!(object_a.disposition, HealObjectDisposition::Unknown);
|
||||
let object_b = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-b")
|
||||
.expect("matching receipt object outcome");
|
||||
assert_eq!(object_b.disposition, HealObjectDisposition::Repaired);
|
||||
assert_eq!(object_b.identity.bucket_incarnation_id, Some(expected_incarnation));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn exhausted_object_does_not_abort_other_objects_or_erase_counts() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
@@ -435,96 +313,6 @@ mod canonical_outcome {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mixed_grace_and_repaired_receipt_transfer_only_repaired_responsibility() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_outcomes: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([MockHealObjectOutcome::DanglingGraceDeferred]),
|
||||
)])),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-b".to_string(),
|
||||
VecDeque::from([object_receipt("object-b", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage);
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("mixed grace and repaired receipt should complete");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.healed, 1);
|
||||
assert_eq!(outcome.counters.skipped, 1);
|
||||
let deferred = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-a")
|
||||
.expect("grace object should remain recorded");
|
||||
assert!(matches!(
|
||||
deferred.disposition,
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::DanglingDeleteGrace,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_ne!(deferred.disposition, HealObjectDisposition::Repaired);
|
||||
let repaired = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-b")
|
||||
.expect("receipt-backed object should be recorded");
|
||||
assert_eq!(repaired.identity.bucket_incarnation_id, Some(incarnation));
|
||||
assert_eq!(repaired.disposition, HealObjectDisposition::Repaired);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_heal_records_matching_positive_storage_receipt() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage);
|
||||
|
||||
task.execute().await.expect("bucket heal should complete");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.processed, 2);
|
||||
assert_eq!(outcome.counters.healed, 1);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|item| item.identity.object == "object-a")
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
let repaired = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-a")
|
||||
.expect("receipt-backed bucket object should be recorded");
|
||||
assert_eq!(repaired.identity.bucket_incarnation_id, Some(incarnation));
|
||||
assert_eq!(repaired.disposition, HealObjectDisposition::Repaired);
|
||||
let legacy = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == "object-b")
|
||||
.expect("legacy bucket object should still be recorded");
|
||||
assert_eq!(legacy.disposition, HealObjectDisposition::Unknown);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn grace_single_object_is_completed_but_deferred() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
@@ -556,18 +344,6 @@ mod canonical_outcome {
|
||||
.lock()
|
||||
.expect("existence fixture")
|
||||
.insert("object".to_string(), MockObjectExists::TransientSkip("retry later"));
|
||||
} else {
|
||||
let incarnation = Uuid::new_v4();
|
||||
*storage.bucket_incarnation_id.lock().expect("bucket incarnation") = Some(incarnation);
|
||||
storage.heal_object_receipts.lock().expect("receipt fixture").insert(
|
||||
"object".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object",
|
||||
None,
|
||||
HealObjectDisposition::VerifiedHealthy,
|
||||
incarnation,
|
||||
)]),
|
||||
);
|
||||
}
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object".to_string(), None);
|
||||
request.options.dry_run = !transient;
|
||||
@@ -1327,7 +1103,6 @@ struct MockStorage {
|
||||
retry_test_events: Mutex<Vec<String>>,
|
||||
listed: Mutex<bool>,
|
||||
list_each_bucket: bool,
|
||||
pool_metadata_required: bool,
|
||||
fail_second_listing_page: bool,
|
||||
recoverable_second_page_failures: Mutex<Option<usize>>,
|
||||
listing_tokens: Mutex<Vec<Option<String>>>,
|
||||
@@ -1341,13 +1116,6 @@ struct MockStorage {
|
||||
heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
|
||||
heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
|
||||
heal_object_receipts: Mutex<HashMap<String, VecDeque<HealObjectReceipt>>>,
|
||||
bucket_incarnation_id: Mutex<Option<Uuid>>,
|
||||
bucket_incarnation_calls: AtomicU64,
|
||||
bucket_incarnation_error: Mutex<Option<Error>>,
|
||||
block_bucket_incarnation: bool,
|
||||
bucket_incarnation_started: tokio::sync::Notify,
|
||||
bucket_incarnation_after_object_heal: Mutex<Option<Uuid>>,
|
||||
bucket_incarnation_unavailable: Mutex<bool>,
|
||||
format_no_heal_required: Mutex<bool>,
|
||||
format_error: Mutex<Option<Error>>,
|
||||
global_format_calls: Mutex<u32>,
|
||||
@@ -1451,19 +1219,14 @@ async fn execute_emits_heal_trace_task_state() {
|
||||
assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed"));
|
||||
}
|
||||
|
||||
fn object_receipt(
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
disposition: HealObjectDisposition,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> HealObjectReceipt {
|
||||
fn object_receipt(object: &str, version_id: Option<&str>, disposition: HealObjectDisposition) -> HealObjectReceipt {
|
||||
HealObjectReceipt {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket-a".to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: version_id.map(ToOwned::to_owned),
|
||||
bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
bucket_incarnation_id: Some(Uuid::new_v4()),
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
@@ -1473,23 +1236,16 @@ fn object_receipt(
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_records_matching_positive_storage_receipt() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::Repaired,
|
||||
incarnation,
|
||||
)]),
|
||||
VecDeque::from([object_receipt("object-a", Some("version-a"), HealObjectDisposition::Repaired)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage.clone(),
|
||||
storage,
|
||||
);
|
||||
|
||||
task.execute().await.expect("mock object heal should complete");
|
||||
@@ -1502,276 +1258,19 @@ async fn object_heal_records_matching_positive_storage_receipt() {
|
||||
assert_eq!(object.identity.version_id.as_deref(), Some("version-a"));
|
||||
assert!(object.identity.bucket_incarnation_id.is_some());
|
||||
assert_eq!(object.disposition, HealObjectDisposition::Repaired);
|
||||
assert_eq!(
|
||||
storage.bucket_incarnation_calls.load(Ordering::Relaxed),
|
||||
1,
|
||||
"latch the owner exactly once before repair"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_owner_lookup_failure_preserves_unverified_repair() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_error: Mutex::new(Some(Error::other("owner metadata unavailable"))),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
None,
|
||||
HealObjectDisposition::Repaired,
|
||||
Uuid::new_v4(),
|
||||
)]),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(HealRequest::object("bucket-a".to_string(), "object-a".to_string(), None), storage.clone());
|
||||
|
||||
task.execute().await.expect("missing receipt owner must not prevent repair");
|
||||
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("heal calls").as_slice(), ["object-a"]);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(
|
||||
outcome.objects.front().expect("unverified outcome").disposition,
|
||||
HealObjectDisposition::Unknown
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_dry_run_skips_owner_lookup_and_positive_receipts() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_error: Mutex::new(Some(Error::other("dry-run must not query the receipt owner"))),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
None,
|
||||
HealObjectDisposition::Repaired,
|
||||
Uuid::new_v4(),
|
||||
)]),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object-a".to_string(), None);
|
||||
request.options.dry_run = true;
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
task.execute().await.expect("dry-run should complete without owner metadata");
|
||||
|
||||
assert!(storage.object_heal_opts.lock().expect("heal options")[0].dry_run);
|
||||
assert_eq!(storage.bucket_incarnation_calls.load(Ordering::Relaxed), 0);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
assert_eq!(outcome.counters.skipped, 1);
|
||||
assert_eq!(
|
||||
outcome.objects.front().expect("dry-run outcome").disposition,
|
||||
HealObjectDisposition::DryRunObserved
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn object_heal_owner_lookup_obeys_task_timeout() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
block_bucket_incarnation: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object-a".to_string(), None);
|
||||
request.options.timeout = Some(Duration::from_secs(5));
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let result = tokio::time::timeout(Duration::from_secs(60), task.execute())
|
||||
.await
|
||||
.expect("owner lookup must honor the task deadline");
|
||||
|
||||
assert!(matches!(result, Err(Error::TaskTimeout)));
|
||||
assert!(storage.heal_object_calls.lock().expect("heal calls").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_owner_lookup_obeys_cancellation() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
block_bucket_incarnation: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_string(), "object-a".to_string(), None);
|
||||
request.options.timeout = None;
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let (result, ()) = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
tokio::join!(task.execute(), async {
|
||||
storage.bucket_incarnation_started.notified().await;
|
||||
task.cancel().await.expect("cancel pending owner lookup");
|
||||
})
|
||||
})
|
||||
.await
|
||||
.expect("cancellation must interrupt owner lookup");
|
||||
|
||||
assert!(matches!(result, Err(Error::TaskCancelled)));
|
||||
assert!(storage.heal_object_calls.lock().expect("heal calls").is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_object_heal_rejects_matching_positive_storage_receipt() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
task.cancel().await.expect("task cancellation should succeed");
|
||||
|
||||
let expected = HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket-a".to_string(),
|
||||
object: "object-a".to_string(),
|
||||
version_id: Some("version-a".to_string()),
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
};
|
||||
let accepted = task
|
||||
.record_verified_storage_receipt(
|
||||
expected,
|
||||
Some(object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::Repaired,
|
||||
incarnation,
|
||||
)),
|
||||
)
|
||||
.await;
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert!(!accepted);
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert!(outcome.objects.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_object_heal_rejects_matching_positive_storage_receipt() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::OkWithOtherError("commit failed"))),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::Repaired,
|
||||
incarnation,
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
let result = task.execute().await;
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unchanged, 0);
|
||||
assert!(outcome.objects.iter().all(|object| {
|
||||
!matches!(
|
||||
object.disposition,
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_quorum_object_heal_rejects_matching_positive_storage_receipt() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::OkWithReadQuorum)),
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::Repaired,
|
||||
incarnation,
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
let result = task.execute().await;
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unchanged, 0);
|
||||
assert!(outcome.objects.iter().all(|object| {
|
||||
!matches!(
|
||||
object.disposition,
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_latches_expected_incarnation_before_repair() {
|
||||
let original_incarnation = Uuid::new_v4();
|
||||
let successor_incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
HealObjectDisposition::VerifiedHealthy,
|
||||
original_incarnation,
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(original_incarnation)),
|
||||
bucket_incarnation_after_object_heal: Mutex::new(Some(successor_incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
task.execute().await.expect("mock object heal should complete");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.unchanged, 1);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
let object = outcome.objects.front().expect("latched receipt should be recorded");
|
||||
assert_eq!(object.identity.bucket_incarnation_id, Some(original_incarnation));
|
||||
assert_eq!(object.disposition, HealObjectDisposition::VerifiedHealthy);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_rejects_mismatched_or_legacy_storage_receipts() {
|
||||
let expected_incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("version-a"),
|
||||
Some("old-version"),
|
||||
HealObjectDisposition::Repaired,
|
||||
Uuid::new_v4(),
|
||||
)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(expected_incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
@@ -1862,7 +1361,6 @@ enum MockHealObjectOutcome {
|
||||
RetryableLock,
|
||||
RetryableLockTimeout,
|
||||
OkWithOtherError(&'static str),
|
||||
OkWithReadQuorum,
|
||||
ErrOther(&'static str),
|
||||
DanglingGraceDeferred,
|
||||
UnavailableDrive(DriveState),
|
||||
@@ -1947,34 +1445,6 @@ impl HealStorageAPI for MockStorage {
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn heal_pool_metadata(&self, opts: &HealOpts) -> Result<Vec<HealResultItem>> {
|
||||
if !self.pool_metadata_required {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let scopes = self.erasure_set_scopes.lock().expect("metadata scopes").clone();
|
||||
let scopes = if scopes.is_empty() {
|
||||
vec![(opts.pool.unwrap_or(0), opts.set.unwrap_or(0))]
|
||||
} else {
|
||||
scopes
|
||||
};
|
||||
let mut results = Vec::new();
|
||||
for (pool, set) in scopes {
|
||||
let scoped_opts = HealOpts {
|
||||
pool: Some(pool),
|
||||
set: Some(set),
|
||||
..*opts
|
||||
};
|
||||
let (result, error) = self
|
||||
.heal_object(RUSTFS_META_BUCKET, crate::heal::POOL_META_NAME, None, &scoped_opts)
|
||||
.await?;
|
||||
if let Some(error) = error {
|
||||
return Err(error);
|
||||
}
|
||||
results.push(result);
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
async fn object_exists(&self, _bucket: &str, object: &str) -> Result<bool> {
|
||||
if let Some(result) = self.object_exists_by_name.lock().unwrap().get(object).copied() {
|
||||
return match result {
|
||||
@@ -1986,21 +1456,6 @@ impl HealStorageAPI for MockStorage {
|
||||
Ok(self.object_exists.lock().unwrap().unwrap_or(true))
|
||||
}
|
||||
|
||||
async fn bucket_incarnation_id(&self, _bucket: &str) -> Result<Option<Uuid>> {
|
||||
self.bucket_incarnation_calls.fetch_add(1, Ordering::Relaxed);
|
||||
self.bucket_incarnation_started.notify_one();
|
||||
if self.block_bucket_incarnation {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
if let Some(error) = self.bucket_incarnation_error.lock().expect("owner lookup error").take() {
|
||||
return Err(error);
|
||||
}
|
||||
if *self.bucket_incarnation_unavailable.lock().unwrap() {
|
||||
return Err(Error::Other("bucket incarnation unavailable".to_string()));
|
||||
}
|
||||
Ok(*self.bucket_incarnation_id.lock().unwrap())
|
||||
}
|
||||
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2058,13 +1513,6 @@ impl HealStorageAPI for MockStorage {
|
||||
MockHealObjectOutcome::RetryableSlowDown => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||
}
|
||||
MockHealObjectOutcome::OkWithReadQuorum => Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
))),
|
||||
)),
|
||||
MockHealObjectOutcome::PermanentOther(message) => Err(Error::other(message)),
|
||||
MockHealObjectOutcome::OkWithOtherError(message) => Ok((HealResultItem::default(), Some(Error::other(message)))),
|
||||
MockHealObjectOutcome::ErrOther(message) => Err(Error::other(message)),
|
||||
@@ -2104,13 +1552,6 @@ impl HealStorageAPI for MockStorage {
|
||||
MockHealObjectOutcome::RetryableSlowDown => {
|
||||
Ok((HealResultItem::default(), Some(Error::Storage(EcstoreError::SlowDown))))
|
||||
}
|
||||
MockHealObjectOutcome::OkWithReadQuorum => Ok((
|
||||
HealResultItem::default(),
|
||||
Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
))),
|
||||
)),
|
||||
};
|
||||
}
|
||||
if bucket == RUSTFS_META_BUCKET && object == format!("{BUCKET_META_PREFIX}/{DATA_USAGE_CACHE_NAME}") {
|
||||
@@ -2125,9 +1566,6 @@ impl HealStorageAPI for MockStorage {
|
||||
return Ok((HealResultItem::default(), Some(Error::Disk(DiskError::FileNotFound))));
|
||||
}
|
||||
self.healed_objects.lock().unwrap().push(object.to_string());
|
||||
if let Some(bucket_incarnation_id) = self.bucket_incarnation_after_object_heal.lock().unwrap().take() {
|
||||
*self.bucket_incarnation_id.lock().unwrap() = Some(bucket_incarnation_id);
|
||||
}
|
||||
Ok((
|
||||
HealResultItem {
|
||||
object_size: 1,
|
||||
@@ -2715,187 +2153,9 @@ async fn test_recursive_bucket_heal_treats_missing_continuation_token_as_end() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_heal_restores_pool_metadata_without_user_buckets() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
listed_buckets: Mutex::new(Some(Vec::new())),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(storage.pool_metadata_required);
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal),
|
||||
storage.clone(),
|
||||
);
|
||||
|
||||
task.execute().await.expect("root heal should restore required pool metadata");
|
||||
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("heal calls").as_slice(),
|
||||
[crate::heal::POOL_META_NAME]
|
||||
);
|
||||
assert!(matches!(task.get_status().await, HealTaskStatus::Completed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_heal_pool_metadata_cannot_hide_a_later_owner_failure() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
listed_buckets: Mutex::new(Some(Vec::new())),
|
||||
erasure_set_scopes: Mutex::new(vec![(0, 0), (1, 1)]),
|
||||
..Default::default()
|
||||
});
|
||||
storage.heal_object_outcomes.lock().expect("metadata outcomes").insert(
|
||||
crate::heal::POOL_META_NAME.to_string(),
|
||||
VecDeque::from([
|
||||
MockHealObjectOutcome::UnavailableDrive(DriveState::Ok),
|
||||
MockHealObjectOutcome::OkWithReadQuorum,
|
||||
]),
|
||||
);
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal),
|
||||
storage.clone(),
|
||||
);
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("one healthy owner cannot satisfy another owner's recovery");
|
||||
|
||||
assert!(matches!(error, Error::Storage(EcstoreError::InsufficientReadQuorum(_, _))));
|
||||
{
|
||||
let opts = storage.object_heal_opts.lock().expect("owner options");
|
||||
assert_eq!(
|
||||
opts.iter().map(|opts| (opts.pool, opts.set)).collect::<Vec<_>>(),
|
||||
vec![(Some(0), Some(0)), (Some(1), Some(1))]
|
||||
);
|
||||
}
|
||||
assert!(!matches!(task.get_status().await, HealTaskStatus::Completed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_heal_pool_metadata_does_not_inherit_remove_or_no_lock() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
listed_buckets: Mutex::new(Some(Vec::new())),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
remove_corrupted: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("metadata repair should retain its write fence and reject destructive options");
|
||||
|
||||
let opts = storage.object_heal_opts.lock().expect("metadata options");
|
||||
assert_eq!(opts.len(), 1, "an empty user namespace must still inspect metadata");
|
||||
assert!(!opts[0].dry_run);
|
||||
assert!(!opts[0].remove);
|
||||
assert!(!opts[0].no_lock);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_heal_pool_metadata_failure_does_not_prevent_user_repairs() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
..Default::default()
|
||||
});
|
||||
storage.heal_object_outcomes.lock().expect("metadata outcome").insert(
|
||||
crate::heal::POOL_META_NAME.to_string(),
|
||||
VecDeque::from([MockHealObjectOutcome::OkWithReadQuorum]),
|
||||
);
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("unrecovered metadata must still fail root completion");
|
||||
|
||||
assert!(matches!(error, Error::Storage(EcstoreError::InsufficientReadQuorum(_, _))));
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("heal calls").as_slice(),
|
||||
["object-a", "object-b", crate::heal::POOL_META_NAME]
|
||||
);
|
||||
assert!(!matches!(task.get_status().await, HealTaskStatus::Completed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_heal_pool_metadata_preserves_typed_quorum_failure() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
listed_buckets: Mutex::new(Some(Vec::new())),
|
||||
..Default::default()
|
||||
});
|
||||
storage.heal_object_outcomes.lock().expect("metadata outcome").insert(
|
||||
crate::heal::POOL_META_NAME.to_string(),
|
||||
VecDeque::from([MockHealObjectOutcome::OkWithReadQuorum]),
|
||||
);
|
||||
let task = HealTask::from_request(HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal), storage);
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("metadata quorum failure must prevent root completion");
|
||||
|
||||
assert!(matches!(error, Error::Storage(EcstoreError::InsufficientReadQuorum(_, _))));
|
||||
assert!(!matches!(task.get_status().await, HealTaskStatus::Completed));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn root_heal_pool_metadata_obeys_task_timeout() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
listed_buckets: Mutex::new(Some(Vec::new())),
|
||||
retry_test_delays: HashMap::from([(crate::heal::POOL_META_NAME.to_string(), Duration::from_secs(10))]),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
timeout: Some(Duration::from_millis(10)),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage,
|
||||
);
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("metadata work must stay inside the root task budget");
|
||||
|
||||
assert!(matches!(error, Error::TaskTimeout));
|
||||
assert!(!matches!(task.get_status().await, HealTaskStatus::Completed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cluster_heal_visits_bucket_objects() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
..Default::default()
|
||||
});
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
let request = HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
@@ -2911,102 +2171,9 @@ async fn test_cluster_heal_visits_bucket_objects() {
|
||||
|
||||
assert_eq!(
|
||||
storage.healed_objects.lock().unwrap().as_slice(),
|
||||
["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()]
|
||||
);
|
||||
assert!(matches!(task.get_status().await, HealTaskStatus::Completed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cluster_recreate_heals_pool_metadata_after_user_buckets() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
..Default::default()
|
||||
});
|
||||
let request = HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
recreate_missing: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("cluster recreate heal should include pool metadata");
|
||||
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||
["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()]
|
||||
);
|
||||
let opts = storage.object_heal_opts.lock().expect("object opts");
|
||||
assert!(opts.last().expect("pool metadata opts").recreate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cluster_recreate_fails_when_pool_metadata_heal_fails() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
..Default::default()
|
||||
});
|
||||
storage.heal_object_outcomes.lock().expect("object outcomes").insert(
|
||||
POOL_META_NAME.to_string(),
|
||||
VecDeque::from([MockHealObjectOutcome::ErrOther("pool metadata missing")]),
|
||||
);
|
||||
let request = HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
recreate_missing: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let err = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("cluster recreate heal must not hide pool metadata failure");
|
||||
|
||||
assert!(matches!(err, Error::Other(message) if message == "pool metadata missing"));
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||
["object-a".to_string(), "object-b".to_string(), POOL_META_NAME.to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cluster_dry_run_does_not_heal_pool_metadata() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
pool_metadata_required: true,
|
||||
..Default::default()
|
||||
});
|
||||
let request = HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
dry_run: true,
|
||||
recreate_missing: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("dry-run cluster heal should preserve existing coverage");
|
||||
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||
["object-a".to_string(), "object-b".to_string()]
|
||||
);
|
||||
assert!(matches!(task.get_status().await, HealTaskStatus::Completed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -29,7 +29,6 @@ use rustfs_heal::heal::{
|
||||
storage::{ECStoreHealStorage, HealStorageAPI},
|
||||
};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
#[cfg(unix)]
|
||||
use std::{
|
||||
fs::{File, OpenOptions},
|
||||
@@ -49,12 +48,6 @@ use storage_api::endpoint_index::{Endpoint, EndpointServerPools, Endpoints, Pool
|
||||
const META_BUCKET: &str = ".rustfs.sys";
|
||||
const JOURNAL_REL: &str = "buckets/.heal/mrf/journal.bin";
|
||||
const SCOPED_JOURNAL_REL: &str = "buckets/.heal/mrf/journal-scoped.bin";
|
||||
const COMMITTED_PAYLOAD_REL: &str = ".heal-mrf-snapshot.0.bin";
|
||||
const COMMITTED_MANIFEST_REL: &str = ".heal-mrf-commit.0.bin";
|
||||
const COMMITTED_PAYLOAD_RELS: [&str; 2] = [".heal-mrf-snapshot.0.bin", ".heal-mrf-snapshot.1.bin"];
|
||||
const COMMITTED_MANIFEST_RELS: [&str; 2] = [".heal-mrf-commit.0.bin", ".heal-mrf-commit.1.bin"];
|
||||
const COMMITTED_MAGIC: &[u8; 8] = b"RFMRFC01";
|
||||
const COMMITTED_MANIFEST_LEN: usize = 8 + 1 + 16 + 8 + 8 + 32 + 32;
|
||||
|
||||
async fn heal_env() -> (Vec<std::path::PathBuf>, Arc<dyn HealStorageAPI>) {
|
||||
heal_env_at(None).await
|
||||
@@ -202,29 +195,6 @@ fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
|
||||
write_journal_path_to_disks(disk_paths, JOURNAL_REL, data);
|
||||
}
|
||||
|
||||
fn committed_manifest(owner: uuid::Uuid, sequence: u64, payload: &[u8]) -> Vec<u8> {
|
||||
let mut manifest = Vec::with_capacity(COMMITTED_MANIFEST_LEN);
|
||||
manifest.extend_from_slice(COMMITTED_MAGIC);
|
||||
manifest.push(1);
|
||||
manifest.extend_from_slice(owner.as_bytes());
|
||||
manifest.extend_from_slice(&sequence.to_le_bytes());
|
||||
manifest.extend_from_slice(
|
||||
&u64::try_from(payload.len())
|
||||
.expect("fixture payload length fits")
|
||||
.to_le_bytes(),
|
||||
);
|
||||
manifest.extend_from_slice(&Sha256::digest(payload));
|
||||
manifest.extend_from_slice(&Sha256::digest(&manifest));
|
||||
assert_eq!(manifest.len(), COMMITTED_MANIFEST_LEN, "committed fixture manifest length");
|
||||
manifest
|
||||
}
|
||||
|
||||
fn write_committed_snapshot_to_disks(disk_paths: &[std::path::PathBuf], sequence: u64, payload: &[u8]) {
|
||||
let manifest = committed_manifest(uuid::Uuid::new_v4(), sequence, payload);
|
||||
write_journal_path_to_disks(disk_paths, COMMITTED_PAYLOAD_REL, payload);
|
||||
write_journal_path_to_disks(disk_paths, COMMITTED_MANIFEST_REL, &manifest);
|
||||
}
|
||||
|
||||
fn journal_exists_on_all_disks(disk_paths: &[std::path::PathBuf], relative_path: &str) -> bool {
|
||||
disk_paths
|
||||
.iter()
|
||||
@@ -237,44 +207,6 @@ fn journal_matches_on_all_disks(disk_paths: &[PathBuf], relative_path: &str, exp
|
||||
.all(|path| std::fs::read(path.join(META_BUCKET).join(relative_path)).is_ok_and(|actual| actual == expected))
|
||||
}
|
||||
|
||||
fn committed_checkpoint_matches_on_all_disks(disk_paths: &[PathBuf], sequence: u64, expected_payload: &[u8]) -> bool {
|
||||
disk_paths.iter().all(|path| {
|
||||
let root = path.join(META_BUCKET);
|
||||
COMMITTED_PAYLOAD_RELS
|
||||
.into_iter()
|
||||
.zip(COMMITTED_MANIFEST_RELS)
|
||||
.any(|(payload_rel, manifest_rel)| {
|
||||
let Ok(payload) = std::fs::read(root.join(payload_rel)) else {
|
||||
return false;
|
||||
};
|
||||
if payload != expected_payload {
|
||||
return false;
|
||||
}
|
||||
let Ok(manifest) = std::fs::read(root.join(manifest_rel)) else {
|
||||
return false;
|
||||
};
|
||||
if manifest.len() != COMMITTED_MANIFEST_LEN || &manifest[..8] != COMMITTED_MAGIC || manifest[8] != 1 {
|
||||
return false;
|
||||
}
|
||||
let Ok(recorded_sequence) = <[u8; 8]>::try_from(&manifest[25..33]).map(u64::from_le_bytes) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(recorded_len) = <[u8; 8]>::try_from(&manifest[33..41]).map(u64::from_le_bytes) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(expected_len) = u64::try_from(expected_payload.len()) else {
|
||||
return false;
|
||||
};
|
||||
if recorded_sequence != sequence || recorded_len != expected_len {
|
||||
return false;
|
||||
}
|
||||
let payload_digest: [u8; 32] = Sha256::digest(expected_payload).into();
|
||||
let manifest_digest: [u8; 32] = Sha256::digest(&manifest[..COMMITTED_MANIFEST_LEN - 32]).into();
|
||||
payload_digest.as_slice() == &manifest[41..73] && manifest_digest.as_slice() == &manifest[73..]
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
@@ -323,12 +255,11 @@ async fn decode_failure_intent_maps_to_urgent_mrf_heal_request() {
|
||||
}
|
||||
|
||||
/// A journal left behind by a previous process must be replayed into the
|
||||
/// manager queue, and a torn tail must not block replay of the intact records.
|
||||
/// The partial-write record keeps the legacy journal as the durable anchor
|
||||
/// until an exact verified repair proof can discharge it.
|
||||
/// manager queue and then removed, and a torn tail must not block replay of
|
||||
/// the intact records.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn journal_replay_arms_intents_and_retains_unproven_partial_write_anchor() {
|
||||
async fn journal_replay_arms_intents_and_deletes_the_file() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
|
||||
// The journal reader resolves disks through the process-local disk map;
|
||||
@@ -354,14 +285,14 @@ async fn journal_replay_arms_intents_and_retains_unproven_partial_write_anchor()
|
||||
assert!(
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
|
||||
"partial-write replay must retain the legacy journal until durable proof"
|
||||
.all(|path| !Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()),
|
||||
"the journal file must be removed after a successful replay"
|
||||
);
|
||||
assert!(
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()),
|
||||
"missing authoritative journal remains absent"
|
||||
"the authoritative journal file must also be removed after replay"
|
||||
);
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
@@ -369,76 +300,6 @@ async fn journal_replay_arms_intents_and_retains_unproven_partial_write_anchor()
|
||||
assert!(snapshot.queued_by_priority.normal >= 1, "the partial-write record must replay as Normal");
|
||||
}
|
||||
|
||||
/// A committed checkpoint published by the new two-slot writer is the
|
||||
/// authoritative startup snapshot. Legacy mirrors are fallback-only and must
|
||||
/// not be merged with or preferred over the committed epoch.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn committed_snapshot_replay_takes_precedence_over_stale_legacy_mirror() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
register_local_disks(&disk_paths, "mrf-committed-replay-test").await;
|
||||
|
||||
let committed = scoped_journal_record(3, "committed-bucket", "committed-object", Some([9u8; 16]), 0, 0, 0);
|
||||
let stale_legacy = journal_record(1, "legacy-bucket", "legacy-object", None, 0);
|
||||
write_committed_snapshot_to_disks(&disk_paths, 7, &committed);
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &stale_legacy);
|
||||
|
||||
let manager = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||
assert_eq!(replayed, 1, "only the committed snapshot epoch may replay");
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
assert_eq!(snapshot.queued_by_source.mrf, 1);
|
||||
assert_eq!(
|
||||
snapshot.queued_by_priority.normal, 1,
|
||||
"the committed partial-write record must replay instead of the stale legacy decode-failure"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot.queued_by_priority.urgent, 0,
|
||||
"stale legacy decode-failure records must not be mixed into committed replay"
|
||||
);
|
||||
assert!(
|
||||
journal_exists_on_all_disks(&disk_paths, COMMITTED_MANIFEST_REL),
|
||||
"the committed checkpoint remains until the accepted partial-write has proof"
|
||||
);
|
||||
}
|
||||
|
||||
/// A damaged committed checkpoint is ambiguous: replay must not fall back to
|
||||
/// older legacy bytes or delete any recovery anchor until another process can
|
||||
/// publish a valid successor.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn damaged_committed_snapshot_blocks_legacy_fallback_and_retains_anchors() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
register_local_disks(&disk_paths, "mrf-damaged-committed-replay-test").await;
|
||||
|
||||
let committed = scoped_journal_record(3, "damaged-committed-bucket", "committed-object", Some([8u8; 16]), 0, 0, 0);
|
||||
let stale_legacy = journal_record(1, "damaged-legacy-bucket", "legacy-object", None, 0);
|
||||
write_journal_path_to_disks(&disk_paths, COMMITTED_PAYLOAD_REL, &committed);
|
||||
let mut manifest = committed_manifest(uuid::Uuid::new_v4(), 9, &committed);
|
||||
manifest[25] ^= 1;
|
||||
write_journal_path_to_disks(&disk_paths, COMMITTED_MANIFEST_REL, &manifest);
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &stale_legacy);
|
||||
|
||||
let manager = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||
assert_eq!(replayed, 0, "damaged committed state must fail closed");
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queued_by_source.mrf,
|
||||
0,
|
||||
"stale legacy bytes must not be replayed when committed state is ambiguous"
|
||||
);
|
||||
assert!(
|
||||
journal_exists_on_all_disks(&disk_paths, COMMITTED_MANIFEST_REL)
|
||||
&& journal_exists_on_all_disks(&disk_paths, COMMITTED_PAYLOAD_REL)
|
||||
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &stale_legacy)
|
||||
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &stale_legacy),
|
||||
"all recovery anchors must remain after a fail-closed committed read"
|
||||
);
|
||||
}
|
||||
|
||||
/// A canonical snapshot and its compatibility mirror may differ after a
|
||||
/// partial flush. Replay must choose the complete canonical epoch instead of
|
||||
/// combining records that never coexisted in memory.
|
||||
@@ -461,24 +322,21 @@ async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
|
||||
assert_eq!(snapshot.queued_by_source.mrf, 1);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"accepted replay responsibilities remain anchored until a verified repair proof"
|
||||
"replay cleanup must remove both journal paths"
|
||||
);
|
||||
|
||||
// A scoped-only snapshot is valid during a rollout where no legacy
|
||||
// compatibility mirror was written. Missing legacy files must not leave
|
||||
// the runtime in a permanent cleanup-retry state.
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
register_local_disks(&disk_paths, "mrf-scoped-authoritative-test").await;
|
||||
let manager = make_manager(storage);
|
||||
let scoped_only = journal_record(1, "scoped-only-bucket", "scoped-only-object", None, 0);
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_only);
|
||||
assert_eq!(mrf_queue::replay_journal_once(&manager).await, 1);
|
||||
assert!(disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
|
||||
let scoped_v2 = scoped_journal_record(1, "scoped-v2-bucket", "scoped-v2-object", None, 0, 3, 7);
|
||||
@@ -492,12 +350,12 @@ async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
|
||||
);
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queued_by_source.mrf,
|
||||
2,
|
||||
"only the scoped-only and scoped-v2 authoritative epochs should have reached the manager"
|
||||
3,
|
||||
"only the three authoritative/scoped-only epochs should have reached the manager"
|
||||
);
|
||||
assert!(disk_paths.iter().all(|path| {
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -542,13 +400,10 @@ async fn authoritative_journal_replay_preserves_kind_and_scope_identity() {
|
||||
snapshot.queued_by_priority.urgent, 1,
|
||||
"decode-failure repair must not merge with object repair responsibility"
|
||||
);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"partial-write responsibilities keep both replay anchors until proof"
|
||||
);
|
||||
assert!(disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
}
|
||||
|
||||
/// If replay reaches a full heal-manager queue, the old journal remains the
|
||||
@@ -666,18 +521,16 @@ fn mrf_successor_flush_child_process_fixture() {
|
||||
}),
|
||||
));
|
||||
mrf_queue::spawn_mrf_consumer(manager.clone());
|
||||
let mut expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||
expected_successor.extend(journal_record(1, "successor-bucket", "first-object", None, 0));
|
||||
let expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||
let flushed = wait_until(Duration::from_secs(10), || async {
|
||||
manager.operations_snapshot().await.queued_by_source.mrf == 1
|
||||
&& committed_checkpoint_matches_on_all_disks(&disk_paths, 2, &expected_successor)
|
||||
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor)
|
||||
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &expected_successor)
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
flushed,
|
||||
"child process must publish the committed pending successor before the delete phase"
|
||||
"child process must publish the pending successor snapshot before the delete phase"
|
||||
);
|
||||
});
|
||||
std::process::exit(78);
|
||||
@@ -715,18 +568,16 @@ fn mrf_successor_flush_waiting_child_process_fixture() {
|
||||
}),
|
||||
));
|
||||
mrf_queue::spawn_mrf_consumer(manager.clone());
|
||||
let mut expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||
expected_successor.extend(journal_record(1, "service-kill-bucket", "first-object", None, 0));
|
||||
let expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||
let flushed = wait_until(Duration::from_secs(10), || async {
|
||||
manager.operations_snapshot().await.queued_by_source.mrf == 1
|
||||
&& committed_checkpoint_matches_on_all_disks(&disk_paths, 2, &expected_successor)
|
||||
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor)
|
||||
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &expected_successor)
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
flushed,
|
||||
"child process must publish the committed pending successor before it can be killed"
|
||||
"child process must publish the pending successor snapshot before it can be killed"
|
||||
);
|
||||
std::fs::write(&ready_path, b"ready").expect("write ready marker");
|
||||
loop {
|
||||
@@ -757,8 +608,7 @@ fn mrf_authoritative_fsync_waiting_child_process_fixture() {
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &startup);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &startup);
|
||||
|
||||
let mut successor = journal_record(1, "fsync-kill-bucket", "second-object", None, 2);
|
||||
successor.extend(journal_record(1, "fsync-kill-bucket", "first-object", None, 0));
|
||||
let successor = journal_record(1, "fsync-kill-bucket", "second-object", None, 2);
|
||||
write_journal_path_to_disks_synced(&disk_paths, SCOPED_JOURNAL_REL, &successor);
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &successor)
|
||||
@@ -815,8 +665,9 @@ async fn journal_replay_retains_child_process_anchor_when_manager_is_full() {
|
||||
);
|
||||
}
|
||||
|
||||
/// A successor flush must preserve both pending work and accepted work whose
|
||||
/// repair has not been proven when the process restarts.
|
||||
/// If a process crashes after flushing a smaller successor snapshot but before
|
||||
/// deleting the startup anchor, the restarted process must replay the
|
||||
/// successor tail rather than losing it or merging it with stale records.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn journal_replay_survives_successor_flush_before_delete() {
|
||||
@@ -831,8 +682,7 @@ async fn journal_replay_survives_successor_flush_before_delete() {
|
||||
assert_eq!(status.code(), Some(78), "child process did not reach the successor flush boundary");
|
||||
|
||||
let (disk_paths, storage) = heal_env_at(Some(temp_dir.path())).await;
|
||||
let mut expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||
expected_successor.extend(journal_record(1, "successor-bucket", "first-object", None, 0));
|
||||
let expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor),
|
||||
"restarted process must see the pending successor snapshot"
|
||||
@@ -840,24 +690,24 @@ async fn journal_replay_survives_successor_flush_before_delete() {
|
||||
|
||||
let restarted = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(replayed, 2, "restart must replay both the admitted and pending responsibilities");
|
||||
assert_eq!(replayed, 1, "restart after successor flush must replay only the still-pending tail");
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
2,
|
||||
"both unproven successor responsibilities must be accepted after restart"
|
||||
1,
|
||||
"the successor tail must be accepted after restart"
|
||||
);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"the accepted successor remains anchored until a verified repair proof"
|
||||
"a fully consumed successor snapshot may be deleted after restart replay"
|
||||
);
|
||||
}
|
||||
|
||||
/// A service-style hard kill after successor flush must be equivalent to a
|
||||
/// crash at the flush-before-delete boundary: restart must recover every
|
||||
/// unproven responsibility from the successor snapshot.
|
||||
/// crash at the flush-before-delete boundary: restart may replay the smaller
|
||||
/// successor snapshot, but must not lose or merge stale startup records.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
#[cfg(unix)]
|
||||
@@ -885,8 +735,7 @@ async fn journal_replay_survives_service_kill_after_successor_flush() {
|
||||
assert!(!status.success(), "child fixture must be terminated instead of exiting cleanly");
|
||||
|
||||
let (disk_paths, storage) = heal_env_at(Some(temp_dir.path())).await;
|
||||
let mut expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||
expected_successor.extend(journal_record(1, "service-kill-bucket", "first-object", None, 0));
|
||||
let expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor),
|
||||
"restarted process must see the successor snapshot produced before the kill"
|
||||
@@ -894,25 +743,25 @@ async fn journal_replay_survives_service_kill_after_successor_flush() {
|
||||
|
||||
let restarted = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(replayed, 2, "service-kill restart must preserve every unproven responsibility");
|
||||
assert_eq!(replayed, 1, "restart after service kill must replay only the still-pending tail");
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
2,
|
||||
"both unproven responsibilities must be accepted after service kill restart"
|
||||
1,
|
||||
"the successor tail must be accepted after service kill restart"
|
||||
);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"the accepted successor remains anchored until a verified repair proof after service-kill restart"
|
||||
"a fully consumed successor snapshot may be deleted after service-kill restart replay"
|
||||
);
|
||||
}
|
||||
|
||||
/// A hard kill between the authoritative successor fsync and the legacy mirror
|
||||
/// rewrite must prefer the canonical successor over the stale legacy startup
|
||||
/// epoch while retaining every unproven responsibility. New readers must not
|
||||
/// merge epochs, while the old mirror remains crash-visible.
|
||||
/// rewrite must prefer the canonical successor tail over the stale legacy
|
||||
/// startup epoch. This models the mixed-version boundary conservatively: new
|
||||
/// readers must not merge epochs, while the old mirror remains crash-visible.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
#[cfg(unix)]
|
||||
@@ -940,8 +789,7 @@ async fn journal_replay_survives_sigkill_after_authoritative_successor_fsync_bef
|
||||
assert!(!status.success(), "child fixture must be terminated instead of exiting cleanly");
|
||||
|
||||
let (disk_paths, storage) = heal_env_at(Some(temp_dir.path())).await;
|
||||
let mut expected_successor = journal_record(1, "fsync-kill-bucket", "second-object", None, 2);
|
||||
expected_successor.extend(journal_record(1, "fsync-kill-bucket", "first-object", None, 0));
|
||||
let expected_successor = journal_record(1, "fsync-kill-bucket", "second-object", None, 2);
|
||||
let stale_startup = {
|
||||
let mut startup = journal_record(1, "fsync-kill-bucket", "first-object", None, 0);
|
||||
startup.extend(journal_record(1, "fsync-kill-bucket", "second-object", None, 0));
|
||||
@@ -958,17 +806,17 @@ async fn journal_replay_survives_sigkill_after_authoritative_successor_fsync_bef
|
||||
|
||||
let restarted = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(replayed, 2, "new reader must recover every responsibility in the authoritative successor");
|
||||
assert_eq!(replayed, 1, "new reader must replay only the authoritative successor tail");
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
2,
|
||||
"both responsibilities must be accepted after the fsync-boundary restart"
|
||||
1,
|
||||
"the successor tail must be accepted after the fsync-boundary restart"
|
||||
);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"the accepted authoritative successor remains anchored until a verified repair proof"
|
||||
"a fully consumed authoritative successor may clean both epochs after restart replay"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2098,7 +2098,11 @@ impl KmsBackend for LocalKmsBackend {
|
||||
let _write_guard = self.client.lock_key_for_write(key_id).await;
|
||||
|
||||
// First, load the key from disk to get the master key
|
||||
let mut master_key = self.client.load_master_key(key_id).await?;
|
||||
let mut master_key = self
|
||||
.client
|
||||
.load_master_key(key_id)
|
||||
.await
|
||||
.map_err(|_| KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
|
||||
let (deletion_date_str, deletion_date_dt) = if request.force_immediate.unwrap_or(false) {
|
||||
// Tombstone first: mark the record Deleted before removing the
|
||||
@@ -2201,7 +2205,11 @@ impl KmsBackend for LocalKmsBackend {
|
||||
let _write_guard = self.client.lock_key_for_write(key_id).await;
|
||||
|
||||
// Load the key from disk to get the master key
|
||||
let mut master_key = self.client.load_master_key(key_id).await?;
|
||||
let mut master_key = self
|
||||
.client
|
||||
.load_master_key(key_id)
|
||||
.await
|
||||
.map_err(|_| KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
|
||||
if master_key.status != KeyStatus::PendingDeletion {
|
||||
return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion")));
|
||||
@@ -3030,111 +3038,6 @@ mod tests {
|
||||
assert!(matches!(error, KmsError::InvalidKey { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_key_preserves_directory_io_error() {
|
||||
let (client, temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("existing-key", "AES_256", None).await.expect("create key");
|
||||
let backend = LocalKmsBackend { client };
|
||||
let offline_dir = TempDir::new().expect("create offline directory");
|
||||
let offline_key_dir = offline_dir.path().join("keys");
|
||||
fs::rename(temp_dir.path(), &offline_key_dir)
|
||||
.await
|
||||
.expect("move key directory offline");
|
||||
fs::write(temp_dir.path(), b"not a directory")
|
||||
.await
|
||||
.expect("replace key directory with a file");
|
||||
|
||||
let error = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "existing-key".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("unreadable storage must prevent scheduling deletion");
|
||||
|
||||
fs::remove_file(temp_dir.path()).await.expect("remove replacement file");
|
||||
fs::rename(&offline_key_dir, temp_dir.path())
|
||||
.await
|
||||
.expect("restore key directory");
|
||||
assert!(matches!(error, KmsError::IoError { .. }), "got {error:?}");
|
||||
let key = backend
|
||||
.client
|
||||
.load_master_key("existing-key")
|
||||
.await
|
||||
.expect("read retained key");
|
||||
assert_eq!(key.status, KeyStatus::Active, "failed deletion must not mutate key state");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_key_deletion_preserves_directory_io_error() {
|
||||
let (client, temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("existing-key", "AES_256", None).await.expect("create key");
|
||||
let backend = LocalKmsBackend { client };
|
||||
backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "existing-key".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("schedule key deletion");
|
||||
let offline_dir = TempDir::new().expect("create offline directory");
|
||||
let offline_key_dir = offline_dir.path().join("keys");
|
||||
fs::rename(temp_dir.path(), &offline_key_dir)
|
||||
.await
|
||||
.expect("move key directory offline");
|
||||
fs::write(temp_dir.path(), b"not a directory")
|
||||
.await
|
||||
.expect("replace key directory with a file");
|
||||
|
||||
let error = backend
|
||||
.cancel_key_deletion(CancelKeyDeletionRequest {
|
||||
key_id: "existing-key".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect_err("unreadable storage must prevent cancelling deletion");
|
||||
|
||||
fs::remove_file(temp_dir.path()).await.expect("remove replacement file");
|
||||
fs::rename(&offline_key_dir, temp_dir.path())
|
||||
.await
|
||||
.expect("restore key directory");
|
||||
assert!(matches!(error, KmsError::IoError { .. }), "got {error:?}");
|
||||
let key = backend
|
||||
.client
|
||||
.load_master_key("existing-key")
|
||||
.await
|
||||
.expect("read retained key");
|
||||
assert_eq!(
|
||||
key.status,
|
||||
KeyStatus::PendingDeletion,
|
||||
"failed cancellation must retain the deletion state"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_master_key_directory_replaced_by_file_is_io_error() {
|
||||
let (client, temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("existing-key", "AES_256", None).await.expect("create key");
|
||||
let offline_dir = TempDir::new().expect("create offline directory");
|
||||
let offline_key_dir = offline_dir.path().join("keys");
|
||||
fs::rename(temp_dir.path(), &offline_key_dir)
|
||||
.await
|
||||
.expect("move key directory offline");
|
||||
fs::write(temp_dir.path(), b"not a directory")
|
||||
.await
|
||||
.expect("replace key directory with a file");
|
||||
|
||||
let error = client
|
||||
.load_master_key("existing-key")
|
||||
.await
|
||||
.expect_err("a file in place of the key directory must fail");
|
||||
|
||||
fs::remove_file(temp_dir.path()).await.expect("remove replacement file");
|
||||
fs::rename(&offline_key_dir, temp_dir.path())
|
||||
.await
|
||||
.expect("restore key directory");
|
||||
assert!(matches!(error, KmsError::IoError { .. }), "got {error:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_master_key_accepts_legacy_rfc3339_timestamp() {
|
||||
let (client, _temp_dir) = create_dev_mode_client().await;
|
||||
|
||||
@@ -577,13 +577,16 @@ impl KmsServiceManager {
|
||||
Some(service_version.probe_worker.as_ref()?.status())
|
||||
}
|
||||
|
||||
/// Health check for the KMS service
|
||||
/// Check backend health without changing the service lifecycle state.
|
||||
///
|
||||
/// A transient backend failure leaves the published service available for
|
||||
/// subsequent checks and operations. Readiness uses the background probe
|
||||
/// to evaluate backend availability independently of lifecycle state.
|
||||
pub async fn health_check(&self) -> Result<bool> {
|
||||
let checked_state = self.state.load_full();
|
||||
match checked_state.current_service.as_ref() {
|
||||
Some(service_version) => {
|
||||
let manager = service_version.manager.clone();
|
||||
let checked_version = service_version.version;
|
||||
// Perform health check on the backend
|
||||
match manager.health_check().await {
|
||||
Ok(healthy) => {
|
||||
@@ -594,8 +597,6 @@ impl KmsServiceManager {
|
||||
}
|
||||
Err(e) => {
|
||||
error!("KMS health check error: {}", e);
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
self.mark_health_error_if_current(checked_version, &e);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
@@ -739,17 +740,6 @@ impl KmsServiceManager {
|
||||
task: std::sync::Mutex::new(Some(task)),
|
||||
}))
|
||||
}
|
||||
|
||||
fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) {
|
||||
let current = self.state.load_full();
|
||||
if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) {
|
||||
self.state.store(Arc::new(RuntimeState {
|
||||
config: current.config.clone(),
|
||||
status: KmsServiceStatus::Error(format!("Health check failed: {error}")),
|
||||
current_service: current.current_service.clone(),
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for KmsServiceManager {
|
||||
@@ -1004,19 +994,6 @@ mod tests {
|
||||
assert!(manager.get_service_version().await.expect("restarted version") > first_version);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_health_failure_cannot_poison_new_service_status() {
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
|
||||
manager.start().await.expect("start");
|
||||
let old_version = manager.get_service_version().await.expect("old version");
|
||||
manager.restart().await.expect("restart");
|
||||
|
||||
manager.mark_health_error_if_current(old_version, &KmsError::backend_error("stale failure"));
|
||||
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forbidden_local_master_key_change_preserves_running_config_and_service() {
|
||||
use crate::types::{CreateKeyRequest, KeyUsage};
|
||||
|
||||
@@ -75,6 +75,44 @@ fn unreachable_vault_config() -> KmsConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_health_failure_does_not_latch_the_service_status() {
|
||||
let kms = TestKms::local().await;
|
||||
let manager = kms.manager();
|
||||
let service = manager.get_encryption_service().await.expect("running service");
|
||||
let version = manager.get_service_version().await.expect("running version");
|
||||
assert!(manager.health_check().await.expect("initial backend health"));
|
||||
|
||||
// Move only this test's keys out of reach, then restore the same backend.
|
||||
let key_dir = kms.key_dir().expect("local key directory");
|
||||
let outage = tempfile::TempDir::new().expect("temporary outage directory");
|
||||
let hidden_keys = outage.path().join("keys");
|
||||
tokio::fs::rename(&key_dir, &hidden_keys)
|
||||
.await
|
||||
.expect("make backend unavailable");
|
||||
let failure = manager.health_check().await;
|
||||
let outage_status = manager.get_status().await;
|
||||
tokio::fs::rename(&hidden_keys, &key_dir).await.expect("restore backend");
|
||||
|
||||
assert!(failure.is_err(), "the outage must surface as a health-check error");
|
||||
assert!(manager.health_check().await.expect("backend recovers without restart"));
|
||||
assert!(Arc::ptr_eq(
|
||||
&service,
|
||||
&manager.get_encryption_service().await.expect("service survives the outage")
|
||||
));
|
||||
assert_eq!(manager.get_service_version().await, Some(version));
|
||||
assert_eq!(
|
||||
manager.get_status().await,
|
||||
KmsServiceStatus::Running,
|
||||
"a recovered backend must not leave service-status and readiness latched in Error"
|
||||
);
|
||||
assert_eq!(
|
||||
outage_status,
|
||||
KmsServiceStatus::Running,
|
||||
"backend health does not change the running service's lifecycle state"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn starting_against_an_unreachable_backend_fails_without_publishing_a_service() {
|
||||
let manager = KmsServiceManager::new();
|
||||
|
||||
@@ -263,14 +263,6 @@ pub struct ScannerUsageRecoveryIntentResponse {
|
||||
pub mode: String,
|
||||
pub intent_id: String,
|
||||
pub state: String,
|
||||
#[serde(default)]
|
||||
pub actor_sha256: Option<String>,
|
||||
#[serde(default)]
|
||||
pub idempotency_key_sha256: Option<String>,
|
||||
#[serde(default)]
|
||||
pub request_sha256: Option<String>,
|
||||
#[serde(default)]
|
||||
pub accepted_at_unix_secs: Option<u64>,
|
||||
#[serde(flatten)]
|
||||
pub extra: serde_json::Map<String, serde_json::Value>,
|
||||
}
|
||||
@@ -877,10 +869,6 @@ mod tests {
|
||||
"mode": "full-rebuild",
|
||||
"intent_id": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"state": "accepted",
|
||||
"actor_sha256": "1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"idempotency_key_sha256": "2222222222222222222222222222222222222222222222222222222222222222",
|
||||
"request_sha256": "3333333333333333333333333333333333333333333333333333333333333333",
|
||||
"accepted_at_unix_secs": 7,
|
||||
"future": {"worker": "pending"}
|
||||
}))
|
||||
.unwrap();
|
||||
@@ -889,33 +877,7 @@ mod tests {
|
||||
assert_eq!(intent.mode, "full-rebuild");
|
||||
assert_eq!(intent.intent_id, "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef");
|
||||
assert_eq!(intent.state, "accepted");
|
||||
assert_eq!(
|
||||
intent.actor_sha256.as_deref(),
|
||||
Some("1111111111111111111111111111111111111111111111111111111111111111")
|
||||
);
|
||||
assert_eq!(
|
||||
intent.idempotency_key_sha256.as_deref(),
|
||||
Some("2222222222222222222222222222222222222222222222222222222222222222")
|
||||
);
|
||||
assert_eq!(
|
||||
intent.request_sha256.as_deref(),
|
||||
Some("3333333333333333333333333333333333333333333333333333333333333333")
|
||||
);
|
||||
assert_eq!(intent.accepted_at_unix_secs, Some(7));
|
||||
assert_eq!(intent.extra["future"]["worker"], "pending");
|
||||
|
||||
let legacy_intent: ScannerUsageRecoveryIntentResponse = serde_json::from_value(json!({
|
||||
"status": "accepted",
|
||||
"action": "usage-full-rebuild",
|
||||
"mode": "full-rebuild",
|
||||
"intent_id": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"state": "accepted"
|
||||
}))
|
||||
.unwrap();
|
||||
assert!(legacy_intent.actor_sha256.is_none());
|
||||
assert!(legacy_intent.idempotency_key_sha256.is_none());
|
||||
assert!(legacy_intent.request_sha256.is_none());
|
||||
assert!(legacy_intent.accepted_at_unix_secs.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1009,7 +971,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn scanner_usage_async_reset_posts_explicit_intent_contract() {
|
||||
let server = TestServer::spawn(
|
||||
r#"{"status":"accepted","action":"usage-full-rebuild","mode":"full-rebuild","intent_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"accepted","actor_sha256":"1111111111111111111111111111111111111111111111111111111111111111","idempotency_key_sha256":"2222222222222222222222222222222222222222222222222222222222222222","request_sha256":"3333333333333333333333333333333333333333333333333333333333333333","accepted_at_unix_secs":7}"#,
|
||||
r#"{"status":"accepted","action":"usage-full-rebuild","mode":"full-rebuild","intent_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","state":"accepted"}"#,
|
||||
202,
|
||||
)
|
||||
.await;
|
||||
@@ -1023,7 +985,6 @@ mod tests {
|
||||
assert_eq!(accepted.status, "accepted");
|
||||
assert_eq!(accepted.mode, "full-rebuild");
|
||||
assert_eq!(accepted.state, "accepted");
|
||||
assert_eq!(accepted.accepted_at_unix_secs, Some(7));
|
||||
let request = server.recorded();
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/scanner/usage-state/reset");
|
||||
|
||||
@@ -166,6 +166,24 @@ fn rule_replicates(rule: &ReplicationRule, obj: &ObjectOpts) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn replication_filter_tags_match(filter: &s3s::dto::ReplicationRuleFilter, object_tags: &HashMap<String, String>) -> bool {
|
||||
let tag_matches = |tag: &s3s::dto::Tag| match (&tag.key, &tag.value) {
|
||||
(None, None) => true,
|
||||
(Some(key), _) if key.is_empty() => true,
|
||||
(Some(key), Some(value)) => object_tags.get(key) == Some(value),
|
||||
_ => false,
|
||||
};
|
||||
|
||||
filter
|
||||
.and
|
||||
.as_ref()
|
||||
.and_then(|and| and.tags.as_deref())
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.chain(filter.tag.iter())
|
||||
.all(tag_matches)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReplicationTargetValidationError {
|
||||
RoleWithMultipleDestinations,
|
||||
@@ -704,7 +722,7 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
|
||||
if let Some(filter) = &rule.filter {
|
||||
let object_tags = ReplicationTagFilter::decode_tags_to_map(&obj.user_tags);
|
||||
if filter.test_tags(&object_tags) {
|
||||
if replication_filter_tags_match(filter, &object_tags) {
|
||||
rules.push(rule.clone());
|
||||
}
|
||||
} else {
|
||||
@@ -1139,6 +1157,47 @@ mod tests {
|
||||
assert_eq!(validate_replication_config_structure(&structure_config(vec![rule])), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actionable_rules_require_every_and_tag_to_match() {
|
||||
let mut rule = replication_rule("rule-1", "arn:target:a");
|
||||
rule.filter = Some(s3s::dto::ReplicationRuleFilter {
|
||||
and: Some(s3s::dto::ReplicationRuleAndOperator {
|
||||
prefix: None,
|
||||
tags: Some(vec![
|
||||
s3s::dto::Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: Some("prod".to_string()),
|
||||
},
|
||||
s3s::dto::Tag {
|
||||
key: Some("tier".to_string()),
|
||||
value: Some("gold".to_string()),
|
||||
},
|
||||
]),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
let config = structure_config(vec![rule]);
|
||||
let object = |user_tags: &str| ObjectOpts {
|
||||
name: "object".to_string(),
|
||||
user_tags: user_tags.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(config.filter_target_arns(&object("env=prod")).is_empty());
|
||||
assert_eq!(config.filter_target_arns(&object("env=prod&tier=gold")), vec!["arn:target:a"]);
|
||||
assert!(config.filter_target_arns(&object("")).is_empty());
|
||||
|
||||
let mut malformed = config;
|
||||
malformed.rules[0].filter.as_mut().unwrap().and.as_mut().unwrap().tags = Some(vec![s3s::dto::Tag {
|
||||
key: Some("env".to_string()),
|
||||
value: None,
|
||||
}]);
|
||||
assert!(
|
||||
malformed.filter_target_arns(&object("env=prod")).is_empty(),
|
||||
"a malformed tag filter must fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structure_validation_allows_tag_filter_when_delete_marker_replication_disabled() {
|
||||
let mut rule = replication_rule("rule-1", "arn:target:a");
|
||||
|
||||
@@ -580,6 +580,30 @@ impl FailStats {
|
||||
FailedMetric { count, size }
|
||||
}
|
||||
|
||||
/// Both rolling windows from one walk of the samples. `short` must be the
|
||||
/// narrower window; the walk stops at `long`. Callers that need both (the
|
||||
/// per-node site snapshot) would otherwise scan the deque twice while
|
||||
/// holding the bucket-stats read lock, and the deque is only bounded by
|
||||
/// the one-hour window - an unreachable target under load fills it.
|
||||
pub fn recent_windows(&self, short: Duration, long: Duration) -> (FailedMetric, FailedMetric) {
|
||||
let now = Instant::now();
|
||||
let mut short_metric = FailedMetric::default();
|
||||
let mut long_metric = FailedMetric::default();
|
||||
for sample in self.recent.iter().rev() {
|
||||
let age = now.duration_since(sample.observed_at);
|
||||
if age > long {
|
||||
break;
|
||||
}
|
||||
if age <= short {
|
||||
short_metric.count += 1;
|
||||
short_metric.size += sample.size;
|
||||
}
|
||||
long_metric.count += 1;
|
||||
long_metric.size += sample.size;
|
||||
}
|
||||
(short_metric, long_metric)
|
||||
}
|
||||
|
||||
pub fn merge(&self, other: &FailStats) -> Self {
|
||||
Self {
|
||||
count: self.count.saturating_add(other.count),
|
||||
@@ -912,6 +936,26 @@ mod tests {
|
||||
assert_eq!(last_hour.size, 96);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_stats_recent_windows_matches_two_separate_scans() {
|
||||
let mut stats = FailStats::default();
|
||||
stats.add_size(64, None::<&()>);
|
||||
stats.add_size(32, None::<&()>);
|
||||
|
||||
let (minute, hour) = stats.recent_windows(Duration::from_secs(60), Duration::from_secs(60 * 60));
|
||||
let expected_minute = stats.recent_since(Duration::from_secs(60));
|
||||
let expected_hour = stats.recent_since(Duration::from_secs(60 * 60));
|
||||
|
||||
assert_eq!((minute.count, minute.size), (expected_minute.count, expected_minute.size));
|
||||
assert_eq!((hour.count, hour.size), (expected_hour.count, expected_hour.size));
|
||||
assert_eq!(minute.count, 2);
|
||||
assert_eq!(hour.size, 96);
|
||||
|
||||
let empty = FailStats::default();
|
||||
let (minute, hour) = empty.recent_windows(Duration::from_secs(60), Duration::from_secs(60 * 60));
|
||||
assert_eq!((minute.count, minute.size, hour.count, hour.size), (0, 0, 0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_stats_saturate_instead_of_wrapping() {
|
||||
let mut stats = FailStats {
|
||||
|
||||
@@ -97,11 +97,9 @@ pub use scanner::{
|
||||
pub use scanner_io::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_dirty_usage_bucket_from_producer, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot,
|
||||
scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
};
|
||||
pub use segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
|
||||
@@ -3,8 +3,7 @@
|
||||
use super::*;
|
||||
use crate::segment_invalidation::{
|
||||
MAX_SEGMENT_INVALIDATION_BYTES, MAX_SEGMENT_INVALIDATION_ENTRIES, SegmentInvalidationDomain, SegmentInvalidationEnvelope,
|
||||
SegmentInvalidationError, SegmentInvalidationProducer, SegmentInvalidationProducerIdentity, SegmentInvalidationProof,
|
||||
admit_segment_invalidation, complete_segment_invalidation_producers,
|
||||
SegmentInvalidationError, SegmentInvalidationProducer, SegmentInvalidationProof, admit_segment_invalidation,
|
||||
};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
@@ -12,8 +11,7 @@ const MAX_WALK_SAMPLES: usize = 32;
|
||||
const MAX_WALK_BYTES: usize = 1024;
|
||||
|
||||
fn segment_producers() -> BTreeSet<SegmentInvalidationProducer> {
|
||||
complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION)
|
||||
.expect("fixture should enumerate the complete production producer matrix")
|
||||
SegmentInvalidationProducer::REQUIRED.into_iter().collect()
|
||||
}
|
||||
|
||||
fn segment_envelope() -> SegmentInvalidationEnvelope {
|
||||
|
||||
@@ -39,7 +39,7 @@ use s3s::dto::{
|
||||
BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration, VersioningConfiguration,
|
||||
};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::collections::{BTreeSet, HashMap, HashSet};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::future::Future;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
@@ -1224,9 +1224,8 @@ pub(crate) use cache::{
|
||||
pub use dirty_usage::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
|
||||
acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
|
||||
record_dirty_usage_bucket_from_producer, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||
record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation,
|
||||
record_dirty_usage_object, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot,
|
||||
scanner_dirty_usage_state, scanner_maintenance_generation,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
||||
|
||||
@@ -22,11 +22,6 @@ pub(super) static DIRTY_USAGE_BUCKETS: LazyLock<StdMutex<DirtyUsageBuckets>> = L
|
||||
// matching scope.
|
||||
pub(super) static DIRTY_USAGE_BUCKET_SCOPES: LazyLock<StdMutex<DirtyUsageBucketScopes>> =
|
||||
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
// Non-authoritative process-local producer coverage. Any future segment reuse
|
||||
// activation must bind this to the exact generation window and durable proof.
|
||||
pub(super) static DIRTY_USAGE_PRODUCER_IDENTITIES: LazyLock<
|
||||
StdMutex<BTreeSet<crate::segment_invalidation::SegmentInvalidationProducerIdentity>>,
|
||||
> = LazyLock::new(|| StdMutex::new(BTreeSet::new()));
|
||||
pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||
pub(super) static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
@@ -158,7 +153,6 @@ fn apply_scoped_dirty_usage_ack(
|
||||
#[cfg(test)]
|
||||
mod scoped_dirty_usage_tests {
|
||||
use super::*;
|
||||
use crate::segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||
|
||||
#[test]
|
||||
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
|
||||
@@ -219,30 +213,6 @@ mod scoped_dirty_usage_tests {
|
||||
assert_eq!(scopes, original_scopes);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dirty_usage_tracks_known_segment_producer_identities_without_authorizing_unknown_sources() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
record_dirty_usage_object_from_producer("photos", "hot/object", SegmentInvalidationProducerIdentity::PutObject);
|
||||
record_dirty_usage_object_from_producer("photos", "archive/object", SegmentInvalidationProducerIdentity::DeleteObject);
|
||||
record_dirty_usage_bucket_from_producer("photos", SegmentInvalidationProducerIdentity::Unknown);
|
||||
|
||||
assert_eq!(
|
||||
dirty_usage_producer_identities_for_tests(),
|
||||
BTreeSet::from([
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteObject
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
dirty_usage_bucket_scopes_for_tests().get("photos"),
|
||||
Some(&DirtyUsageBucketScope::WholeBucket),
|
||||
"an unknown producer keeps the bucket dirty but must not count as producer coverage"
|
||||
);
|
||||
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
assert!(dirty_usage_producer_identities_for_tests().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
|
||||
@@ -255,13 +225,6 @@ fn dirty_usage_bucket_scopes() -> MutexGuard<'static, DirtyUsageBucketScopes> {
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
fn dirty_usage_producer_identities()
|
||||
-> MutexGuard<'static, BTreeSet<crate::segment_invalidation::SegmentInvalidationProducerIdentity>> {
|
||||
DIRTY_USAGE_PRODUCER_IDENTITIES
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
pub(super) fn usize_to_u64_saturated(value: usize) -> u64 {
|
||||
u64::try_from(value).unwrap_or(u64::MAX)
|
||||
}
|
||||
@@ -277,22 +240,6 @@ pub fn record_dirty_usage_bucket(bucket: &str) {
|
||||
return;
|
||||
}
|
||||
|
||||
record_dirty_usage_bucket_inner(bucket);
|
||||
}
|
||||
|
||||
pub fn record_dirty_usage_bucket_from_producer(
|
||||
bucket: &str,
|
||||
producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity,
|
||||
) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
record_segment_invalidation_producer_identity(producer);
|
||||
record_dirty_usage_bucket_inner(bucket);
|
||||
}
|
||||
|
||||
fn record_dirty_usage_bucket_inner(bucket: &str) {
|
||||
let pending_buckets = {
|
||||
let mut dirty_buckets = dirty_usage_buckets();
|
||||
let mut dirty_scopes = dirty_usage_bucket_scopes();
|
||||
@@ -316,23 +263,6 @@ fn record_dirty_usage_bucket_inner(bucket: &str) {
|
||||
/// local: after restart or any unverified distributed path the scanner falls
|
||||
/// back to its ordinary bucket scan.
|
||||
pub fn record_dirty_usage_object(bucket: &str, object: &str) {
|
||||
record_dirty_usage_object_inner(bucket, object);
|
||||
}
|
||||
|
||||
pub fn record_dirty_usage_object_from_producer(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity,
|
||||
) {
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
record_segment_invalidation_producer_identity(producer);
|
||||
record_dirty_usage_object_inner(bucket, object);
|
||||
}
|
||||
|
||||
fn record_dirty_usage_object_inner(bucket: &str, object: &str) {
|
||||
let Some(top_level_entry) = dirty_usage_top_level_entry(object) else {
|
||||
record_dirty_usage_bucket(bucket);
|
||||
return;
|
||||
@@ -366,17 +296,6 @@ fn record_dirty_usage_object_inner(bucket: &str, object: &str) {
|
||||
DIRTY_USAGE_BUCKET_NOTIFY.notify_one();
|
||||
}
|
||||
|
||||
fn record_segment_invalidation_producer_identity(producer: crate::segment_invalidation::SegmentInvalidationProducerIdentity) {
|
||||
if producer.producer().is_some() {
|
||||
dirty_usage_producer_identities().insert(producer);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn dirty_usage_producer_identities_for_tests() -> BTreeSet<crate::segment_invalidation::SegmentInvalidationProducerIdentity> {
|
||||
dirty_usage_producer_identities().clone()
|
||||
}
|
||||
|
||||
fn dirty_usage_top_level_entry(object: &str) -> Option<String> {
|
||||
let (top_level_entry, _) = object.split_once('/').unwrap_or((object, ""));
|
||||
(!top_level_entry.is_empty()
|
||||
@@ -658,7 +577,6 @@ pub(super) fn dirty_usage_bucket_count() -> usize {
|
||||
pub(crate) fn clear_dirty_usage_buckets_for_tests() {
|
||||
dirty_usage_buckets().clear();
|
||||
dirty_usage_bucket_scopes().clear();
|
||||
dirty_usage_producer_identities().clear();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -16,29 +16,23 @@ use super::*;
|
||||
use crate::data_usage_define::{DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision};
|
||||
|
||||
async fn create_cohort_bucket(store: &ECStore, bucket: &str) {
|
||||
create_cohort_bucket_objects(store, bucket, 1).await;
|
||||
}
|
||||
|
||||
async fn create_cohort_bucket_objects(store: &ECStore, bucket: &str, objects: usize) {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket");
|
||||
for set in store.all_set_disks() {
|
||||
for index in 0..objects {
|
||||
let mut reader = ScannerPutObjReader::from_vec(b"cohort".to_vec());
|
||||
set.put_object(
|
||||
bucket,
|
||||
&format!("object-{index:04}"),
|
||||
&mut reader,
|
||||
&ScannerObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("fixture object and all rename tails should persist");
|
||||
}
|
||||
let mut reader = ScannerPutObjReader::from_vec(b"cohort".to_vec());
|
||||
set.put_object(
|
||||
bucket,
|
||||
"initial",
|
||||
&mut reader,
|
||||
&ScannerObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("fixture object and all rename tails should persist");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,92 +168,6 @@ async fn service_cohort_production_dispatch_services_waiters_across_sources() {
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn service_cohort_flat_bucket_budget_does_not_publish_unscanned_small_bucket() {
|
||||
let (_dir, store) = setup_two_pool_scanner_store().await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let flat = format!("a-flat-{}", Uuid::new_v4().simple());
|
||||
let small = format!("z-small-{}", Uuid::new_v4().simple());
|
||||
create_cohort_bucket_objects(&store, &flat, 6).await;
|
||||
create_cohort_bucket(&store, &small).await;
|
||||
let cohort = Arc::new(StdMutex::new(ScannerServiceCohort::default()));
|
||||
|
||||
let expected_flat = store
|
||||
.all_set_disks()
|
||||
.iter()
|
||||
.map(|set| (DataUsageCacheSource::new(set.pool_index, set.set_index), flat.clone()))
|
||||
.collect::<HashSet<_>>();
|
||||
let expected_small = store
|
||||
.all_set_disks()
|
||||
.iter()
|
||||
.map(|set| (DataUsageCacheSource::new(set.pool_index, set.set_index), small.clone()))
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let (result, usage) = run_cohort_cycle(&store, cohort.clone(), 1, budget.clone()).await;
|
||||
assert_eq!(result.status, ScannerCycleStatus::Incomplete);
|
||||
assert!(usage.is_none(), "wide-bucket budget exhaustion must not publish a partial aggregate");
|
||||
assert!(budget.budget_elapsed());
|
||||
let first_round_admitted = cohort
|
||||
.lock()
|
||||
.expect("cohort lock")
|
||||
.admitted_members()
|
||||
.into_iter()
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(
|
||||
!first_round_admitted.is_disjoint(&expected_flat),
|
||||
"the first fixed budget round should exercise the wide flat bucket"
|
||||
);
|
||||
assert!(
|
||||
first_round_admitted.is_disjoint(&expected_small),
|
||||
"a small bucket not yet reached by the real scanner must not be marked admitted"
|
||||
);
|
||||
|
||||
for cycle in 2..=4 {
|
||||
let ctx = CancellationToken::new();
|
||||
let (result, usage) = run_cohort_cycle(
|
||||
&store,
|
||||
cohort.clone(),
|
||||
cycle,
|
||||
ScannerCycleBudget::new_with_progress_tracking(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.status, ScannerCycleStatus::Incomplete);
|
||||
assert!(usage.is_none(), "mixed partial coverage still cannot publish the set root");
|
||||
}
|
||||
let admitted_after_budgeted_rounds = cohort
|
||||
.lock()
|
||||
.expect("cohort lock")
|
||||
.admitted_members()
|
||||
.into_iter()
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(
|
||||
expected_small.is_subset(&admitted_after_budgeted_rounds),
|
||||
"tracked small buckets must receive real execution opportunities within their fixed service-round bound"
|
||||
);
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let (result, usage) =
|
||||
run_cohort_cycle(&store, cohort, 5, ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default())).await;
|
||||
assert_eq!(result.status, ScannerCycleStatus::Complete);
|
||||
assert_eq!(usage.expect("final complete aggregate").objects_total_count, 14);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn service_cohort_fresh_complete_aggregate_preserves_reordered_sources() {
|
||||
|
||||
@@ -25,7 +25,6 @@ pub enum SegmentInvalidationError {
|
||||
ByteLimit,
|
||||
InvalidProof,
|
||||
InvalidKey,
|
||||
UnknownProducer,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
@@ -51,74 +50,6 @@ impl SegmentInvalidationProducer {
|
||||
];
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum SegmentInvalidationProducerIdentity {
|
||||
PutObject,
|
||||
DeleteObject,
|
||||
DeleteMarker,
|
||||
CompleteMultipartUpload,
|
||||
Replication,
|
||||
TierTransition,
|
||||
TierExpiration,
|
||||
DirectoryObject,
|
||||
Unknown,
|
||||
TestFixture,
|
||||
}
|
||||
|
||||
impl SegmentInvalidationProducerIdentity {
|
||||
pub const REQUIRED_PRODUCTION: [Self; 8] = [
|
||||
Self::PutObject,
|
||||
Self::DeleteObject,
|
||||
Self::DeleteMarker,
|
||||
Self::CompleteMultipartUpload,
|
||||
Self::Replication,
|
||||
Self::TierTransition,
|
||||
Self::TierExpiration,
|
||||
Self::DirectoryObject,
|
||||
];
|
||||
|
||||
pub fn producer(self) -> Option<SegmentInvalidationProducer> {
|
||||
match self {
|
||||
Self::PutObject => Some(SegmentInvalidationProducer::Put),
|
||||
Self::DeleteObject => Some(SegmentInvalidationProducer::Delete),
|
||||
Self::DeleteMarker => Some(SegmentInvalidationProducer::DeleteMarker),
|
||||
Self::CompleteMultipartUpload => Some(SegmentInvalidationProducer::Multipart),
|
||||
Self::Replication => Some(SegmentInvalidationProducer::Replication),
|
||||
Self::TierTransition | Self::TierExpiration => Some(SegmentInvalidationProducer::Tier),
|
||||
Self::DirectoryObject => Some(SegmentInvalidationProducer::DirectoryObject),
|
||||
Self::Unknown | Self::TestFixture => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn complete_segment_invalidation_producers<I>(
|
||||
identities: I,
|
||||
) -> Result<BTreeSet<SegmentInvalidationProducer>, SegmentInvalidationError>
|
||||
where
|
||||
I: IntoIterator<Item = SegmentInvalidationProducerIdentity>,
|
||||
{
|
||||
let mut covered_identities = BTreeSet::new();
|
||||
let mut producers = BTreeSet::new();
|
||||
for identity in identities {
|
||||
let Some(producer) = identity.producer() else {
|
||||
return Err(SegmentInvalidationError::UnknownProducer);
|
||||
};
|
||||
covered_identities.insert(identity);
|
||||
producers.insert(producer);
|
||||
}
|
||||
if SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION
|
||||
.iter()
|
||||
.all(|identity| covered_identities.contains(identity))
|
||||
&& SegmentInvalidationProducer::REQUIRED
|
||||
.iter()
|
||||
.all(|producer| producers.contains(producer))
|
||||
{
|
||||
Ok(producers)
|
||||
} else {
|
||||
Err(SegmentInvalidationError::InvalidProof)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum SegmentInvalidationDomain {
|
||||
LocalSingleSet,
|
||||
@@ -239,8 +170,7 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
fn producers() -> BTreeSet<SegmentInvalidationProducer> {
|
||||
complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION)
|
||||
.expect("production producer matrix should be complete")
|
||||
SegmentInvalidationProducer::REQUIRED.into_iter().collect()
|
||||
}
|
||||
|
||||
fn envelope() -> SegmentInvalidationEnvelope {
|
||||
@@ -390,63 +320,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segment_invalidation_producer_identities_must_be_known_and_complete() {
|
||||
assert_eq!(
|
||||
complete_segment_invalidation_producers(SegmentInvalidationProducerIdentity::REQUIRED_PRODUCTION),
|
||||
Ok(SegmentInvalidationProducer::REQUIRED.into_iter().collect())
|
||||
);
|
||||
assert_eq!(
|
||||
complete_segment_invalidation_producers([
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||
SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||
SegmentInvalidationProducerIdentity::Replication,
|
||||
SegmentInvalidationProducerIdentity::TierTransition,
|
||||
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||
SegmentInvalidationProducerIdentity::Unknown,
|
||||
]),
|
||||
Err(SegmentInvalidationError::UnknownProducer)
|
||||
);
|
||||
assert_eq!(
|
||||
complete_segment_invalidation_producers([
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||
SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||
SegmentInvalidationProducerIdentity::Replication,
|
||||
SegmentInvalidationProducerIdentity::TierTransition,
|
||||
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||
SegmentInvalidationProducerIdentity::TestFixture,
|
||||
]),
|
||||
Err(SegmentInvalidationError::UnknownProducer)
|
||||
);
|
||||
assert_eq!(
|
||||
complete_segment_invalidation_producers([
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||
SegmentInvalidationProducerIdentity::Replication,
|
||||
SegmentInvalidationProducerIdentity::TierTransition,
|
||||
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||
]),
|
||||
Err(SegmentInvalidationError::InvalidProof)
|
||||
);
|
||||
assert_eq!(
|
||||
complete_segment_invalidation_producers([
|
||||
SegmentInvalidationProducerIdentity::PutObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteObject,
|
||||
SegmentInvalidationProducerIdentity::DeleteMarker,
|
||||
SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||
SegmentInvalidationProducerIdentity::Replication,
|
||||
SegmentInvalidationProducerIdentity::TierTransition,
|
||||
SegmentInvalidationProducerIdentity::DirectoryObject,
|
||||
]),
|
||||
Err(SegmentInvalidationError::InvalidProof)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segment_invalidation_entries_are_bounded_and_key_checked() {
|
||||
let envelope = envelope();
|
||||
|
||||
@@ -919,6 +919,22 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// The filename's item count is untrusted. Reject a payload that contains
|
||||
// more items than advertised instead of returning success and allowing the
|
||||
// caller to delete the entry with trailing events still in the file.
|
||||
match deserializer.next() {
|
||||
None => {}
|
||||
Some(Ok(_)) => {
|
||||
return Err(StoreError::Deserialization(format!(
|
||||
"Batch for key {key} contains more than {} items",
|
||||
key.item_count
|
||||
)));
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
return Err(StoreError::Deserialization(format!("Failed to deserialize trailing batch item: {e}")));
|
||||
}
|
||||
}
|
||||
|
||||
if items.is_empty() && key.item_count > 0 {
|
||||
return Err(StoreError::Deserialization("No items found".to_string()));
|
||||
}
|
||||
@@ -1381,6 +1397,39 @@ mod tests {
|
||||
let _ = store.delete();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn get_multiple_errors_on_batch_with_trailing_items_instead_of_partial_success() {
|
||||
let dir = temp_store_dir("trailing-batch-items");
|
||||
let store = QueueStore::<String>::new_with_compression(&dir, 8, ".test", false);
|
||||
store.open().unwrap();
|
||||
|
||||
let items = vec!["aa".to_string(), "bb".to_string(), "cc".to_string()];
|
||||
let original_key = store.put_multiple(items).unwrap();
|
||||
assert_eq!(original_key.item_count, 3);
|
||||
|
||||
// Keep the three-item payload but make its filename claim that it contains
|
||||
// only two items, simulating a corrupt or otherwise untrusted queue key.
|
||||
let original_path = store.file_path(&original_key);
|
||||
let advertised_key = Key {
|
||||
item_count: 2,
|
||||
..original_key
|
||||
};
|
||||
let advertised_path = store.file_path(&advertised_key);
|
||||
std::fs::rename(&original_path, &advertised_path).unwrap();
|
||||
|
||||
let err = store.get_multiple(&advertised_key).unwrap_err();
|
||||
assert!(
|
||||
matches!(err, StoreError::Deserialization(_)),
|
||||
"expected Deserialization error, got {err:?}"
|
||||
);
|
||||
|
||||
// Because get_multiple failed, the batch entry remains available for
|
||||
// inspection or recovery instead of being silently discarded.
|
||||
assert!(advertised_path.exists());
|
||||
|
||||
let _ = store.delete();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_put_raw_respects_entry_limit() {
|
||||
let dir = temp_store_dir("concurrent-limit");
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
# Documentation
|
||||
|
||||
Use the focused indexes rather than treating this directory as an unordered
|
||||
collection:
|
||||
|
||||
- [Architecture knowledge base](architecture/README.md)
|
||||
- [Testing references](testing/README.md)
|
||||
|
||||
## Operations
|
||||
|
||||
Operational runbooks live under [`operations/`](operations/). Replication
|
||||
operators should start with:
|
||||
|
||||
| Runbook | Use it for |
|
||||
|---|---|
|
||||
| [Site replication operations](operations/site-replication-operations.md) | Health fields, pending operations, outage recovery, re-pair admission, IAM/SSE boundaries, and upgrades. |
|
||||
| [Replication target check](operations/replication-check.md) | Validating an S3 destination and version fidelity before enabling replication. |
|
||||
| [Replication object size limits](operations/replication-object-size-limits.md) | Multipart routing, large-object limits, and retry characteristics. |
|
||||
| [Replication outbound transport](operations/replication-outbound-transport.md) | Integrity headers, generic target behavior, and transport knobs. |
|
||||
|
||||
Other runbooks remain grouped by filename in [`operations/`](operations/);
|
||||
architecture pages link to the relevant runbook where a cross-boundary
|
||||
procedure is required.
|
||||
@@ -60,6 +60,8 @@ Required headings and strings in these files are asserted by `scripts/check_arch
|
||||
| [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md) | a client or `mc` call that works against MinIO fails against RustFS and you need to know whether the endpoint is missing, stubbed, or deliberately different |
|
||||
| [minio-file-format-compat.md](minio-file-format-compat.md) | deciding whether a MinIO drive set, bucket-metadata blob, or SSE object can be read or imported by a given RustFS build, or before touching a listed version anchor |
|
||||
|
||||
Operations runbooks live in [../operations/](../operations/) and testing references in [../testing/README.md](../testing/README.md).
|
||||
Operations runbooks are registered in the [documentation operations index](../README.md#operations), and testing references live in [../testing/README.md](../testing/README.md).
|
||||
|
||||
For replication operations, start with [site replication operations](../operations/site-replication-operations.md), [replication target check](../operations/replication-check.md), [replication object size limits](../operations/replication-object-size-limits.md), and [replication outbound transport](../operations/replication-outbound-transport.md).
|
||||
|
||||
For per-node HTTP failure ratios and cached storage probe provenance, see [S3 write failure diagnostics](../operations/s3-write-failure-diagnostics.md).
|
||||
|
||||
@@ -38,6 +38,42 @@ Counts ignore blank lines and comments; compute them from the files. The lifecyc
|
||||
|
||||
"Supported" for the SSE row means RustFS encrypts and decrypts its own objects. MinIO SSE objects (SSE-S3, SSE-KMS, SSE-C) are not readable in default builds; see [minio-file-format-compat.md Part C](minio-file-format-compat.md#part-c--server-side-encryption-sse) for the `rio-v2` migration build.
|
||||
|
||||
## Replication Support Boundary
|
||||
|
||||
Site replication and bucket replication are not the same compatibility claim.
|
||||
Site replication requires RustFS-compatible peer admin APIs and coordinates
|
||||
IAM, topology, buckets, and metadata. A generic S3-compatible service can only
|
||||
be a bucket-replication data target.
|
||||
|
||||
For a generic S3 target, RustFS supports object PUT/HEAD/DELETE, multipart
|
||||
uploads, tags, version deletes, and Object Lock mutations when the target
|
||||
implements the corresponding S3 APIs and has versioning enabled. Targets that
|
||||
mint their own version IDs are supported through a per-target version ledger;
|
||||
pre-ledger replicas are adopted only when exact key and ETag identify one
|
||||
unambiguous target version. `NoSuchVersion` for an already absent addressed
|
||||
replica is treated as converged.
|
||||
|
||||
The following are capability boundaries, not universal S3 claims:
|
||||
|
||||
- `GET /BUCKET?replication-check` must pass the phases required by the intended
|
||||
workload. `VersionFidelity` may report a minting target as mismatched even
|
||||
though ledger-addressed delete and Object Lock phases succeed.
|
||||
- A target that rejects standard multipart constraints, required Object Lock
|
||||
integrity headers, or the configured checksum framing is unsupported until
|
||||
its transport settings are made compatible.
|
||||
- SSE-S3 and SSE-KMS are decrypted at the source and re-encrypted by the
|
||||
destination's KMS. SSE-C uses ciphertext passthrough and requires target
|
||||
evidence. Unsupported or ambiguous encryption metadata fails closed.
|
||||
- ACL authorization is intentionally unsupported, and generic targets never
|
||||
receive RustFS IAM/site-control-plane state.
|
||||
- RustFS does not guess between multiple target versions with the same key and
|
||||
ETag. The mutation remains failed and retryable until repair establishes an
|
||||
unambiguous mapping.
|
||||
|
||||
See [site replication operations](../operations/site-replication-operations.md)
|
||||
for health, recovery, and upgrade rules and [replication outbound transport](../operations/replication-outbound-transport.md)
|
||||
for the tested target classes and knobs.
|
||||
|
||||
## Not Yet Passing
|
||||
|
||||
Standard S3 areas that must not be described as complete:
|
||||
|
||||
@@ -66,7 +66,6 @@ The manifest has the following JSON contract (all fields are required):
|
||||
| `rounds`, `duration_seconds`, `min_free_bytes` | 3..10 groups, 900..86400 seconds for measured runs, and the independently estimated free-space reservation in bytes. Synthetic runs may use 1 second. |
|
||||
| `baseline`, `candidate` | Each contains executable `binary`, full 40-character `revision`, and verified `sha256`. The runner rehashes binaries before every leg. |
|
||||
| `fixed` | `config_sha256`, `dataset_sha256`, `release_flags`, `durability`, `disk_type`, `cache_state`, `load_command`, `resource_isolation`, `topology` (`EC8+4`), and positive `offered_load_ops`. Hashes use 64 lowercase hexadecimal characters. |
|
||||
| `release_evidence` | Required for `measured` runs. It binds the 3x4 EC8+4 topology, multi-pool/multi-set coverage, per-node metrics endpoints, same-window distributed sampling, process restart and crash-restart fault modes, mixed-version reader/writer/rollback participation, and allocation/flamegraph/RSS/save-frequency profile artifact requirements. Synthetic runs do not need this field and still cannot approve release evidence. |
|
||||
| `oracles` | A map with all five scenario names. Each value contains positive integer `objects`, `versions`, `bytes`, and `sha256` of the independently prepared canonical object/version/content manifest. |
|
||||
| `expected_healed_objects` | A map with all five scenario names and independently seeded repair counts. Running-heal and MRF-replay require a positive count. |
|
||||
|
||||
@@ -76,14 +75,6 @@ object/version/content result. Fix the foreground arrival rate (offered load),
|
||||
cache preparation procedure, configuration, and hardware across every leg.
|
||||
Do not include credentials in the manifest, adapter output, or saved commands;
|
||||
the collector reads `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` from its environment.
|
||||
The adapter must echo the measured run's `release_evidence` object in every
|
||||
measurement response. A mismatch fails the cell because it means the deployment,
|
||||
mixed-version set, crash mode, or profiler contract no longer matches the
|
||||
operator-reviewed manifest. This echo is provenance binding only; it does not
|
||||
replace the independent correctness oracle, distributed metrics samples, profile
|
||||
artifacts, or ABBA comparison thresholds. The summary tool revalidates the same
|
||||
manifest contract before it can print a measured PASS result, so hand-built or
|
||||
trimmed reports without this provenance fail closed.
|
||||
|
||||
#### Deployment Adapter Contract
|
||||
|
||||
@@ -213,14 +204,6 @@ The command prints only `PASS scanner_heal_perf ...` for measured passing ABBA
|
||||
evidence, otherwise `FAIL scanner_heal_perf ...`. The JSON and Markdown outputs
|
||||
carry the key p99/throughput/P1/P2/cache-cost fields and artifact provenance
|
||||
hashes; raw per-cell logs remain in the original artifact tree for audit.
|
||||
Failed or interrupted ABBA reports that contain only `status`, `performance`,
|
||||
`completed_cells`, and `error` also summarize as `FAIL`; they do not become
|
||||
performance evidence, and a missing comparison matrix is accepted only for a
|
||||
non-passing report.
|
||||
Measured passing reports must also retain the W10/W11 foreground-pressure,
|
||||
heal-lock-wait, and heal-attempt-cost fields emitted by the ABBA evaluator. If
|
||||
those fields are removed, empty, malformed, or length-mismatched, the quiet
|
||||
summary fails closed instead of treating the report as performance evidence.
|
||||
|
||||
They cover the complete 120-cell schedule, data isolation, missing builds and
|
||||
oracles, zero samples/requests, swallowed request errors, offered-load drift,
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
# Site Replication Operations
|
||||
|
||||
**Use this when:** operating a site-replication deployment, diagnosing a peer
|
||||
outage or incomplete topology change, pairing sites that already contain data,
|
||||
or planning an upgrade.
|
||||
|
||||
**Source of truth:** `rustfs/src/admin/handlers/site_replication.rs`,
|
||||
`rustfs/src/site_replication/`, and the bucket-replication worker under
|
||||
`crates/ecstore/src/bucket/replication/`.
|
||||
|
||||
Site replication combines two different convergence paths:
|
||||
|
||||
- the control plane replicates buckets, bucket metadata, IAM, and topology;
|
||||
- ordinary bucket replication moves object versions and delete operations.
|
||||
|
||||
An `enabled: true` response only says that a site has more than one configured
|
||||
peer. It does not prove that every peer is reachable or caught up. Always read
|
||||
`pendingOperation`, `retryStats`, `PeerErrors`, and `Metrics` as well.
|
||||
|
||||
## Routine checks
|
||||
|
||||
Run these commands from an admin workstation with one alias per site:
|
||||
|
||||
```console
|
||||
mc admin replicate info site-a
|
||||
mc admin replicate status site-a
|
||||
```
|
||||
|
||||
Check more than one site. A partition can leave each side with a different but
|
||||
locally valid view.
|
||||
|
||||
`replicate info` is the compact control-plane view:
|
||||
|
||||
| Field | Interpretation |
|
||||
|---|---|
|
||||
| `enabled` | More than one site is configured; this is not a health verdict. |
|
||||
| `sites` | The locally persisted topology. Compare deployment IDs and endpoints on every site. |
|
||||
| `retryStats.pending` | Collapsed peer deliveries waiting to be retried. |
|
||||
| `retryStats.failed` | Deliveries that crossed the escalation threshold and require attention. |
|
||||
| `retryStats.lastError` | A redacted summary of the most recent delivery failure. |
|
||||
| `pendingOperation` | A durable multi-step topology operation described below. Absence is the healthy steady state. |
|
||||
|
||||
`replicate status` adds detailed convergence state:
|
||||
|
||||
| Field | Interpretation |
|
||||
|---|---|
|
||||
| `Sites` / `PeerStates` | Configured peers and derived reachability/configuration state. |
|
||||
| `PeerErrors` | A peer could not be queried. Its detailed counters may be absent; do not read zeros as success. |
|
||||
| `BucketStats` | Per-bucket presence and versioning, replication, lifecycle, Object Lock, and metadata mismatches. |
|
||||
| `PolicyStats`, `UserStats`, `GroupStats` | IAM inventory mismatches. |
|
||||
| `RetryStats` | Durable control-plane retry backlog and escalation count. |
|
||||
| `Metrics.replMetrics` | Per-destination online state, downtime, replicated counts/bytes, and `failed` totals/windows. |
|
||||
| `Metrics.queued` / `Metrics.inProgress` | Object work waiting or active on the responding node. |
|
||||
| `Metrics.errors` | Node-level object-replication failures. When only queue statistics are available, RustFS synthesizes a node entry and preserves this counter rather than reporting zero. |
|
||||
| `Metrics.retries` | Redeliveries. Always zero today: a failed object is not retried by an event, it waits for the scanner pass described below. Read `errors` instead. |
|
||||
|
||||
Healthy means: the same topology is visible on all sites, no pending operation,
|
||||
no peer error, no failed retry escalation, required bucket/IAM state is in sync,
|
||||
and queue/error counters are stable or falling. Counters are cumulative; alert on
|
||||
their rate and on a backlog that does not drain, not merely on a non-zero total.
|
||||
|
||||
## Pending operations and recovery
|
||||
|
||||
`pendingOperation` contains `operation`, an opaque `id`, `pendingPeers`, and
|
||||
`ackedPeers`. Do not edit the site-replication state object by hand. The marker
|
||||
is the crash-recovery journal and removing it can make a partially applied
|
||||
operation look complete.
|
||||
|
||||
The heavyweight reconciler runs once at startup and every 600 seconds. The
|
||||
lightweight retry drain runs every 30 seconds. A restart is therefore a valid
|
||||
way to cause an immediate heavyweight pass after the underlying fault has been
|
||||
fixed, but it is not a substitute for fixing connectivity, credentials, TLS,
|
||||
or the remote endpoint.
|
||||
|
||||
### `remove`
|
||||
|
||||
The original topology and each peer acknowledgement are persisted before the
|
||||
operation finalizes. While peers remain in `pendingPeers`, restore access to
|
||||
them and wait for reconciliation. If a peer is permanently gone, a new remove
|
||||
request may remove all currently active unacknowledged peers; RustFS permits
|
||||
that request and then finalizes against the remaining topology. Removing the
|
||||
local site or all sites is also an explicit completion path.
|
||||
|
||||
Do not re-add a site merely to hide this marker. First compare the topology on
|
||||
all reachable peers. If the same operation ID makes no progress for more than
|
||||
one heavyweight interval, collect `PeerErrors`, `RetryStats`, and the
|
||||
site-replication logs before retrying the remove.
|
||||
|
||||
### `rotate-svc-acct`
|
||||
|
||||
Service-account rotation keeps the candidate secrets and peer acknowledgements
|
||||
until every current remote peer accepts the rotation. Restore the failing peer
|
||||
and allow the reconciler to resume it. Do not manually delete either candidate
|
||||
credential during this window: doing so can remove the only credential that a
|
||||
not-yet-acknowledged peer accepts.
|
||||
|
||||
After the marker clears, verify `replicate status` from every site, then retire
|
||||
any separately retained old credential material according to local policy.
|
||||
|
||||
### `endpoint-refresh`
|
||||
|
||||
An endpoint, CA, or TLS-verification edit first refreshes the replication
|
||||
target on every active peer and records acknowledgements. On startup and every
|
||||
heavyweight pass, RustFS probes peer capability, uses the endpoint-refresh API
|
||||
when supported (or the legacy peer-edit fallback), refreshes local bucket
|
||||
targets, and commits the edit only after every still-active peer acknowledges.
|
||||
|
||||
If this marker is stuck:
|
||||
|
||||
1. Confirm that the proposed endpoint and CA are correct and reachable from
|
||||
every site, not only from the admin workstation.
|
||||
2. Restore the site-replication service account and TLS trust path.
|
||||
3. Wait for one 600-second pass or restart one healthy node to trigger the
|
||||
startup pass.
|
||||
4. Re-run the identical edit only if the operation remains visible; a different
|
||||
endpoint edit is rejected while the existing refresh is pending. The journal
|
||||
pins the edit's payload, so a re-run without `--replicate-ilm-expiry` keeps
|
||||
the value the first attempt recorded, and a re-run asking for a different
|
||||
value is rejected. Finish or remove the pending refresh before changing it.
|
||||
|
||||
A peer removed from the topology no longer blocks completion. A remove request
|
||||
is accepted when it removes every active unacknowledged peer.
|
||||
|
||||
While this marker is present, control-plane retry replay to the other peers
|
||||
keeps running, but bucket wiring reconciliation waits: it rewrites the same
|
||||
targets the refresh is changing. Expect bucket-level drift on this site to
|
||||
persist until the refresh settles.
|
||||
|
||||
## Outage recovery and convergence time
|
||||
|
||||
Control-plane retry begins on the 30-second drain, while heavyweight snapshots,
|
||||
pending topology operations, and bucket wiring are revisited on the 600-second
|
||||
pass. Object MRF entries are persisted every 10 seconds by default and target
|
||||
health is probed every 5 seconds. These are scheduling bounds, not delivery
|
||||
SLAs: network timeouts and the amount of queued work add to them.
|
||||
|
||||
Objects that must be rediscovered by the scanner have this conservative upper
|
||||
bound before discovery:
|
||||
|
||||
```text
|
||||
RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES
|
||||
× max(RUSTFS_SCANNER_CYCLE, actual duration of one scanner cycle)
|
||||
```
|
||||
|
||||
The defaults re-descend a compacted directory every 16 cycles. A practical
|
||||
production starting point for a tighter recovery objective is
|
||||
`RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=4`; `1` forces re-descent every cycle.
|
||||
Measure the additional disk and metadata load before lowering it further or
|
||||
tuning the scanner cadence. For an immediate operator-driven recovery, start a
|
||||
site resync with `mc admin replicate resync start` and monitor its status.
|
||||
Transfer time after discovery remains proportional to backlog size, bandwidth,
|
||||
worker capacity, and target latency. Use queue depth and the rate of
|
||||
`Metrics.errors` rather than the formula alone to decide whether convergence is
|
||||
progressing.
|
||||
|
||||
## Pairing sites that already contain data
|
||||
|
||||
When more than one requested site is non-empty, preflight considers each bucket
|
||||
name held by more than one site:
|
||||
|
||||
- versioning must be `Enabled` on every site holding the shared bucket;
|
||||
- Object Lock enablement must be identical on every holder.
|
||||
|
||||
A bucket present on only one site is safe: post-add backfill creates it on the
|
||||
other peers. A shared unversioned bucket is rejected because merging can
|
||||
overwrite the only copy of an object. An Object Lock mismatch is rejected
|
||||
because lock enablement cannot be changed after bucket creation and convergence
|
||||
could otherwise strip a WORM guarantee.
|
||||
|
||||
If preflight rejects the pair, keep the authoritative copy, delete the
|
||||
conflicting bucket (or its contents) from all other sites, run `replicate add`
|
||||
again, and then start `replicate resync` from the surviving site. Back up and
|
||||
validate the authoritative data before deleting anything.
|
||||
|
||||
## IAM convergence and repair boundary
|
||||
|
||||
Ordinary IAM changes are delivered to each peer. A successful bulk IAM import
|
||||
also schedules one collapsed full-IAM snapshot per remote peer. A failed IAM
|
||||
deletion is replayed before that snapshot so the snapshot cannot re-create a
|
||||
principal or grant that was already revoked.
|
||||
|
||||
The safety state has two bounds:
|
||||
|
||||
- deletion high-water marks are retained for 30 days;
|
||||
- deletion replay bodies are capped at 256 distinct entities per peer.
|
||||
|
||||
Repeated deletion of the same entity replaces its saved body. When the per-peer
|
||||
cap is exceeded or the body cannot be serialized, the retry entry remains
|
||||
escalated rather than pretending the deletion is replayable. An item from an
|
||||
older sender without a source timestamp cannot install the 30-day high-water
|
||||
mark, so verify it explicitly after a prolonged split. A successful drain
|
||||
clears replay bodies; removing the peer prunes its bodies. For an escalated IAM
|
||||
retry, use the site-replication repair workflow for the affected peer and IAM
|
||||
family, then verify users, service accounts, groups, policies, and mappings on
|
||||
both sides. Repair is the operator's explicit accountability transfer and
|
||||
clears the saved deletion bodies only after the IAM repair succeeds.
|
||||
|
||||
A group's status converges in one direction. An explicit disable is applied
|
||||
everywhere, including through a snapshot, but a membership change never
|
||||
carries an enable - it would otherwise re-enable a group frozen on the
|
||||
receiving site. If a group ended up disabled on one site only, re-enable it
|
||||
there explicitly with `mc admin group enable`; a snapshot or repair will not
|
||||
do it.
|
||||
|
||||
Treat IAM divergence as a security incident: a user deleted on one site can
|
||||
remain usable on an unreachable peer until replay or repair completes. A peer
|
||||
whose IAM entry is escalated does not receive scheduled snapshots either -
|
||||
including the one a bulk import schedules - until the repair settles it.
|
||||
|
||||
## Encrypted objects
|
||||
|
||||
| Source form | Replication behavior | Fail-closed condition |
|
||||
|---|---|---|
|
||||
| SSE-S3 | The source decrypts the object; the request sends only `AES256` intent; the destination encrypts with its own KMS. Source envelope material never leaves the site. | The destination cannot satisfy the encryption request, or the source metadata is incomplete/unsupported. The replica is `FAILED`; plaintext is not silently stored. |
|
||||
| SSE-KMS | The source decrypts the object; the request sends `aws:kms` intent without the source-local key ID; the destination selects its own configured KMS key. | Either side cannot decrypt/encrypt, or the metadata mixes incompatible encryption evidence. |
|
||||
| SSE-C | Stored ciphertext and the required SSE-C replication transport metadata pass through. RustFS verifies target evidence before accepting the replica. | The target does not echo the customer-algorithm evidence, required material/layout is absent, or the metadata is ambiguous. |
|
||||
|
||||
Unknown MinIO/RustFS encryption markers are never forwarded as ordinary user
|
||||
metadata. They fail replication so an operator must migrate or repair the
|
||||
object with a supported format.
|
||||
|
||||
## Rolling upgrades and rollback
|
||||
|
||||
Keep every node in one site on the same version whenever possible. Upgrade all
|
||||
nodes of one site consecutively, verify its startup reconciliation and status,
|
||||
then move to the next site. Do not intentionally leave a site mixed-version:
|
||||
admin requests can land on different nodes, and an older node may not resume a
|
||||
new pending-operation shape or expose its health fields.
|
||||
|
||||
Current state additions are optional and defaulted, so older readers ignore
|
||||
them. The target-version ledger is stored as dual-prefixed internal object
|
||||
metadata and is also ignored by older readers; rollback does not corrupt the
|
||||
object format, but older code loses the assigned-version routing improvement.
|
||||
|
||||
Before rolling back across the fix that retains the data directory of a version
|
||||
awaiting purge replication (rustfs/rustfs#7307), ensure no version purge is
|
||||
pending. Older code can free that retained version's data directory before the
|
||||
remote purge is acknowledged, leaving unreadable metadata and blocking bucket
|
||||
deletion. Drain or repair replication and take a metadata/data backup first.
|
||||
|
||||
## Runtime knobs
|
||||
|
||||
These values are read when the owning background task starts. Restart the
|
||||
server after changing them. The millisecond intervals have a 10 ms floor;
|
||||
invalid values fall back to the default with a warning.
|
||||
|
||||
| Variable | Default | Effect |
|
||||
|---|---:|---|
|
||||
| `RUSTFS_REPL_HEALTH_CHECK_INTERVAL_MS` | `5000` | Remote-target health probe interval. Lowering it increases outbound probes. |
|
||||
| `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS` | `10000` | Maximum periodic interval between MRF persistence flushes; 1,000 new entries also trigger a flush. |
|
||||
| `RUSTFS_REPL_RESYNC_POLL_MAX_MS` | `60000` | Upper bound for randomized resync retry-poll sleep. |
|
||||
| `RUSTFS_REPL_RESYNC_MAX_JOBS` | `2` | Concurrent resync jobs; values are bounded to `1..=32`. |
|
||||
|
||||
Transport-specific controls and target behavior are documented in
|
||||
[Replication outbound transport](replication-outbound-transport.md). Validate a
|
||||
new destination with [Replication target check](replication-check.md), and read
|
||||
[Replication object size limits](replication-object-size-limits.md) before
|
||||
moving large objects.
|
||||
+38
-97
@@ -3,7 +3,7 @@
|
||||
**Use this when:** a check is red and you need to know whether it blocks the merge, which workflow and job produced it, and how to reproduce it locally.
|
||||
**Source of truth:** the live `main` ruleset (command below) for required status; `.github/workflows/<file>.yml` for triggers, `paths`, `timeout-minutes`, and cron; `.config/nextest.toml` for e2e profile filters; `.github/scheduled-validations.json` for the freshness-watchdog list.
|
||||
|
||||
A job blocks a merge only when its exact check name is in the live `main` ruleset. A workflow name, a `merge_group` trigger, or a red PR check does not make a job required by itself.
|
||||
A job blocks a merge when its exact check name is required by the live `main` ruleset, or when its result is required by the `Test and Lint` aggregate. A workflow name, a `merge_group` trigger, or an unrelated red PR check does not make a job required by itself.
|
||||
|
||||
## Required merge checks
|
||||
|
||||
@@ -13,9 +13,11 @@ The `main` ruleset (`6436880`) requires exactly these contexts, with `strict_req
|
||||
|---|---|---|
|
||||
| `CLA Check` | `cla.yml` | Contributor agreement |
|
||||
| `Quick Checks` | `ci.yml` job `quick-checks` | Formatting and repository guard scripts |
|
||||
| `Test and Lint` | `ci.yml` job `test-and-lint` | Clippy, workspace nextest (`ci` profile, excluding `e2e_test`), doctests, migration-gate count (`scripts/check_migration_gate_count.sh`) |
|
||||
| `Test and Lint` | `ci.yml` job `required-checks` | Exact expected results for every CI validation job, including workspace checks, critical E2E, feature lanes, and event-specific full suites |
|
||||
|
||||
For PRs limited to the `paths-ignore` list in `ci.yml`, `ci-docs-only.yml` reports `Quick Checks` and `Test and Lint` under the same names; it runs the quick checks and `scripts/check_no_planning_docs.sh`, not a Rust build or tests. `scripts/check_ci_paths_sync.sh` keeps the two path lists aligned.
|
||||
Every PR enters `ci.yml`. The `classify-changes` job uses the base revision of `scripts/ci_gate.py` to select a conservative documentation-only path: root Markdown/licenses, `AGENTS.md`, Markdown under `docs/` or `.agents/skills/`, and documentation images. Unknown paths, unavailable Git history, an empty diff, or a missing base policy select the full matrix. Renames include their deleted source path. Documentation-only PRs still run Quick Checks and Typos; the aggregate requires the expensive jobs to be skipped exactly as selected.
|
||||
|
||||
`required-checks` runs even after failed or skipped dependencies. `scripts/ci_gate.py verify` rejects missing jobs, unexpected jobs, failure, cancellation, and unexpected skips; optional lanes are required only on their declared events. `Workspace Test and Lint` is the ordinary Rust job, while `Test and Lint` uniquely names the aggregate. New validation jobs must update both its direct dependencies and the script contract. Test this wiring and its failure cases with `python3 scripts/ci_gate.py --self-test`.
|
||||
|
||||
Verify the live rule before changing merge policy:
|
||||
|
||||
@@ -24,37 +26,36 @@ gh api repos/rustfs/rustfs/rulesets/6436880 \
|
||||
--jq '.rules[] | select(.type == "required_status_checks") | .parameters'
|
||||
```
|
||||
|
||||
Promotion rule: never promote a report-only lane to required from one green run. Require at least 14 days and 30 representative PRs with at least 99% complete execution, then update the ruleset and this file together.
|
||||
The aggregate requires the validation lanes already selected by `ci.yml`; this closes the gap where a failing critical lane left the required workspace check green. Independent workflows remain report-only unless separately required. Before adding a new expensive lane or moving existing PR coverage to a schedule, collect representative execution and regression evidence, establish ownership and a working scheduled replacement, and update this reference with the resulting policy.
|
||||
|
||||
## Pull request and merge matrix
|
||||
|
||||
"Report-only" means visible and actionable but not in the required list. Budgets are each job's `timeout-minutes` in the named workflow and are not copied here.
|
||||
"Via aggregate" means a wrong result fails the required `Test and Lint` check. "Report-only" means visible and actionable but outside both the required list and aggregate. Budgets are each job's `timeout-minutes` in the named workflow and are not copied here.
|
||||
|
||||
| Event | Check name | Workflow / job | Merge status | Reproduce |
|
||||
|---|---|---|---|---|
|
||||
| PR, non-doc change | `Quick Checks` | `ci.yml` `quick-checks` | Required | `make pre-commit` |
|
||||
| PR, non-doc change | `Test and Lint` | `ci.yml` `test-and-lint` | Required | `cargo clippy --all-targets -- -D warnings`; `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --doc`; `scripts/check_migration_gate_count.sh` |
|
||||
| PR, non-doc change | `Typos` | `ci.yml` `typos` | Report-only | `typos` |
|
||||
| PR, non-doc change | `ILM Integration (serial)` | `ci.yml` `test-ilm-integration-serial` | Report-only | exact command in the job |
|
||||
| PR, non-doc change | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Report-only | `cargo nextest run` with the job's `--features` |
|
||||
| PR, non-doc change | `Connect Short Credential Boundary` | `ci.yml` `connect-short-credential-boundary` | Report-only | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail |
|
||||
| PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Report-only; prerequisite for the black-box jobs | `cargo build -p rustfs --bins` |
|
||||
| PR, non-doc change | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Report-only | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
|
||||
| PR, non-doc change | `End-to-End Tests` | `ci.yml` `e2e-tests` | Report-only | `cargo nextest run --profile e2e-smoke -p e2e_test`, then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>`; membership guards `scripts/check_test_wiring.py --check-profile e2e-smoke <listing.json>` and `scripts/check_security_smoke_count.sh check <listing.json>` |
|
||||
| PR, non-doc change | `S3 Implemented Tests` | `ci.yml` `s3-implemented-tests` | Report-only | build `rustfs`, then `scripts/s3-tests/run.sh` with the job's `DEPLOY_MODE` / `TEST_MODE` / `MAXFAIL` env |
|
||||
| PR, non-doc change | `S3 Lifecycle Behavior Tests` | `ci.yml` `s3-lifecycle-behavior-tests` | Report-only | `scripts/s3-tests/run.sh` with the job's accelerated-scanner env |
|
||||
| PR to `main` or `release` touching `paths` in `audit.yml` | `Cargo Deny`, `Workflow Pin Report`, `Dependency Review` | `audit.yml` `cargo-deny`, `workflow-pin-report`, `dependency-review` | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| Push to `main` or `release` touching `paths` in `audit.yml` | `Cargo Deny`, `Workflow Pin Report` | `audit.yml` `cargo-deny`, `workflow-pin-report` | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| PR, non-doc change | `Workspace Test and Lint` | `ci.yml` `test-and-lint` | Via aggregate | `cargo clippy --all-targets -- -D warnings`; `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --doc`; `scripts/check_migration_gate_count.sh` |
|
||||
| PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` |
|
||||
| PR, non-doc change | `ILM Integration (serial)` | `ci.yml` `test-ilm-integration-serial` | Via aggregate | exact command in the job |
|
||||
| PR, non-doc change | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate | `cargo nextest run` with the job's `--features` |
|
||||
| PR, non-doc change | `Connect Short Credential Boundary` | `ci.yml` `connect-short-credential-boundary` | Via aggregate | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail |
|
||||
| PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `cargo build -p rustfs --bins --features e2e-test-hooks` |
|
||||
| PR, non-doc change | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Via aggregate | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
|
||||
| PR, non-doc change | `End-to-End Tests` | `ci.yml` `e2e-tests` | Via aggregate | `cargo nextest run --profile e2e-smoke -p e2e_test`, then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>`; membership guards `scripts/check_test_wiring.py --check-profile e2e-smoke <listing.json>` and `scripts/check_security_smoke_count.sh check <listing.json>` |
|
||||
| PR, non-doc change | `S3 Implemented Tests` | `ci.yml` `s3-implemented-tests` | Via aggregate | build `rustfs`, then `scripts/s3-tests/run.sh` with the job's `DEPLOY_MODE` / `TEST_MODE` / `MAXFAIL` env |
|
||||
| PR, non-doc change | `S3 Lifecycle Behavior Tests` | `ci.yml` `s3-lifecycle-behavior-tests` | Via aggregate | `scripts/s3-tests/run.sh` with the job's accelerated-scanner env |
|
||||
| PR touching `paths` in `audit.yml` | `Cargo Deny`, `Workflow Pin Report`, `Dependency Review` | `audit.yml` `cargo-deny`, `workflow-pin-report`, `dependency-review` | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| PR touching `paths` in `architecture-migration-rules.yml` | `Architecture Migration Rules` | `architecture-migration-rules.yml` `architecture-migration-rules` | Report-only | `scripts/check_architecture_migration_rules.sh` |
|
||||
| PR touching `paths` in `nix.yml` | `Nix Build & Check` | `nix.yml` `nix-validation` | Report-only | `nix flake check` |
|
||||
| PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / <target>` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` |
|
||||
| PR touching `paths` in `windows-filesystem.yml` | `Rename Safety` | `windows-filesystem.yml` `rename-safety` | Report-only | the `cargo test -p rustfs-ecstore --lib <filter>` commands in the job, on Windows |
|
||||
| PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` |
|
||||
| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from the previous release`, `Mixed-version rolling upgrade from the previous release`, `Bucket configuration survives the upgrade`, `Rollback reads current bucket metadata`, `ODM configuration recovery after rc.5 rollback`, `Multipart layouts survive the rc.5 upgrade`, `rc.5 multipart replication baseline` | `e2e-upgrade.yml` `upgrade` matrix | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
|
||||
| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from the previous release`, `Mixed-version rolling upgrade from the previous release`, `Bucket configuration survives the upgrade`, `Rollback reads current bucket metadata` | `e2e-upgrade.yml` `upgrade` matrix | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
|
||||
| PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` |
|
||||
| PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test <name> -- --ignored --test-threads=1` commands |
|
||||
| PR limited to main-CI-excluded paths | `Quick Checks`, `Test and Lint` | `ci-docs-only.yml` `quick-checks`, `test-and-lint` | Required | `git diff --check`; `make doc-paths-check`; `scripts/check_no_planning_docs.sh` |
|
||||
| `merge_group`; push to `main` or `release` | `End-to-End Tests (full merge gate)` | `ci.yml` `e2e-full` | Report-only | `cargo nextest run --profile e2e-full -p e2e_test` |
|
||||
| PR, documentation-only selection | `Quick Checks`, `Typos`, `Test and Lint` | `ci.yml` `quick-checks`, `typos`, `required-checks` | Required directly or via aggregate | Quick Checks commands; `python3 scripts/ci_gate.py --self-test` |
|
||||
| `merge_group`; push to `main` | `End-to-End Tests (full merge gate)` | `ci.yml` `e2e-full` | Via aggregate on these events | `cargo nextest run --profile e2e-full -p e2e_test` |
|
||||
|
||||
e2e filters live in `.config/nextest.toml`; extend a profile instead of adding a second selector. Before a profile runs, `scripts/check_test_wiring.py` compares its listing to the committed digest in `.config/e2e-<profile>-selection.txt`, so a silent test drop fails closed.
|
||||
|
||||
@@ -63,16 +64,16 @@ cost. `data_usage_test` runs in the PR `e2e-smoke` lane so changes that affect
|
||||
authoritative scanner usage publication, quota-visible usage, or admin usage
|
||||
snapshots get an end-to-end signal before merge review. `heal_erasure_disk_rebuild_test`
|
||||
runs in `e2e-full` so core erasure heal rebuild regressions are caught no later
|
||||
than the merge queue or `main`/`release` push lane; it also remains in `e2e-nightly` with
|
||||
than the merge queue or `main` push lane; it also remains in `e2e-nightly` with
|
||||
the serialized cluster fault-domain suites for scheduled soak signal.
|
||||
|
||||
## Scheduled validation
|
||||
|
||||
Scheduled lanes never block a PR. Their workflow-local gate fails the run, scheduled failures route to the shared failure-issue action, and `scheduled-validation-freshness.yml` fails when a workflow listed in `.github/scheduled-validations.json` has not run within its `max_age_hours` (a `never_ran_grace_until` entry covers the window before a newly enabled cron's first slot). Cadence is qualitative here; the cron lives in each workflow's `on.schedule`.
|
||||
Scheduled lanes never block a PR. Their workflow-local gate fails the run, scheduled failures route to the shared failure-issue action, and `scheduled-validation-freshness.yml` fails when a workflow listed in `.github/scheduled-validations.json` has no recent attempt or completed successful scheduled run within its `max_age_hours` (a `never_ran_grace_until` entry covers the window before a newly enabled cron's first slot). Cadence is qualitative here; the cron lives in each workflow's `on.schedule`.
|
||||
|
||||
| Workflow (cadence) | Jobs | Verdict and artifacts | In freshness list | Reproduce |
|
||||
|---|---|---|---|---|
|
||||
| `ci.yml` (weekly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | per-job | yes | dispatch `ci.yml` |
|
||||
| `ci.yml` (weekly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | strict aggregate; the full E2E lane runs on dispatch, merge groups, and main pushes | yes | dispatch `ci.yml` |
|
||||
| `build.yml` (weekly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
|
||||
| `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
|
||||
| `e2e-distributed.yml` (storage-sensitive PRs + nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes, with `never_ran_grace_until` | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, then `cargo nextest run --profile e2e-distributed -p e2e_test` |
|
||||
@@ -86,10 +87,10 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched
|
||||
| `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow |
|
||||
| `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes | `make coverage` |
|
||||
| `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
|
||||
| `e2e-upgrade.yml` (weekly) | `upgrade` (7-case matrix) | upgrade and rollback gate; server logs | no | see the PR row |
|
||||
| `e2e-upgrade.yml` (weekly) | `upgrade` (4-case matrix) | upgrade and rollback gate; server logs | no | see the PR row |
|
||||
| `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | no | see the PR row |
|
||||
| `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | no | see the PR row |
|
||||
| `scheduled-validation-freshness.yml` (nightly) | `check-freshness` | fails on a never-created or stale schedule | n/a | dispatch |
|
||||
| `scheduled-validation-freshness.yml` (nightly) | `check-freshness` | fails on missing or stale attempts or completed successes | n/a | dispatch |
|
||||
|
||||
Manual `workflow_dispatch` runs are debugging evidence and do not open scheduled-failure issues. A manual performance run may explicitly allow a known regression; that override is not a passing baseline.
|
||||
|
||||
@@ -149,21 +150,11 @@ and exact S3 content.
|
||||
This case is a **four-node, one-drive-per-node process-restart test**. It is not
|
||||
power-loss validation, a 3x4 EC8+4 experiment, an all-version inventory, or proof
|
||||
of scanner enumeration, exact MRF disposition, legacy migration, or rollback.
|
||||
The schema 2 registry separates the implemented single-set restart lane from
|
||||
structured release lanes for authority coverage, checkpoint/crash, status and
|
||||
outcome, MRF responsibility, mixed-version rollback, scheduler pressure,
|
||||
maintenance producers, and EC8+4 multi-set coverage. All G01-G14/P1-P4 and
|
||||
R-E/R-D/R-L release requirements stay `pending` until their actual
|
||||
feature-specific oracles, measurements and required topologies exist. Missing
|
||||
cases cannot be supplied by synthetic W20 results. W20's bounded JSON and
|
||||
file-hash helpers are reused; its ABBA performance contracts remain in
|
||||
The registry keeps all G01-G14/P1-P4 and R-E/R-D/R-L release requirements pending
|
||||
until their actual feature-specific oracles and required topologies exist.
|
||||
Missing cases cannot be supplied by synthetic W20 results. W20's bounded JSON
|
||||
and file-hash helpers are reused; its ABBA performance contracts remain in
|
||||
`docs/operations/scanner-benchmark-runbook.md`.
|
||||
Measured ABBA manifests must also carry the runbook's `release_evidence`
|
||||
contract. The runner rejects reports that cannot bind the exact 3x4 EC8+4
|
||||
topology, multi-pool/multi-set shape, distributed same-window metrics endpoints,
|
||||
restart/crash modes, mixed-version reader/writer/rollback participation, and
|
||||
allocation/flamegraph/RSS/save-frequency profile artifact plan. Synthetic runs
|
||||
and manifests missing that contract remain harness-only evidence.
|
||||
|
||||
### Recording One Case
|
||||
|
||||
@@ -243,64 +234,14 @@ For automation, `--check-scanner-heal-release "$RUN_DIR"` emits one compact
|
||||
JSON decision and exits nonzero while blocked. `verified_cases` contains only
|
||||
cases that pass the complete receipt, build provenance, nextest/JUnit and real
|
||||
oracle checks; `rejected_cases` names registered cases that do not, and
|
||||
`pending_gates` names the unimplemented release requirements and
|
||||
`pending_lanes` names the structured release lanes that still need real
|
||||
evidence. Schema 1 is deliberately marked `release_schema_capable: false`
|
||||
because it models only the single-version, unversioned-object restart/crash
|
||||
cases. Schema 2 can describe the wider release matrix, but approval still
|
||||
requires every registered case to verify and every required gate to leave
|
||||
`pending` only after a future checker can bind it to real feature-specific
|
||||
evidence. The current checker hard-rejects missing structured requirements and
|
||||
pending gates mapped to an implemented lane, so clearing pending text cannot
|
||||
become approval. A focused run, synthetic harness, compile-only result,
|
||||
skipped/retried test, ordinary CI success, or unregistered mixed-version,
|
||||
rollback, EC8+4 or performance claim therefore cannot become a release approval.
|
||||
For high-risk rollback gates, `evidence_fields` records the specific proof
|
||||
fields that a future real-evidence checker must bind before a pending gate can
|
||||
move out of the blocked set. G03 keeps scoped ACK tied to durable root
|
||||
publication, ACK request identity, participating peer capability snapshots, and
|
||||
mixed-peer fallback oracles; G09 keeps mixed-version reader, writer, and rollback
|
||||
payload evidence explicit. These fields are part of the release contract, not
|
||||
evidence by themselves.
|
||||
|
||||
When the real release lanes have produced their dedicated artifacts, validate
|
||||
the complete hard-gate bundle with:
|
||||
|
||||
```bash
|
||||
scripts/python_bin.sh scripts/check_test_wiring.py \
|
||||
--check-scanner-heal-release-bundle /path/to/release-evidence.json
|
||||
```
|
||||
|
||||
The bundle checker is intentionally stricter than the case checker. It requires
|
||||
schema 2 registry metadata, `evidence: measured`, the current checkout revision,
|
||||
all G01-G14/P1-P4/R-E/R-D/R-L gates, per-gate `status: pass`, lane identity,
|
||||
relative artifact paths, matching SHA256 hashes, and non-empty summaries. It
|
||||
also binds each evidence field to its own run provenance: `source_revision`,
|
||||
`run_id`, `measurement_window_id`, timezone-qualified `started_at` and
|
||||
`finished_at`, command arguments, and artifact format. The field
|
||||
`source_revision` must match the bundle revision, and measured performance
|
||||
duration cannot exceed the recorded run window.
|
||||
|
||||
The hard evidence shape remains claim-specific: mixed-version gates must name at
|
||||
least two participating versions, crash/durable replay gates must include
|
||||
crash-boundary evidence, G14 must record EC8+4 with at least three nodes and four
|
||||
drives per node plus multi-set and multi-pool evidence, performance gates need
|
||||
measured durations, P3's pressure run needs at least two hours, and P1 needs a
|
||||
symbolized profile summary with resolved samples. Every G14 field and every
|
||||
performance gate's fields must also share one `measurement_window_id`, so EC8+4,
|
||||
multi-set/multi-pool, ABBA, throughput, and profiling artifacts cannot be
|
||||
stitched together from unrelated runs. P1 `profile_evidence` must bind every
|
||||
required profile artifact kind (`allocation-profile`, `flamegraph`,
|
||||
`rss-samples`, and `save-frequency`) with a relative path, artifact format,
|
||||
non-empty file, matching SHA256, and the same measurement window when a
|
||||
per-artifact window is declared. Missing, synthetic, stale, tampered, undersized,
|
||||
or topology-mismatched evidence returns a compact blocked or invalid JSON result
|
||||
and a nonzero exit.
|
||||
|
||||
This command validates the evidence package; it does not create evidence. A
|
||||
handwritten JSON file, a synthetic harness pass, a single focused case, or a
|
||||
local unit fixture still cannot satisfy the distributed, mixed-version,
|
||||
crash-restart, durable MRF replay, EC8+4, ABBA, or profiling gates.
|
||||
`pending_gates` names the unimplemented release requirements. Approval requires
|
||||
every registered case to verify, `pending_gates` to be empty, and a future
|
||||
registry schema capable of representing the complete release matrix. Schema 1
|
||||
is deliberately marked `release_schema_capable: false`: it models only the
|
||||
single-version, unversioned-object restart/crash cases and cannot represent
|
||||
mixed-version, rollback, EC8+4 or performance evidence. A focused run,
|
||||
synthetic harness, compile-only result, skipped/retried test, ordinary CI
|
||||
success, or removal of pending text therefore cannot become a release approval.
|
||||
|
||||
Run parser/receipt regressions with
|
||||
`scripts/python_bin.sh scripts/check_test_wiring.py --self-test`. Those fixtures
|
||||
|
||||
@@ -12,52 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
fn git(root: &Path, args: &[&str]) -> Option<String> {
|
||||
let output = Command::new("git").args(args).current_dir(root).output().ok()?;
|
||||
output
|
||||
.status
|
||||
.success()
|
||||
.then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
|
||||
}
|
||||
|
||||
fn main() -> shadow_rs::SdResult<()> {
|
||||
let manifest = std::env::var_os("CARGO_MANIFEST_DIR").ok_or("missing CARGO_MANIFEST_DIR")?;
|
||||
let root = Path::new(&manifest).join("..");
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
// Match the E2E build identity's source and configuration inputs.
|
||||
for path in [
|
||||
"crates",
|
||||
"rustfs",
|
||||
"Cargo.toml",
|
||||
"Cargo.lock",
|
||||
"rust-toolchain.toml",
|
||||
".cargo",
|
||||
".config",
|
||||
] {
|
||||
let path = root.join(path);
|
||||
if path.exists() {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
}
|
||||
// Resolve worktree-local HEAD/index and common refs through Git. Watching
|
||||
// refs also covers tags and a new loose ref created from packed refs.
|
||||
for name in ["HEAD", "index", "refs", "packed-refs"] {
|
||||
if let Some(path) = git(&root, &["rev-parse", "--git-path", name]) {
|
||||
let path = Path::new(&path);
|
||||
let path = if path.is_absolute() {
|
||||
path.to_owned()
|
||||
} else {
|
||||
root.join(path)
|
||||
};
|
||||
if path.exists() {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("cargo:rerun-if-env-changed=RUSTFS_BUILD_VERSION");
|
||||
if let Ok(version) = std::env::var("RUSTFS_BUILD_VERSION")
|
||||
&& !version.is_empty()
|
||||
|
||||
@@ -627,6 +627,32 @@ pub(crate) async fn cluster_replication_stats(bucket: &str, context: Option<Arc<
|
||||
.await
|
||||
}
|
||||
|
||||
/// Reload the bucket's metadata on every peer so a follow-up
|
||||
/// `put-bucket-replication` on another node does not read a stale target.
|
||||
///
|
||||
/// Best effort, like every S3 bucket-config write path
|
||||
/// (`app::bucket_usecase::notify_bucket_metadata_reload`): the target is
|
||||
/// already persisted and live on this node, and the 15-minute refresh closes
|
||||
/// the gap, so a peer that cannot be reached must not turn a completed write
|
||||
/// into a failed request.
|
||||
async fn notify_remote_target_metadata_reload(bucket: &str, context: Option<Arc<AppContext>>, action: &'static str) {
|
||||
let Some(notification_system) = current_notification_system_for_context(context.as_deref()) else {
|
||||
return;
|
||||
};
|
||||
if let Err(err) = notification_system.load_bucket_metadata(bucket).await {
|
||||
warn!(
|
||||
event = EVENT_ADMIN_REMOTE_TARGET_STATE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
action = action,
|
||||
result = "peer_metadata_reload_failed",
|
||||
bucket = %bucket,
|
||||
error = ?err,
|
||||
"admin remote target state"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn unique_replication_peers(peer_clients: &[Option<PeerRestClient>]) -> (Vec<&PeerRestClient>, u32) {
|
||||
let mut seen_grid_hosts = HashSet::new();
|
||||
let peers: Vec<_> = peer_clients
|
||||
@@ -699,6 +725,7 @@ pub struct SetRemoteTargetHandler {}
|
||||
impl Operation for SetRemoteTargetHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let cred = validate_replication_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
|
||||
let app_context = app_context_from_req(&req);
|
||||
|
||||
let queries = extract_query_params(&req.uri);
|
||||
|
||||
@@ -926,6 +953,8 @@ impl Operation for SetRemoteTargetHandler {
|
||||
.map_err(map_bucket_target_error)?;
|
||||
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
|
||||
let arn = persist_remote_target_write(bucket, remote_target, incarnation, mode).await?;
|
||||
drop(_targets_guard);
|
||||
notify_remote_target_metadata_reload(bucket, app_context, "set_remote_target").await;
|
||||
let arn_str = serde_json::to_string(&arn)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize target ARN"))?;
|
||||
|
||||
@@ -1006,6 +1035,7 @@ pub struct RemoveRemoteTargetHandler {}
|
||||
impl Operation for RemoveRemoteTargetHandler {
|
||||
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
validate_replication_admin_request(&req, AdminAction::SetBucketTargetAction).await?;
|
||||
let app_context = app_context_from_req(&req);
|
||||
|
||||
debug!("remove remote target called");
|
||||
let queries = extract_query_params(&req.uri);
|
||||
@@ -1081,6 +1111,7 @@ impl Operation for RemoveRemoteTargetHandler {
|
||||
}
|
||||
let json_targets = serde_json::to_vec(&targets)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "Failed to serialize targets"))?;
|
||||
let notification_bucket = bucket.clone();
|
||||
let bucket = bucket.clone();
|
||||
let arn = arn_str.clone();
|
||||
// The pool cancellation owns a detached task. Both outer guards must
|
||||
@@ -1101,6 +1132,8 @@ impl Operation for RemoveRemoteTargetHandler {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, format!("remote target removal task failed: {error}"))
|
||||
})??;
|
||||
|
||||
notify_remote_target_metadata_reload(¬ification_bucket, app_context, "remove_remote_target").await;
|
||||
|
||||
Ok(S3Response::new((StatusCode::NO_CONTENT, Body::from("".to_string()))))
|
||||
}
|
||||
}
|
||||
@@ -1787,6 +1820,25 @@ mod tests {
|
||||
pairs.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_target_writes_notify_peer_metadata_caches() {
|
||||
let source = include_str!("replication.rs");
|
||||
for (start, end) in [
|
||||
("impl Operation for SetRemoteTargetHandler", "pub struct ListRemoteTargetHandler"),
|
||||
("impl Operation for RemoveRemoteTargetHandler", "async fn cancel_active_resync_intent"),
|
||||
] {
|
||||
let body = source
|
||||
.split(start)
|
||||
.nth(1)
|
||||
.and_then(|rest| rest.split(end).next())
|
||||
.expect(start);
|
||||
assert!(
|
||||
body.contains("notify_remote_target_metadata_reload"),
|
||||
"{start} must notify every node before returning success"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_ops_parse_minio_query_contract() {
|
||||
let ops = parse_remote_target_update_ops(&query_map(&[
|
||||
|
||||
@@ -75,10 +75,6 @@ struct ScannerRecoveryIntentResponse {
|
||||
mode: String,
|
||||
intent_id: String,
|
||||
state: String,
|
||||
actor_sha256: String,
|
||||
idempotency_key_sha256: String,
|
||||
request_sha256: String,
|
||||
accepted_at_unix_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -336,10 +332,6 @@ fn scanner_recovery_intent_record_response(
|
||||
mode: record.mode,
|
||||
intent_id: record.intent_id,
|
||||
state: record.state,
|
||||
actor_sha256: record.actor_sha256,
|
||||
idempotency_key_sha256: record.idempotency_key_sha256,
|
||||
request_sha256: record.request_sha256,
|
||||
accepted_at_unix_secs: record.accepted_at_unix_secs,
|
||||
};
|
||||
let body = serde_json::to_vec(&response).map_err(|err| {
|
||||
S3Error::with_message(
|
||||
@@ -379,7 +371,10 @@ fn scanner_recovery_intent_accept_response(
|
||||
|
||||
fn scanner_recovery_intent_executor_id(result: &rustfs_scanner::ScannerRecoveryIntentAcceptResult) -> Option<String> {
|
||||
match result {
|
||||
rustfs_scanner::ScannerRecoveryIntentAcceptResult::Accepted { record } if record.state == "accepted" => {
|
||||
rustfs_scanner::ScannerRecoveryIntentAcceptResult::Accepted { record }
|
||||
| rustfs_scanner::ScannerRecoveryIntentAcceptResult::Replayed { record }
|
||||
if matches!(record.state.as_str(), "accepted" | "running") =>
|
||||
{
|
||||
Some(record.intent_id.clone())
|
||||
}
|
||||
_ => None,
|
||||
@@ -678,7 +673,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_recovery_intent_executor_starts_only_newly_accepted_work() {
|
||||
fn scanner_recovery_intent_executor_only_starts_non_terminal_work() {
|
||||
let mut record = rustfs_scanner::ScannerRecoveryIntentRecord {
|
||||
schema_version: 1,
|
||||
intent_id: "0".repeat(64),
|
||||
@@ -704,16 +699,7 @@ mod tests {
|
||||
record: record.clone(),
|
||||
})
|
||||
.as_deref(),
|
||||
None,
|
||||
"a lost-response retry must not start a duplicate executor"
|
||||
);
|
||||
record.state = "accepted".to_string();
|
||||
assert!(
|
||||
scanner_recovery_intent_executor_id(&rustfs_scanner::ScannerRecoveryIntentAcceptResult::Replayed {
|
||||
record: record.clone(),
|
||||
})
|
||||
.is_none(),
|
||||
"replayed accepted records remain durable for startup/control recovery instead of duplicating work"
|
||||
Some(record.intent_id.as_str())
|
||||
);
|
||||
record.state = "completed".to_string();
|
||||
assert!(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1318,6 +1318,24 @@ impl Operation for ImportIam {
|
||||
failed,
|
||||
};
|
||||
|
||||
// The entities are already imported locally. A snapshot that cannot be
|
||||
// scheduled is a convergence delay the reconcile pass still closes, so
|
||||
// it must not turn a completed import into a failed request - the same
|
||||
// best-effort contract every other site-replication hook here follows.
|
||||
if let Err(err) =
|
||||
crate::site_replication::enqueue_site_replication_iam_snapshot("iam import scheduled a full snapshot").await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_USER,
|
||||
event = EVENT_ADMIN_USER_STATE,
|
||||
action = "import_iam",
|
||||
result = "site_replication_snapshot_not_scheduled",
|
||||
error = ?err,
|
||||
"admin user state"
|
||||
);
|
||||
}
|
||||
|
||||
let body = serde_json::to_vec(&ret).map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
|
||||
|
||||
let mut header = HeaderMap::new();
|
||||
@@ -1424,6 +1442,16 @@ mod tests {
|
||||
assert!(include_str!("user.rs").contains(mapper_call));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_iam_enqueues_a_site_replication_snapshot() {
|
||||
let body = source_block(include_str!("user.rs"), "impl Operation for ImportIam");
|
||||
|
||||
assert!(
|
||||
body.contains("enqueue_site_replication_iam_snapshot"),
|
||||
"a successful IAM import must schedule a full IAM snapshot for every remote site"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_check_deny_only_for_regular_self_request() {
|
||||
let cred = Credentials {
|
||||
|
||||
@@ -409,6 +409,26 @@ fn transfer_summaries(stats: &InternalReplicationStats) -> (XferSummaryWire, Tar
|
||||
(summary, per_target)
|
||||
}
|
||||
|
||||
/// Node-level failure counters for `errors`. The sibling `retries` field
|
||||
/// stays zero on purpose: it means redeliveries in the minio-go shape, and a
|
||||
/// failed object is not retried by an event today (it waits for the scanner's
|
||||
/// heal pass), so reporting failures there would claim a redelivery that
|
||||
/// never happened.
|
||||
fn failure_counters(stats: &InternalReplicationStats) -> CounterSummaryWire {
|
||||
let (total, last1m, last1hr) = stats.stats.values().fold((0i64, 0i64, 0i64), |acc, stat| {
|
||||
(
|
||||
acc.0.saturating_add(stat.fail_stats.count),
|
||||
acc.1.saturating_add(stat.fail_stats.last_minute.count),
|
||||
acc.2.saturating_add(stat.fail_stats.last_hour.count),
|
||||
)
|
||||
});
|
||||
CounterSummaryWire {
|
||||
total: u64::try_from(total.max(0)).unwrap_or_default(),
|
||||
last1m: u64::try_from(last1m.max(0)).unwrap_or_default(),
|
||||
last1hr: u64::try_from(last1hr.max(0)).unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
impl MetricsV2Wire {
|
||||
/// Project the aggregated internal stats onto the `MetricsV2` shape.
|
||||
///
|
||||
@@ -418,6 +438,7 @@ impl MetricsV2Wire {
|
||||
/// `queueStats.nodes` and treats an empty list as "no data".
|
||||
pub(crate) fn from_stats(bucket_stats: &BucketStats, node_name: &str) -> Self {
|
||||
let (xfer_stats, tgt_xfer_stats) = transfer_summaries(&bucket_stats.replication_stats);
|
||||
let failed = failure_counters(&bucket_stats.replication_stats);
|
||||
let mut nodes: Vec<ReplQNodeStatsWire> = bucket_stats
|
||||
.queue_stats
|
||||
.nodes
|
||||
@@ -436,6 +457,7 @@ impl MetricsV2Wire {
|
||||
q_stats: InQueueMetricWire::from(&bucket_stats.replication_stats.q_stat),
|
||||
xfer_stats: xfer_stats.clone(),
|
||||
tgt_xfer_stats: tgt_xfer_stats.clone(),
|
||||
errors: failed,
|
||||
..Default::default()
|
||||
});
|
||||
} else {
|
||||
@@ -444,6 +466,7 @@ impl MetricsV2Wire {
|
||||
if let Some(first) = nodes.first_mut() {
|
||||
first.xfer_stats = xfer_stats.clone();
|
||||
first.tgt_xfer_stats = tgt_xfer_stats.clone();
|
||||
first.errors = failed;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -478,6 +501,12 @@ mod tests {
|
||||
target.replicated_size = 4096;
|
||||
target.failed.count = 3;
|
||||
target.failed.size = 900;
|
||||
target.fail_stats.count = 3;
|
||||
target.fail_stats.size = 900;
|
||||
target.fail_stats.last_minute.count = 2;
|
||||
target.fail_stats.last_minute.size = 600;
|
||||
target.fail_stats.last_hour.count = 3;
|
||||
target.fail_stats.last_hour.size = 900;
|
||||
target.bandwidth_limit_bytes_per_sec = 1024;
|
||||
target.current_bandwidth_bytes_per_sec = 512.5;
|
||||
stats
|
||||
@@ -537,6 +566,10 @@ mod tests {
|
||||
assert_eq!(node["queueStats"]["peak"], node["queueStats"]["max"]);
|
||||
assert!(node["activeWorkers"].get("curr").is_some());
|
||||
assert!(node["transferSummary"].get("Total").is_some());
|
||||
assert_eq!(node["errors"]["total"], 3);
|
||||
assert_eq!(node["errors"]["last1m"], 2);
|
||||
assert_eq!(node["errors"]["last1hr"], 3);
|
||||
assert_eq!(node["retries"]["total"], 0, "failures are not redeliveries; retries must not claim one");
|
||||
assert_eq!(json["downtimeInfo"], serde_json::json!({}));
|
||||
}
|
||||
|
||||
|
||||
@@ -814,11 +814,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject,
|
||||
);
|
||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
||||
Ok::<_, S3Error>((oi, dest_versioned))
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1175,12 +1175,7 @@ impl DefaultObjectUsecase {
|
||||
let manager = get_capacity_manager();
|
||||
manager.record_write_operation().await;
|
||||
let _ = helper.complete(&result);
|
||||
let producer = if delete_marker && version_id_clone.is_none() {
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteMarker
|
||||
} else {
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::DeleteObject
|
||||
};
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(&bucket, &key, producer);
|
||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
@@ -689,11 +689,7 @@ impl DefaultObjectUsecase {
|
||||
schedule_object_replication(obj_info.clone(), store, completion_replication_decision).await;
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::CompleteMultipartUpload,
|
||||
);
|
||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
||||
Ok::<_, ApiError>(obj_info)
|
||||
}
|
||||
});
|
||||
|
||||
@@ -2054,11 +2054,7 @@ impl DefaultObjectUsecase {
|
||||
schedule_object_replication(obj_info.clone(), store, dsc).await;
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::PutObject,
|
||||
);
|
||||
rustfs_scanner::record_dirty_usage_object(&bucket, &key);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_post_store_bookkeeping", post_store_stage_start);
|
||||
|
||||
let capacity_update_stage_start = put_stage_metrics_enabled.then(Instant::now);
|
||||
|
||||
@@ -217,6 +217,26 @@ pub(crate) fn settle_observed_site_replication_retry_event(
|
||||
before.saturating_sub(queue.len())
|
||||
}
|
||||
|
||||
/// Make sure `peer` has a collapsed entry for `path` without counting the
|
||||
/// call as a delivery failure. A bulk local mutation (`import-iam`) needs the
|
||||
/// entry to exist so the next drain sends the snapshot; routing it through
|
||||
/// [`upsert_site_replication_retry_event`] would raise `retry_count` on every
|
||||
/// import and escalate a healthy peer to `failed` after
|
||||
/// [`SITE_REPLICATION_RETRY_FAILED_AFTER`] of them, with the scheduling note
|
||||
/// shown to operators as `lastError`.
|
||||
pub(crate) fn ensure_site_replication_retry_event(
|
||||
queue: &mut Vec<SiteReplicationRetryEvent>,
|
||||
peer: &PeerInfo,
|
||||
path: &str,
|
||||
reason: &str,
|
||||
) -> S3Result<Vec<SiteReplicationRetryEvent>> {
|
||||
let path = collapsed_retry_queue_path(path).unwrap_or(path);
|
||||
if queue.iter().any(|event| retry_event_matches(event, peer, path)) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
push_site_replication_retry_event(queue, peer, path, summarize_peer_error_detail(reason), false, None)
|
||||
}
|
||||
|
||||
pub(crate) fn upsert_site_replication_retry_event(
|
||||
queue: &mut Vec<SiteReplicationRetryEvent>,
|
||||
peer: &PeerInfo,
|
||||
@@ -244,6 +264,17 @@ pub(crate) fn upsert_site_replication_retry_event(
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
push_site_replication_retry_event(queue, peer, path, detail, peer_unreachable, generation)
|
||||
}
|
||||
|
||||
fn push_site_replication_retry_event(
|
||||
queue: &mut Vec<SiteReplicationRetryEvent>,
|
||||
peer: &PeerInfo,
|
||||
path: &str,
|
||||
detail: String,
|
||||
peer_unreachable: bool,
|
||||
generation: Option<u64>,
|
||||
) -> S3Result<Vec<SiteReplicationRetryEvent>> {
|
||||
let slots_needed = queue
|
||||
.len()
|
||||
.saturating_add(1)
|
||||
@@ -274,7 +305,7 @@ pub(crate) fn upsert_site_replication_retry_event(
|
||||
retry_count: 1,
|
||||
failed: false,
|
||||
last_error: detail,
|
||||
updated_at: Some(now),
|
||||
updated_at: Some(OffsetDateTime::now_utc()),
|
||||
edit_generation: generation,
|
||||
peer_unreachable,
|
||||
deletions_recorded: false,
|
||||
@@ -365,6 +396,60 @@ pub(crate) async fn enqueue_site_replication_retry_event_for_generation(
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of peers whose snapshot entry is escalated and therefore
|
||||
/// will not carry this scheduling: the marker records a deletion that a
|
||||
/// snapshot cannot replay, and only a repair settles it, so clearing it to make
|
||||
/// the entry drainable again would drop that liability.
|
||||
pub(crate) fn record_iam_snapshot_retries(
|
||||
state: &mut SiteReplicationState,
|
||||
local_peer: &PeerInfo,
|
||||
reason: &str,
|
||||
) -> S3Result<usize> {
|
||||
let peers = state
|
||||
.peers
|
||||
.values()
|
||||
.filter(|peer| {
|
||||
peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let mut escalated = 0usize;
|
||||
for peer in peers {
|
||||
if state.retry_queue.iter().any(|event| {
|
||||
retry_event_matches(event, &peer, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH)
|
||||
&& event.last_error == SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER
|
||||
}) {
|
||||
escalated += 1;
|
||||
continue;
|
||||
}
|
||||
ensure_site_replication_retry_event(&mut state.retry_queue, &peer, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH, reason)?;
|
||||
}
|
||||
Ok(escalated)
|
||||
}
|
||||
|
||||
/// Schedule one collapsed full-IAM snapshot per remote peer after a bulk
|
||||
/// local mutation such as `import-iam`.
|
||||
pub(crate) async fn enqueue_site_replication_iam_snapshot(reason: &str) -> S3Result<()> {
|
||||
let state = load_site_replication_state().await?;
|
||||
if !state.enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
let local_peer = current_local_runtime_peer(&state);
|
||||
let reason = reason.to_string();
|
||||
let escalated = update_site_replication_state(move |state| record_iam_snapshot_retries(state, &local_peer, &reason)).await?;
|
||||
if escalated > 0 {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
|
||||
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
|
||||
escalated,
|
||||
result = "iam_snapshot_not_scheduled_for_escalated_peer",
|
||||
"site replication peers hold an escalated IAM entry; the snapshot waits for a repair"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) const SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH: &str = "/rustfs/admin/v3/site-replication/peer/iam-item";
|
||||
|
||||
/// Per-peer cap on recorded deletion bodies. Beyond it the peer's collapsed
|
||||
|
||||
@@ -168,6 +168,8 @@ mod rfc3339_map {
|
||||
pub(crate) struct PendingEndpointRefresh {
|
||||
pub(crate) id: String,
|
||||
pub(crate) peer: PeerInfo,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) ilm_expiry_override: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub(crate) remote_peers: BTreeMap<String, PeerInfo>,
|
||||
#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user