mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 03:59:14 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 48f0f04e0c | |||
| cef1b21638 |
@@ -1,2 +0,0 @@
|
||||
sha256-linux=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
|
||||
sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
|
||||
@@ -40,21 +40,6 @@
|
||||
"invariant": "corrupt-part-arrays",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
|
||||
},
|
||||
{
|
||||
"invariant": "odm-source-contract-s3",
|
||||
"suite": "rustfs",
|
||||
"name": "on_demand_migration::source_client::tests::s3_backend_satisfies_the_shared_backend_contract"
|
||||
},
|
||||
{
|
||||
"invariant": "odm-source-contract-azure",
|
||||
"suite": "rustfs",
|
||||
"name": "on_demand_migration::azure::tests::azure_backend_satisfies_the_shared_backend_contract"
|
||||
},
|
||||
{
|
||||
"invariant": "odm-source-contract-gcs",
|
||||
"suite": "rustfs",
|
||||
"name": "on_demand_migration::gcs::tests::gcs_native_backend_satisfies_the_shared_backend_contract"
|
||||
}
|
||||
],
|
||||
"fixtures": [
|
||||
|
||||
@@ -3,10 +3,9 @@
|
||||
.NOTPARALLEL: pre-commit pre-pr dev-check
|
||||
|
||||
.PHONY: setup-hooks
|
||||
setup-hooks: ## Install the configured pre-commit hooks
|
||||
setup-hooks: ## Set up git hooks
|
||||
@echo "🔧 Setting up git hooks..."
|
||||
pre-commit validate-config
|
||||
pre-commit install
|
||||
chmod +x .git/hooks/pre-commit
|
||||
@echo "✅ Git hooks setup complete!"
|
||||
|
||||
.PHONY: doc-paths-check
|
||||
|
||||
@@ -31,7 +31,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
./scripts/test_hotpath_warp_abba.sh
|
||||
./scripts/test_scanner_validation_harness.sh
|
||||
./scripts/test_exact_1mib_handoff_abba.sh
|
||||
./scripts/test_pinned_paired_abba_bench.sh
|
||||
./scripts/test_manual_transition_runbooks.sh
|
||||
@@ -42,7 +41,6 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
# Committed floor for the number of tests selected by the migration-critical
|
||||
# CI gate (see scripts/check_migration_gate_count.sh, backlog#1153 infra-12).
|
||||
#
|
||||
# The floor equals the exact count of rustfs-ecstore --lib tests, with the
|
||||
# test-util feature enabled, matching the gate filter (name substrings:
|
||||
# data_movement, rebalance, decommission, source_cleanup, delete_marker) at
|
||||
# the time this file was last updated.
|
||||
# The floor equals the exact count of rustfs-ecstore --lib tests matching the
|
||||
# gate filter (name substrings: data_movement, rebalance, decommission,
|
||||
# source_cleanup, delete_marker) at the time this file was last updated.
|
||||
# CI fails if the selected count drops below this number, so renames or
|
||||
# removals that thin the gate must update this file in the same PR.
|
||||
# Adding tests does not require a bump, but bumping keeps the guard tight.
|
||||
946
|
||||
571
|
||||
|
||||
@@ -183,13 +183,6 @@ test-group = 'e2e-reliability'
|
||||
filter = 'package(e2e_test) & test(/^inline_fast_path_cluster_test::/)'
|
||||
test-group = 'e2e-inline-boundaries'
|
||||
|
||||
# 4-node 4-drive distributed Actions suite: each case starts four rustfs
|
||||
# processes and up to sixteen data directories. Serialize across nextest's
|
||||
# process boundary so several 4x4 clusters never overlap.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^distributed::/)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# Vault KMS tests share the fixed dev-server port 8200. serial_test's #[serial]
|
||||
# does not cross nextest process boundaries, so keep every Vault-backed test in
|
||||
# one group.
|
||||
@@ -533,27 +526,6 @@ path = "junit.xml"
|
||||
filter = 'package(e2e_test)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
# Serialized via e2e-cluster-nightly with no retries.
|
||||
[profile.e2e-distributed]
|
||||
default-filter = 'package(e2e_test) & test(/^distributed::/)'
|
||||
fail-fast = false
|
||||
# Decommission / rebalance cases poll for up to 180s with little stdout.
|
||||
slow-timeout = { period = "120s", terminate-after = 6 }
|
||||
|
||||
[profile.e2e-distributed.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
[[profile.e2e-distributed.overrides]]
|
||||
filter = 'package(e2e_test)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -614,10 +586,6 @@ path = "junit.xml"
|
||||
# cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
|
||||
# excluded here because backlog#2213 promotes core heal rebuild coverage to
|
||||
# this merge/main lane while retaining nightly coverage.
|
||||
# * distributed:: — 4-node 4-disk Actions suite (S3, lock, versioning,
|
||||
# replication, quota, observability, expand/decommission/rebalance, site
|
||||
# replication, chaos, upgrade history/IAM). Owns [profile.e2e-distributed] and
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# * on_demand_migration::interop_test — the ODM-20 provider interoperability
|
||||
# cases, which are meaningless without a source: they run in the dedicated
|
||||
# [profile.e2e-odm-interop] lane below, where the workflow points them at a
|
||||
@@ -639,7 +607,6 @@ default-filter = """
|
||||
package(e2e_test)
|
||||
& !test(/^protocols::/)
|
||||
& !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
|
||||
& !test(/^distributed::/)
|
||||
& !test(/^replication_extension_test::/)
|
||||
& !test(/^replication_target_matrix_test::/)
|
||||
& !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"cases": {
|
||||
"background-target-restart": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart",
|
||||
"oracle": "background-target-restart.json",
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
}
|
||||
},
|
||||
"release_pending": {
|
||||
"G01": "W02/W04 complete root and quota authority coverage",
|
||||
"G02": "W03 bounded checkpoint progress and independent version inventory",
|
||||
"G03": "W17/W18 exact scoped ACK with durable publication and mixed peers",
|
||||
"G04": "W03/W15/W16 crash at every cache/root/floor/intent boundary",
|
||||
"G05": "W06/W07 per-object outcomes and bounded terminal retention",
|
||||
"G06": "W06/W08/W23 concurrent status, legacy clients and truncation",
|
||||
"G07": "W12/W13/W14 durable MRF responsibility at every commit boundary",
|
||||
"G08": "W12/W13/W14 MRF capacity, disk-full and replica-loss matrix",
|
||||
"G09": "W13/W18/W23 actual mixed-version reader/writer and rollback payloads",
|
||||
"G10": "W05/W09/W10/W11 bounded scheduling and pressure recovery",
|
||||
"G11": "W04/W19/W24 maintenance and complete producer coverage",
|
||||
"G12": "W02/W15/W16 both quota paths during reset and settlement",
|
||||
"G13": "W07/W14 quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
||||
"G14": "W20/W21 same-window field evidence; 3x4 EC8+4 and multi-set/pool coverage",
|
||||
"P1": "W20 measured cold-walk share and foreground latency/throughput",
|
||||
"P2": "W20/W24 measured post-stop convergence and cold segment reuse",
|
||||
"P3": "W20 measured two-hour pressure/heal capacity and recovery window",
|
||||
"P4": "W20 measured MRF scale and replay cost with retained responsibility",
|
||||
"R-E": "W03/W05 fixed-budget real process restart through enumeration and classification",
|
||||
"R-D": "W07/W14 manager-to-event-to-ledger exact disposition, including grace",
|
||||
"R-L": "W13/W14 legacy source conflicts, migration gaps and crash-safe source retirement"
|
||||
}
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Quick Checks
|
||||
description: Run the shared compile-free RustFS quality checks.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install quality tools
|
||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
||||
with:
|
||||
tool: |
|
||||
ripgrep@15.2.0
|
||||
shellcheck@0.11.0
|
||||
|
||||
- name: Install actionlint
|
||||
shell: bash
|
||||
run: |
|
||||
actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
|
||||
curl --fail --location --silent --show-error \
|
||||
--output "$actionlint_dir/actionlint.tar.gz" \
|
||||
https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
|
||||
echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
|
||||
tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
|
||||
rm "$actionlint_dir/actionlint.tar.gz"
|
||||
echo "$actionlint_dir" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
with:
|
||||
components: rustfmt
|
||||
|
||||
- name: Check workflow syntax and shell scripts
|
||||
shell: bash
|
||||
run: shellcheck --version && actionlint
|
||||
|
||||
- name: Check code formatting
|
||||
shell: bash
|
||||
run: cargo fmt --all --check
|
||||
|
||||
- name: Check unsafe code allowances
|
||||
shell: bash
|
||||
run: ./scripts/check_unsafe_code_allowances.sh
|
||||
|
||||
- name: Check layered dependencies
|
||||
shell: bash
|
||||
run: ./scripts/check_layer_dependencies.sh
|
||||
|
||||
- name: Check architecture migration rules
|
||||
shell: bash
|
||||
run: ./scripts/check_architecture_migration_rules.sh
|
||||
|
||||
- name: Check logging guardrails
|
||||
shell: bash
|
||||
run: ./scripts/check_logging_guardrails.sh
|
||||
|
||||
- name: Check error other(format!) ratchet
|
||||
shell: bash
|
||||
run: ./scripts/check_error_other_format_ratchet.sh
|
||||
|
||||
- name: Check tokio io-uring feature guard
|
||||
shell: bash
|
||||
run: ./scripts/check_no_tokio_io_uring.sh
|
||||
|
||||
- name: Check extension schema boundaries
|
||||
shell: bash
|
||||
run: ./scripts/check_extension_schema_boundaries.sh
|
||||
|
||||
- name: Check body-cache whitelist guard
|
||||
shell: bash
|
||||
run: ./scripts/check_body_cache_whitelist.sh
|
||||
|
||||
- name: Check s3s footprint ratchet
|
||||
shell: bash
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
shell: bash
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
- name: Check no embedded secret material
|
||||
shell: bash
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Run script contract tests
|
||||
shell: bash
|
||||
run: make script-tests
|
||||
|
||||
- name: Check test wiring
|
||||
shell: bash
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/test_security_workflow.py
|
||||
python3 ./scripts/test_nightly_candidate.py
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
shell: bash
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
- name: Check CI paths stay in sync
|
||||
shell: bash
|
||||
run: ./scripts/check_ci_paths_sync.sh
|
||||
|
||||
- name: Check io_uring lane --lib precondition
|
||||
shell: bash
|
||||
run: ./scripts/check_uring_lane_lib_only.sh
|
||||
@@ -10,16 +10,16 @@ Use N/A when there is no related issue.
|
||||
|
||||
## Summary of Changes
|
||||
<!--
|
||||
Describe the concrete problem and resulting behavior. For a behavior change, name the input or state that triggers it and the expected outcome. Explain any new dependency or abstraction that the change needs.
|
||||
Briefly explain what changed and why reviewers should accept it.
|
||||
Focus on behavior, compatibility, and review-relevant context.
|
||||
-->
|
||||
|
||||
## Verification
|
||||
<!--
|
||||
Give 1–3 concrete pieces of evidence for the changed behavior: the test or command, its observed result, and the regression it catches. For a bug fix, record a failing-before/passing-after check or explain why it was unavailable.
|
||||
List the commands or checks you ran, for example:
|
||||
- `make pre-commit`
|
||||
|
||||
Identify the tested commit and any local changes. When testing a prebuilt binary or external service, include its source/version and artifact identity; a successful run against a different build is not evidence for this change.
|
||||
|
||||
List relevant checks not run and the remaining risk. Use the validation tier in AGENTS.md; do not run broader checks solely to fill this section. For documentation-only changes, list the applicable documentation checks. Use N/A only when verification is not applicable.
|
||||
Use N/A only when verification is not applicable.
|
||||
-->
|
||||
|
||||
## Impact
|
||||
|
||||
@@ -4,11 +4,6 @@
|
||||
{ "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
|
||||
{
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"max_age_hours": 36,
|
||||
"never_ran_grace_until": "2026-09-18T00:00:00Z"
|
||||
},
|
||||
{ "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
|
||||
|
||||
@@ -12,10 +12,24 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Reports the existing required checks for paths excluded by ci.yml.
|
||||
# Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared
|
||||
# action to keep validation coverage aligned. Keep this paths list in sync with
|
||||
# ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh.
|
||||
# Companion to ci.yml for required status checks.
|
||||
#
|
||||
# ci.yml skips docs-only pull requests via paths-ignore, but the branch ruleset
|
||||
# requires a check named "Test and Lint" — without this workflow a docs-only PR
|
||||
# would wait on it forever. This workflow triggers on exactly the paths ci.yml
|
||||
# ignores and reports success under the same job name. Mixed PRs trigger both
|
||||
# workflows and the real check still gates: a required check with any failing
|
||||
# run blocks the merge.
|
||||
# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
|
||||
#
|
||||
# "Quick Checks" is mirrored here ahead of the ruleset change that will make it
|
||||
# required too (rustfs/backlog#1599). Until that change lands this job is
|
||||
# inert; mirroring it first is what lets the ruleset change happen without
|
||||
# stranding docs-only PRs on a check nobody reports.
|
||||
#
|
||||
# Keep the paths list below in sync with the pull_request paths-ignore list
|
||||
# in ci.yml, and keep the quick-checks steps below byte-identical to the
|
||||
# quick-checks job in ci.yml.
|
||||
|
||||
name: Continuous Integration (docs only)
|
||||
|
||||
@@ -45,6 +59,19 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Deliberately NOT a bare `echo`. Once "Quick Checks" becomes a required
|
||||
# check, ci.yml gates every expensive job behind it, so a mixed PR reports
|
||||
# two check runs with this name: the real one (45-51s) and this companion.
|
||||
# GitHub has no written contract for how it picks between same-named
|
||||
# required check runs ("latest wins" vs "any failure blocks"), so instead of
|
||||
# relying on ordering we make both runs execute the same commands against
|
||||
# the same merge ref — their conclusions are then necessarily identical and
|
||||
# the choice does not matter. Keep these steps byte-identical to the
|
||||
# quick-checks job in ci.yml (a guard script that asserts this, and the paths
|
||||
# sync below, is tracked in rustfs/backlog#1603).
|
||||
#
|
||||
# For a genuinely docs-only PR this adds no strictness (no code changed, so
|
||||
# fmt and the guards always pass) and costs ~50s of ubuntu-latest.
|
||||
quick-checks:
|
||||
name: Quick Checks
|
||||
runs-on: ubuntu-latest
|
||||
@@ -55,8 +82,64 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
- name: Install ripgrep
|
||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
||||
with:
|
||||
tool: ripgrep@15.2.0
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
with:
|
||||
components: rustfmt
|
||||
|
||||
- name: Check code formatting
|
||||
run: cargo fmt --all --check
|
||||
|
||||
- name: Check unsafe code allowances
|
||||
run: ./scripts/check_unsafe_code_allowances.sh
|
||||
|
||||
- name: Check layered dependencies
|
||||
run: ./scripts/check_layer_dependencies.sh
|
||||
|
||||
- name: Check architecture migration rules
|
||||
run: ./scripts/check_architecture_migration_rules.sh
|
||||
|
||||
- name: Check logging guardrails
|
||||
run: ./scripts/check_logging_guardrails.sh
|
||||
|
||||
- name: Check tokio io-uring feature guard
|
||||
run: ./scripts/check_no_tokio_io_uring.sh
|
||||
|
||||
- name: Check extension schema boundaries
|
||||
run: ./scripts/check_extension_schema_boundaries.sh
|
||||
|
||||
- name: Check body-cache whitelist guard
|
||||
run: ./scripts/check_body_cache_whitelist.sh
|
||||
|
||||
- name: Check s3s footprint ratchet
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
- name: Check no embedded secret material
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Check test wiring
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/test_security_workflow.py
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
- name: Check CI paths stay in sync
|
||||
run: ./scripts/check_ci_paths_sync.sh
|
||||
|
||||
- name: Check io_uring lane --lib precondition
|
||||
run: ./scripts/check_uring_lane_lib_only.sh
|
||||
|
||||
test-and-lint:
|
||||
name: Test and Lint
|
||||
|
||||
@@ -100,7 +100,12 @@ jobs:
|
||||
- name: Typos check with custom config file
|
||||
uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
|
||||
|
||||
# Fail early with compile-free checks shared with docs-only CI.
|
||||
# Fast, compile-free checks that fail early so contributors get feedback in
|
||||
# ~1 minute instead of waiting for the full test job.
|
||||
#
|
||||
# These steps are mirrored byte-for-byte in ci-docs-only.yml so that a mixed
|
||||
# PR, which reports two check runs named "Quick Checks", cannot get one red
|
||||
# and one green. Edit both jobs together.
|
||||
quick-checks:
|
||||
name: Quick Checks
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
@@ -112,8 +117,67 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
- name: Install ripgrep
|
||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
||||
with:
|
||||
tool: ripgrep@15.2.0
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
with:
|
||||
components: rustfmt
|
||||
|
||||
- name: Check code formatting
|
||||
run: cargo fmt --all --check
|
||||
|
||||
- name: Check unsafe code allowances
|
||||
run: ./scripts/check_unsafe_code_allowances.sh
|
||||
|
||||
- name: Check layered dependencies
|
||||
run: ./scripts/check_layer_dependencies.sh
|
||||
|
||||
- name: Check architecture migration rules
|
||||
run: ./scripts/check_architecture_migration_rules.sh
|
||||
|
||||
- name: Check logging guardrails
|
||||
run: ./scripts/check_logging_guardrails.sh
|
||||
|
||||
- name: Check error other(format!) ratchet
|
||||
run: ./scripts/check_error_other_format_ratchet.sh
|
||||
|
||||
- name: Check tokio io-uring feature guard
|
||||
run: ./scripts/check_no_tokio_io_uring.sh
|
||||
|
||||
- name: Check extension schema boundaries
|
||||
run: ./scripts/check_extension_schema_boundaries.sh
|
||||
|
||||
- name: Check body-cache whitelist guard
|
||||
run: ./scripts/check_body_cache_whitelist.sh
|
||||
|
||||
- name: Check s3s footprint ratchet
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
- name: Check no embedded secret material
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Check test wiring
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/test_security_workflow.py
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
- name: Check CI paths stay in sync
|
||||
run: ./scripts/check_ci_paths_sync.sh
|
||||
|
||||
- name: Check io_uring lane --lib precondition
|
||||
run: ./scripts/check_uring_lane_lib_only.sh
|
||||
|
||||
test-and-lint:
|
||||
name: Test and Lint
|
||||
@@ -850,11 +914,6 @@ jobs:
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install network fault-injection tools
|
||||
run: |
|
||||
sudo apt-get install -y iptables
|
||||
sudo -n iptables --version
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
|
||||
with:
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# 4-node 4-disk distributed e2e lane.
|
||||
#
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
# independent `statfs` capacity. This job runs on GitHub-hosted
|
||||
# `ubuntu-latest` because the self-hosted `sm-standard-4` ARC pods cannot
|
||||
# create filesystems: `mount -o loop` fails with ENOENT (no
|
||||
# `/dev/loop-control`), and `mount -t tmpfs` fails with "cannot mount tmpfs
|
||||
# read-only" (no `CAP_SYS_ADMIN` in the initial namespace). The same reason
|
||||
# `uring-integration` and `e2e-s3tests.yml` left that label. The prepare
|
||||
# step mounts four 1 GiB tmpfs instances and exports `RUSTFS_E2E_POOL_ROOTS`.
|
||||
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
description: "Optional nextest -E filter (default: the whole e2e-distributed profile)"
|
||||
required: false
|
||||
default: ""
|
||||
schedule:
|
||||
# 05:53 UTC nightly — clear of e2e-nightly (04:29) and ODM interop (05:23).
|
||||
- cron: "53 5 * * *"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'schedule' }}
|
||||
|
||||
jobs:
|
||||
distributed:
|
||||
name: Distributed 4-node 4-disk e2e
|
||||
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
|
||||
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
HTTP_PROXY: ""
|
||||
HTTPS_PROXY: ""
|
||||
# Pinned previous release used by distributed::upgrade_test (same pin as e2e-upgrade.yml).
|
||||
UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
|
||||
UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
|
||||
UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
# Dedicated key: ubuntu-latest and sm-standard-4 share runner.os, so
|
||||
# a shared key would mix VM and ARC pod target/ artifacts.
|
||||
cache-shared-key: ci-e2e-distributed-hosted
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Prepare isolated filesystems for pool movement
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
|
||||
mkdir -p "${mount_base}"
|
||||
roots=()
|
||||
for pool in 0 1 2 3; do
|
||||
mountpoint="${mount_base}/pool-${pool}"
|
||||
mkdir -p "${mountpoint}"
|
||||
# Sized tmpfs reports a distinct st_dev and independent 1G
|
||||
# statfs capacity. Requires a VM runner (ubuntu-latest).
|
||||
if ! sudo mount -t tmpfs -o size=1G,nosuid,nodev,mode=1777 tmpfs "${mountpoint}"; then
|
||||
echo "tmpfs mount failed on $(uname -a)" >&2
|
||||
findmnt || true
|
||||
grep Cap /proc/self/status || true
|
||||
exit 1
|
||||
fi
|
||||
sudo chmod 1777 "${mountpoint}"
|
||||
roots+=("${mountpoint}")
|
||||
done
|
||||
printf -v joined_roots '%s:' "${roots[@]}"
|
||||
echo "RUSTFS_E2E_POOL_ROOTS=${joined_roots%:}" >> "${GITHUB_ENV}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[0]}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[1]}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[2]}"
|
||||
findmnt --noheadings --output TARGET,SOURCE,FSTYPE,SIZE --target "${roots[3]}"
|
||||
|
||||
- name: Download pinned previous release
|
||||
env:
|
||||
SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$SOURCE_DIR"
|
||||
archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET"
|
||||
curl --fail --location --retry 3 --output "$archive" \
|
||||
"https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
|
||||
echo "$UPGRADE_SOURCE_SHA256 $archive" | sha256sum --check --strict
|
||||
unzip -q "$archive" -d "$SOURCE_DIR"
|
||||
chmod +x "$SOURCE_DIR/rustfs"
|
||||
test -x "$SOURCE_DIR/rustfs"
|
||||
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build rustfs binary
|
||||
run: |
|
||||
cargo build -p rustfs --bins
|
||||
: > target/debug/rustfs.features
|
||||
|
||||
- name: Verify distributed e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-distributed -p e2e_test --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-distributed "${NEXTEST_LISTING}"
|
||||
|
||||
- name: Run distributed 4-node e2e suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
FILTER: ${{ inputs.filter }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${FILTER}" ]; then
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test -E "${FILTER}"
|
||||
else
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test --no-tests=fail
|
||||
fi
|
||||
|
||||
- name: Upload distributed e2e diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-distributed-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-distributed/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-logs/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Unmount isolated pool filesystems
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mount_base="${RUNNER_TEMP}/rustfs-e2e-pools"
|
||||
for pool in 0 1 2 3; do
|
||||
mountpoint="${mount_base}/pool-${pool}"
|
||||
if mountpoint --quiet "${mountpoint}"; then
|
||||
sudo umount "${mountpoint}"
|
||||
fi
|
||||
done
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [distributed]
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -19,9 +19,7 @@ on:
|
||||
paths:
|
||||
- ".github/workflows/e2e-upgrade.yml"
|
||||
- "crates/e2e_test/src/common.rs"
|
||||
- "crates/e2e_test/src/fake_s3_target/**"
|
||||
- "crates/e2e_test/src/lib.rs"
|
||||
- "crates/e2e_test/src/replication_extension_test.rs"
|
||||
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
@@ -46,9 +44,9 @@ concurrency:
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
UPGRADE_SOURCE_VERSION: 1.0.0-rc.5
|
||||
UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip
|
||||
UPGRADE_SOURCE_SHA256: 3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d
|
||||
UPGRADE_SOURCE_VERSION: 1.0.0-rc.2
|
||||
UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip
|
||||
UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7
|
||||
|
||||
jobs:
|
||||
upgrade:
|
||||
@@ -57,31 +55,14 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# The two `_from_rc2_` tests keep their names: they assert
|
||||
# release-independent object contracts and pass unchanged against the
|
||||
# newer pinned source, so renaming them would only churn history and
|
||||
# the CI required-check names. UPGRADE_SOURCE_VERSION above is the
|
||||
# single source of truth for which release they actually run against.
|
||||
- name: Direct upgrade from the previous release
|
||||
- name: Direct upgrade from rc.2
|
||||
cache_key: e2e-direct-upgrade
|
||||
test: direct_upgrade_from_rc2_preserves_object_contracts
|
||||
artifact: direct-upgrade
|
||||
- name: Mixed-version rolling upgrade from the previous release
|
||||
- name: Mixed-version rolling upgrade from rc.2
|
||||
cache_key: e2e-mixed-version-upgrade
|
||||
test: rolling_upgrade_from_rc2_preserves_mixed_version_contracts
|
||||
artifact: mixed-version-upgrade
|
||||
- name: Bucket configuration survives the upgrade
|
||||
cache_key: e2e-bucket-config-upgrade
|
||||
test: direct_upgrade_from_previous_release_preserves_bucket_configuration
|
||||
artifact: bucket-config-upgrade
|
||||
- name: Rollback reads current bucket metadata
|
||||
cache_key: e2e-bucket-config-rollback
|
||||
test: rollback_to_previous_release_reads_current_bucket_metadata
|
||||
artifact: bucket-config-rollback
|
||||
- name: ODM configuration recovery after rc.5 rollback
|
||||
cache_key: e2e-odm-config-rollback
|
||||
test: rc5_rollback_requires_restoring_odm_configuration
|
||||
artifact: odm-config-rollback
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
|
||||
@@ -166,9 +166,8 @@ jobs:
|
||||
# e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
|
||||
# Skipped when the R2 secrets are not configured (artifact-only mode).
|
||||
- name: Upload DEB to Cloudflare R2
|
||||
id: publish
|
||||
if: env.R2_ACCESS_KEY_ID != ''
|
||||
env:
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
@@ -183,70 +182,28 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y -qq awscli
|
||||
fi
|
||||
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
SOURCE_SHA="$(git rev-parse HEAD)"
|
||||
if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
|
||||
echo "Checkout SHA does not match the nightly build run" >&2
|
||||
exit 1
|
||||
fi
|
||||
DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
|
||||
CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
|
||||
CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
|
||||
|
||||
# Old AWS CLI models lack conditional PutObject support. Never fall
|
||||
# back to an overwriting upload for a candidate.
|
||||
AWS_CLI=aws
|
||||
if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y -qq python3-venv
|
||||
AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
|
||||
trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
|
||||
python3 -m venv "${AWS_CLI_DIR}"
|
||||
"${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
|
||||
AWS_CLI="${AWS_CLI_DIR}/bin/aws"
|
||||
fi
|
||||
"${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
|
||||
"${AWS_CLI}" --version
|
||||
"${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
|
||||
--body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
|
||||
PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
|
||||
if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
|
||||
echo "Published candidate checksum does not match the built package" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DEB_FILE="${{ steps.deb.outputs.deb_file }}"
|
||||
R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"
|
||||
|
||||
echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
|
||||
"${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
|
||||
# Stable "latest" alias so tests can fetch the newest nightly
|
||||
# without knowing today's date.
|
||||
echo "📤 Uploading latest alias"
|
||||
"${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
|
||||
aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
|
||||
--endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
|
||||
echo "✅ R2 upload complete"
|
||||
|
||||
CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --arg source_sha "${SOURCE_SHA}" \
|
||||
--argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
|
||||
'{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
|
||||
> "${CANDIDATE_FILE}"
|
||||
echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Upload nightly candidate manifest
|
||||
if: ${{ steps.publish.outputs.candidate_file != '' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
|
||||
# Functional chain driver: runs the ten functional suites in a fixed order
|
||||
# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
|
||||
# replication -> performance). Each suite attempts the next handoff even
|
||||
# when its tests fail.
|
||||
# replication, with performance on its own runner in parallel) and guarantees
|
||||
# the chain keeps moving even when individual suites fail.
|
||||
#
|
||||
# Each suite workflow can still be dispatched standalone (workflow_dispatch);
|
||||
# only chain-triggered runs forward to the next suite via repository_dispatch,
|
||||
@@ -59,3 +59,16 @@ jobs:
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-upgrade' \
|
||||
-F 'client_payload[from_suite]=nightly-build'
|
||||
|
||||
- name: Dispatch performance suite (parallel, own runner)
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
|
||||
exit 1
|
||||
fi
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-performance' \
|
||||
-F 'client_payload[from_suite]=nightly-build'
|
||||
|
||||
@@ -54,26 +54,14 @@ env:
|
||||
jobs:
|
||||
heal-test:
|
||||
runs-on: smoke-testing
|
||||
# Requirement: a failing suite must not fail the workflow; failures
|
||||
# are filed to rustfs/backlog and the chain continues.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 480
|
||||
# Standalone manual run, or one link of the nightly functional chain
|
||||
# (storage -> heal -> pool). Pool expansion no longer re-runs heal.
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Initialize functional evidence
|
||||
id: evidence
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-heal-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'RUSTFS_WARP_LOG_FILE=%s/warp.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
@@ -129,7 +117,7 @@ jobs:
|
||||
else
|
||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||
fi
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Preflight checks
|
||||
run: |
|
||||
@@ -139,7 +127,7 @@ jobs:
|
||||
else
|
||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||
fi
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Run heal test (write -> outage -> heal -> verify)
|
||||
id: test
|
||||
@@ -149,10 +137,13 @@ jobs:
|
||||
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
||||
--stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
|
||||
--warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
|
||||
--log-file "${LOG_FILE}"
|
||||
--log-file /tmp/rustfs-heal-test.log
|
||||
|
||||
- name: Generate report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-heal-test.log
|
||||
REPORT_FILE: /tmp/rustfs-heal-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
@@ -161,9 +152,8 @@ jobs:
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
STEPS_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/steps.md"
|
||||
CASE_RESULT=success
|
||||
python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY' || CASE_RESULT=failure
|
||||
STEPS_TABLE="/tmp/rustfs-heal-steps.md"
|
||||
python3 - "${LOG_FILE}" "${STEPS_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
@@ -175,7 +165,6 @@ jobs:
|
||||
|
||||
steps = {}
|
||||
order = []
|
||||
status_rank = {'SKIP': 0, 'PASS': 1, 'FAIL': 2}
|
||||
version = None
|
||||
version_node = None
|
||||
verdict = None
|
||||
@@ -189,15 +178,14 @@ jobs:
|
||||
n, desc, status = m.group(1), m.group(2), m.group(3)
|
||||
if n not in steps:
|
||||
order.append(n)
|
||||
if n not in steps or status_rank[status] > status_rank[steps[n][1]]:
|
||||
steps[n] = (desc, status)
|
||||
steps[n] = (desc, status) # later lines win (fail after pass)
|
||||
continue
|
||||
m = ver_re.match(line)
|
||||
if m:
|
||||
version, version_node = m.group(1), m.group(2)
|
||||
continue
|
||||
m = result_re.match(line)
|
||||
if m and verdict != 'FAIL':
|
||||
if m:
|
||||
verdict, verdict_detail = m.group(1), m.group(2)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
@@ -217,43 +205,30 @@ jobs:
|
||||
out.write(f'| {n} | {desc} | {status} |\n')
|
||||
if not order:
|
||||
out.write('| - | - | NOT RUN (no step result lines found) |\n')
|
||||
complete = set(steps) == {str(n) for n in range(1, 8)}
|
||||
sys.exit(0 if complete and verdict != 'FAIL' and all(status == 'PASS' for _, status in steps.values()) else 1)
|
||||
PY
|
||||
RESULT=failure
|
||||
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
|
||||
RESULT=success
|
||||
fi
|
||||
{
|
||||
echo "# RustFS heal test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- Test Step Outcome: ${RESULT}"
|
||||
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
if [ "${RESULT}" = "success" ]; then
|
||||
cat "${STEPS_TABLE}"
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}"
|
||||
echo '```'
|
||||
else
|
||||
echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log."
|
||||
fi
|
||||
cat "${STEPS_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
[ "${RESULT}" = "success" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-heal-report.md
|
||||
SUITE: heal
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -263,32 +238,28 @@ jobs:
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
# Base64-encode the report into a temp file and feed it to jq via
|
||||
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
||||
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'heal'
|
||||
SUITE_LABEL: 'Heal'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: '/tmp/rustfs-heal-report.md'
|
||||
LOG_FILE: '/tmp/rustfs-heal-test.log'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
@@ -316,16 +287,14 @@ jobs:
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
if [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
@@ -341,16 +310,14 @@ jobs:
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload test logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-heal-test-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
name: rustfs-heal-test-${{ github.run_id }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/warp.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/steps.md
|
||||
if-no-files-found: error
|
||||
/tmp/rustfs-heal-test*.log
|
||||
/tmp/rustfs-warp.*.log
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
|
||||
@@ -49,28 +49,10 @@ env:
|
||||
jobs:
|
||||
kms-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 420
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize functional evidence
|
||||
id: evidence
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-kms-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
@@ -127,6 +109,9 @@ jobs:
|
||||
|
||||
- name: Run KMS suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-kms.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-kms-test.sh
|
||||
@@ -156,7 +141,10 @@ jobs:
|
||||
./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-kms.log
|
||||
REPORT_FILE: /tmp/rustfs-kms-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
@@ -168,43 +156,79 @@ jobs:
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
CASE_RESULT=success
|
||||
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
|
||||
RESULT=failure
|
||||
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
|
||||
RESULT=success
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-kms-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS KMS test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- Test Step Outcome: ${RESULT}"
|
||||
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
if [ "${RESULT}" = "success" ]; then
|
||||
cat "${CASE_TABLE}"
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}"
|
||||
echo '```'
|
||||
else
|
||||
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
|
||||
fi
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
[ "${RESULT}" = "success" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-kms-report.md
|
||||
SUITE: kms
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -214,32 +238,28 @@ jobs:
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
# Base64-encode the report into a temp file and feed it to jq via
|
||||
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
||||
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'kms'
|
||||
SUITE_LABEL: 'KMS'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: '/tmp/rustfs-kms-report.md'
|
||||
LOG_FILE: '/tmp/rustfs-kms.log'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
@@ -267,16 +287,14 @@ jobs:
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
if [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
@@ -292,15 +310,14 @@ jobs:
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-kms-test-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
name: rustfs-kms-test-${{ github.run_id }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
|
||||
if-no-files-found: error
|
||||
/tmp/rustfs-kms.log
|
||||
/tmp/rustfs-kms-report.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: always()
|
||||
|
||||
@@ -49,16 +49,17 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the replication suite finishes.
|
||||
# Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
|
||||
# pf-testing runner, in parallel with the shared-VM chain).
|
||||
types: [rustfs-chain-performance]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The default performance nodes overlap the other suites' remote VMs, even
|
||||
# though the runner differs. Hold the shared lock through cleanup as well.
|
||||
# Dedicated pf-testing runner/environment: own concurrency group so perf runs
|
||||
# never block (or are blocked by) the pool-expansion / heal tests.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
group: rustfs-performance-test
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
@@ -75,33 +76,22 @@ env:
|
||||
# Package used by the nightly run (workflow_dispatch inputs are empty for
|
||||
# workflow_run events), i.e. the latest nightly deb published by nightly-gnu.yml.
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
# Fixed benchmark result directory so later steps can read summary.md
|
||||
RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
|
||||
# Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
performance-test:
|
||||
runs-on: pf-testing
|
||||
# Requirement: a failing benchmark must not fail the workflow;
|
||||
# failures are filed to rustfs/backlog.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 900
|
||||
# Run on manual dispatch, or when the nightly build completed successfully.
|
||||
# Skipped when nightly failed.
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Initialize functional evidence
|
||||
id: evidence
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-performance-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'RUSTFS_RESULT_DIR=%s/results\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'VERSION_FILE=%s/version.txt\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
@@ -133,7 +123,7 @@ jobs:
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
chmod +x auto-testing/rustfs_performance_test.sh
|
||||
./auto-testing/rustfs_performance_test.sh --step 1 -y --log-file "${LOG_FILE:-/dev/null}"
|
||||
./auto-testing/rustfs_performance_test.sh --step 1 -y
|
||||
|
||||
- name: Install RustFS package & start cluster (4x4)
|
||||
run: |
|
||||
@@ -143,7 +133,7 @@ jobs:
|
||||
else
|
||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||
fi
|
||||
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Preflight checks
|
||||
run: |
|
||||
@@ -153,7 +143,7 @@ jobs:
|
||||
else
|
||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||
fi
|
||||
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"
|
||||
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Run benchmark (GET/PUT/MIXED)
|
||||
id: benchmark
|
||||
@@ -166,15 +156,17 @@ jobs:
|
||||
--step 5 -y \
|
||||
--warp-duration "${{ inputs.warp_duration || '5m' }}" \
|
||||
--warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
|
||||
--log-file "${LOG_FILE}"
|
||||
--log-file /tmp/rustfs-perf-test.log
|
||||
|
||||
- name: Analyze results
|
||||
if: ${{ steps.benchmark.conclusion == 'success' }}
|
||||
run: |
|
||||
./auto-testing/rustfs_performance_test.sh --step 6 -y --log-file "${LOG_FILE:-/dev/null}"
|
||||
./auto-testing/rustfs_performance_test.sh --step 6 -y
|
||||
|
||||
- name: Collect RustFS version info
|
||||
if: ${{ steps.benchmark.conclusion == 'success' }}
|
||||
env:
|
||||
VERSION_FILE: /tmp/rustfs-version.txt
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES}"
|
||||
@@ -194,6 +186,7 @@ jobs:
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
|
||||
VERSION_FILE: /tmp/rustfs-version.txt
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
@@ -201,7 +194,7 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
SUMMARY="${RESULT_DIR}/summary.md"
|
||||
[ -s "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
|
||||
[ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="reports/${DATE}.md"
|
||||
{
|
||||
@@ -209,8 +202,6 @@ jobs:
|
||||
echo ""
|
||||
echo "- **Date**: ${DATE}"
|
||||
echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- **Attempt**: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- **Workflow Commit**: ${GITHUB_SHA}"
|
||||
echo "- **Trigger**: ${{ github.event_name }}"
|
||||
echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo ""
|
||||
@@ -220,8 +211,8 @@ jobs:
|
||||
echo '```text'
|
||||
cat "${VERSION_FILE}"
|
||||
echo '```'
|
||||
} > "${REPORT_FILE}"
|
||||
CONTENT="$(python3 -c 'import base64,sys; print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
} > /tmp/rustfs-perf-report.md
|
||||
CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
@@ -240,10 +231,11 @@ jobs:
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'performance'
|
||||
SUITE_LABEL: 'Performance'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: '/tmp/rustfs-perf-report.md'
|
||||
LOG_FILE: '/tmp/rustfs-perf-test.log'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
@@ -271,16 +263,14 @@ jobs:
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
if [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
@@ -296,26 +286,20 @@ jobs:
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload test logs & results
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-perf-test-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
name: rustfs-perf-test-${{ github.run_id }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/version.txt
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/master.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/summary.tsv
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/get_*.txt
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/put_*.txt
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results/mixed_*.txt
|
||||
if-no-files-found: error
|
||||
/tmp/rustfs-perf-test*.log
|
||||
/tmp/rustfs-perf-results/**
|
||||
/tmp/rustfs-version.txt
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Reset test environment (after)
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs_performance_test.sh --step 7 -y --log-file "${LOG_FILE:-/dev/null}"
|
||||
./auto-testing/rustfs_performance_test.sh --step 7 -y
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -76,6 +76,9 @@ jobs:
|
||||
pool-expansion-test:
|
||||
name: Pool expansion / decommission test
|
||||
runs-on: smoke-testing
|
||||
# Requirement: a failing suite must not fail the workflow; failures
|
||||
# are filed to rustfs/backlog and the chain continues.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
env:
|
||||
@@ -539,22 +542,17 @@ jobs:
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
# Base64-encode the report into a temp file and feed it to jq via
|
||||
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
||||
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
|
||||
|
||||
@@ -34,7 +34,8 @@ on:
|
||||
- site
|
||||
default: all
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the security suite finishes.
|
||||
# Chain handoff: dispatched when the security suite finishes. This is the
|
||||
# last link of the functional chain.
|
||||
types: [rustfs-chain-replication]
|
||||
|
||||
permissions:
|
||||
@@ -61,28 +62,12 @@ env:
|
||||
jobs:
|
||||
replication-test:
|
||||
runs-on: smoke-testing
|
||||
# A failed replication run must not break the chain or the workflow: the
|
||||
# failure is reported to rustfs/backlog instead (see the issue step).
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize functional evidence
|
||||
id: evidence
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-replication-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
@@ -131,6 +116,9 @@ jobs:
|
||||
|
||||
- name: Run replication suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-replication.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-replication-test.sh
|
||||
@@ -153,7 +141,10 @@ jobs:
|
||||
./auto-testing/rustfs-replication-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-replication.log
|
||||
REPORT_FILE: /tmp/rustfs-replication-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
@@ -175,44 +166,80 @@ jobs:
|
||||
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
|
||||
fi
|
||||
fi
|
||||
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
CASE_RESULT=success
|
||||
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
|
||||
RESULT=failure
|
||||
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
|
||||
RESULT=success
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-replication-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS replication test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
|
||||
echo "- Test Step Outcome: ${RESULT}"
|
||||
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
if [ "${RESULT}" = "success" ]; then
|
||||
cat "${CASE_TABLE}"
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}"
|
||||
echo '```'
|
||||
else
|
||||
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
|
||||
fi
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
[ "${RESULT}" = "success" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-replication-report.md
|
||||
SUITE: replication
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -222,32 +249,28 @@ jobs:
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
# Base64-encode the report into a temp file and feed it to jq via
|
||||
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
||||
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'replication'
|
||||
SUITE_LABEL: 'Replication (bucket + site)'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: '/tmp/rustfs-replication-report.md'
|
||||
LOG_FILE: '/tmp/rustfs-replication.log'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
@@ -275,16 +298,14 @@ jobs:
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
if [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
@@ -300,15 +321,14 @@ jobs:
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-replication-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
name: rustfs-replication-${{ github.run_id }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
|
||||
if-no-files-found: error
|
||||
/tmp/rustfs-replication.log
|
||||
/tmp/rustfs-replication-report.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: always()
|
||||
@@ -329,50 +349,13 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: Performance)"
|
||||
- name: Chain complete
|
||||
# Replication is the last link of the functional chain: nothing to
|
||||
# dispatch after it. This step just records that the chain finished.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-performance' \
|
||||
-F 'client_payload[from_suite]=replication'; then
|
||||
echo "dispatched next suite Performance (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
trap 'rm -f "${BODY_FILE}"' EXIT
|
||||
{
|
||||
echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-performance'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Functional chain complete: replication (final suite) finished."
|
||||
echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -37,28 +37,10 @@ env:
|
||||
jobs:
|
||||
s3-compat-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize functional evidence
|
||||
id: evidence
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-s3-compat-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
@@ -106,6 +88,9 @@ jobs:
|
||||
|
||||
- name: Run S3 compatibility suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-s3-compat.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-s3-compat-test.sh
|
||||
@@ -122,7 +107,10 @@ jobs:
|
||||
./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-s3-compat.log
|
||||
REPORT_FILE: /tmp/rustfs-s3-compat-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
@@ -144,44 +132,83 @@ jobs:
|
||||
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
|
||||
fi
|
||||
fi
|
||||
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
CASE_RESULT=success
|
||||
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
|
||||
RESULT=failure
|
||||
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
|
||||
RESULT=success
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
current = None
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
current = case_id
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
current = None
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS S3 compatibility test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
|
||||
echo "- Test Step Outcome: ${RESULT}"
|
||||
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
if [ "${RESULT}" = "success" ]; then
|
||||
cat "${CASE_TABLE}"
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}"
|
||||
echo '```'
|
||||
else
|
||||
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
|
||||
fi
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
[ "${RESULT}" = "success" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-s3-compat-report.md
|
||||
SUITE: s3
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -191,32 +218,28 @@ jobs:
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
# Base64-encode the report into a temp file and feed it to jq via
|
||||
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
||||
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 's3'
|
||||
SUITE_LABEL: 'S3 compatibility'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
|
||||
LOG_FILE: '/tmp/rustfs-s3-compat.log'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
@@ -244,16 +267,14 @@ jobs:
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
if [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
@@ -269,15 +290,14 @@ jobs:
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-s3-compat-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
name: rustfs-s3-compat-${{ github.run_id }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
|
||||
if-no-files-found: error
|
||||
/tmp/rustfs-s3-compat.log
|
||||
/tmp/rustfs-s3-compat-report.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: always()
|
||||
|
||||
@@ -77,14 +77,10 @@ jobs:
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
# Checkout the repository into its own subdirectory. Checking out at
|
||||
# the workspace root would wipe the auto-testing clone above (that is
|
||||
# exactly how run 33934141181 lost rustfs-security-test.sh).
|
||||
- name: Checkout repository (for the OIDC live gate script)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
path: rustfs-repo
|
||||
|
||||
- name: Initialize security evidence
|
||||
id: evidence
|
||||
@@ -92,7 +88,7 @@ jobs:
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${SECURITY_ARTIFACTS_DIR}" "${SECURITY_ARTIFACTS_DIR}-scratch"
|
||||
mkdir -- "${SECURITY_ARTIFACTS_DIR}"
|
||||
printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
|
||||
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
@@ -148,8 +144,8 @@ jobs:
|
||||
continue-on-error: true
|
||||
env:
|
||||
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
|
||||
TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}-scratch
|
||||
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh
|
||||
TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
|
||||
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-security-test.sh
|
||||
@@ -172,7 +168,7 @@ jobs:
|
||||
else
|
||||
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" 2>&1 | tee "${SECURITY_ARTIFACTS_DIR}/suite.log"
|
||||
GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
id: report
|
||||
@@ -223,22 +219,17 @@ jobs:
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
# Base64-encode the report into a temp file and feed it to jq via
|
||||
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
||||
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
@@ -305,10 +296,7 @@ jobs:
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.SECURITY_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
|
||||
path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
|
||||
|
||||
@@ -46,28 +46,10 @@ env:
|
||||
jobs:
|
||||
storage-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize functional evidence
|
||||
id: evidence
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-storage-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
@@ -115,6 +97,9 @@ jobs:
|
||||
|
||||
- name: Run storage engine suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-storage.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-storage-test.sh
|
||||
@@ -137,7 +122,10 @@ jobs:
|
||||
./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-storage.log
|
||||
REPORT_FILE: /tmp/rustfs-storage-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
@@ -159,44 +147,83 @@ jobs:
|
||||
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
|
||||
fi
|
||||
fi
|
||||
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
CASE_RESULT=success
|
||||
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" || CASE_RESULT=failure
|
||||
RESULT=failure
|
||||
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
|
||||
RESULT=success
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-storage-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
current = None
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
current = case_id
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
current = None
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS storage engine test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
|
||||
echo "- Test Step Outcome: ${RESULT}"
|
||||
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
if [ "${RESULT}" = "success" ]; then
|
||||
cat "${CASE_TABLE}"
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}"
|
||||
echo '```'
|
||||
else
|
||||
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
|
||||
fi
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
[ "${RESULT}" = "success" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-storage-report.md
|
||||
SUITE: storage
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -206,32 +233,28 @@ jobs:
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
# Base64-encode the report into a temp file and feed it to jq via
|
||||
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
||||
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'storage'
|
||||
SUITE_LABEL: 'Storage engine'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: '/tmp/rustfs-storage-report.md'
|
||||
LOG_FILE: '/tmp/rustfs-storage.log'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
@@ -259,16 +282,14 @@ jobs:
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
if [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
@@ -284,15 +305,14 @@ jobs:
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-storage-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
name: rustfs-storage-${{ github.run_id }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
|
||||
if-no-files-found: error
|
||||
/tmp/rustfs-storage.log
|
||||
/tmp/rustfs-storage-report.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: always()
|
||||
|
||||
@@ -61,6 +61,9 @@ env:
|
||||
jobs:
|
||||
tier-test:
|
||||
runs-on: smoke-testing
|
||||
# Requirement: a failing suite must not fail the workflow; failures
|
||||
# are filed to rustfs/backlog and the chain continues.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 420
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
@@ -377,22 +380,17 @@ jobs:
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
# Base64-encode the report into a temp file and feed it to jq via
|
||||
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
||||
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Verify required tier evidence
|
||||
id: evidence_verify
|
||||
|
||||
@@ -79,28 +79,10 @@ env:
|
||||
jobs:
|
||||
upgrade-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 420
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout repository (for report parser)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize functional evidence
|
||||
id: evidence
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-upgrade-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}-scratch"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
@@ -160,7 +142,9 @@ jobs:
|
||||
|
||||
- name: Run upgrade compatibility suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-upgrade.log
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -218,7 +202,10 @@ jobs:
|
||||
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-upgrade.log
|
||||
REPORT_FILE: /tmp/rustfs-upgrade-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
FROM_URL='${{ inputs.from_url }}'
|
||||
@@ -239,47 +226,103 @@ jobs:
|
||||
else
|
||||
TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
CASE_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/cases.md"
|
||||
MATRIX_TABLE="${FUNCTIONAL_ARTIFACTS_DIR}/matrix.md"
|
||||
CASE_RESULT=success
|
||||
python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" || CASE_RESULT=failure
|
||||
RESULT=failure
|
||||
if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then
|
||||
RESULT=success
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
|
||||
MATRIX_TABLE="/tmp/rustfs-upgrade-matrix.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" "${MATRIX_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file, matrix_file = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
|
||||
topo_re = re.compile(
|
||||
r'^\[UPG-TOPO\]\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+PASS=(\d+)\s+FAIL=(\d+)\s*$')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
topo_rows = []
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = topo_re.match(line)
|
||||
if m:
|
||||
topo_rows.append(m.groups())
|
||||
continue
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
|
||||
# Upgrade matrix: one row per topology/backend with the versions
|
||||
# captured on the nodes (rustfs --version) and the aggregated
|
||||
# result. The dashboard renders this table directly.
|
||||
with open(matrix_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Upgrade Matrix\n\n')
|
||||
out.write('| Topology | KMS Backend | From Version | To Version | Result |\n')
|
||||
out.write('| --- | --- | --- | --- | --- |\n')
|
||||
for topo, backend, old_v, new_v, npass, nfail in topo_rows:
|
||||
result = 'PASS' if nfail == '0' else 'FAIL'
|
||||
out.write(f'| {topo} | {backend} | {old_v} | {new_v} | {result} (PASS={npass} FAIL={nfail}) |\n')
|
||||
if not topo_rows:
|
||||
out.write('| - | - | - | - | NOT RUN (suite failed before upgrade) |\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS upgrade compatibility report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- From: ${FROM_SOURCE}"
|
||||
echo "- To: ${TO_SOURCE}"
|
||||
echo "- Test Step Outcome: ${RESULT}"
|
||||
echo "- Suite Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
if [ "${RESULT}" = "success" ]; then
|
||||
cat "${MATRIX_TABLE}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}"
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}"
|
||||
echo '```'
|
||||
else
|
||||
echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log."
|
||||
fi
|
||||
cat "${MATRIX_TABLE}" || true
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
[ "${RESULT}" = "success" ]
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-upgrade-report.md
|
||||
SUITE: upgrade
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -289,32 +332,28 @@ jobs:
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
# Base64-encode the report into a temp file and feed it to jq via
|
||||
# --rawfile: large reports (e.g. pool) exceed the OS argv limit and
|
||||
# make `jq --arg content "${CONTENT}"` fail with "Argument list too long".
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'upgrade'
|
||||
SUITE_LABEL: 'Upgrade compatibility'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
|
||||
LOG_FILE: '/tmp/rustfs-upgrade.log'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
@@ -342,16 +381,14 @@ jobs:
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
if [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
@@ -367,16 +404,14 @@ jobs:
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-upgrade-test-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
name: rustfs-upgrade-test-${{ github.run_id }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/cases.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/matrix.md
|
||||
if-no-files-found: error
|
||||
/tmp/rustfs-upgrade-report.md
|
||||
/tmp/rustfs-upgrade.*/*
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
|
||||
@@ -22,7 +22,6 @@ on:
|
||||
- "Continuous Integration"
|
||||
- "coverage"
|
||||
- "e2e-nightly"
|
||||
- "e2e-distributed"
|
||||
- "e2e-s3tests"
|
||||
- "Fuzz"
|
||||
- "mint"
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: rustfs-fmt-check
|
||||
name: Rust formatting
|
||||
entry: cargo fmt --all --check
|
||||
- id: rustfs-dev-check
|
||||
name: rustfs dev-check
|
||||
entry: make dev-check
|
||||
language: system
|
||||
types: [rust]
|
||||
pass_filenames: false
|
||||
|
||||
+1
-4
@@ -18,10 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Read paths: an object at or below `policy.inline_max_bytes` (16 MiB by default) is teed to the client and to the local store in a single source read; a larger object or a Range read streams through and a background pull stores the whole object. A HEAD miss is proxied to the source and stores nothing (`policy.head = local_only` disables it). Every source-backed response carries `x-rustfs-on-demand-migration: source`
|
||||
- Protections: a per-source circuit breaker, a per-key negative cache, singleflight per key, a concurrency limit and a bounded pull queue shared by the inline and background paths, an optional bandwidth limit, an anti-loop request marker, and the shared outbound-endpoint (SSRF) policy
|
||||
- Metrics under `rustfs_on_demand_migration_*` (`requests_total`, `pulled_bytes_total`, `pulled_objects_total`, `pull_failures_total`, `inflight_pulls`, `queue_depth`, `source_latency_seconds_*`, `breaker_state`), mirrored per node by the admin status route
|
||||
- Listings: `ListObjects` v1 remains local with ordinary key markers. `ListObjectsV2` can merge source objects when `policy.list_through = true`; this is off by default
|
||||
- Upgrade and rollback: finish upgrading every node before enabling ODM. An rc.5 node that writes bucket configuration drops the ODM fields from metadata; neither a later restart nor moving the service out of ECStore recovers them. Before rollback, disable ODM and securely retain the original full configuration and credentials. After every node returns to a compatible version, restore and validate that configuration. Redacted exports cannot replace the credential backup; source-only objects are unavailable through RustFS while ODM is disabled. See the upgrade and rollback section of `docs/operations/on-demand-migration.md`
|
||||
- Optional Google dependencies: default and `full` server builds retain native GCS support. `cargo build -p rustfs --no-default-features --features ftps,webdav` excludes Google SDKs while preserving configuration decoding and redaction; native GCS ODM and tier operations require the `gcs` feature. Do not use that build with existing GCS-tiered data
|
||||
- Limitations: PUT and DELETE never reach the source; a source object updated after it was pulled is not re-fetched; SSE-C source objects are unsupported and answer 424; `Last-Modified` on a pulled object is the local write time, with the source timestamp kept in metadata
|
||||
- Limitations: listings show only local objects (the source is not merged into `ListObjectsV2`); PUT and DELETE never reach the source; a source object updated after it was pulled is not re-fetched; SSE-C source objects are unsupported and answer 424; `Last-Modified` on a pulled object is the local write time, with the source timestamp kept in metadata
|
||||
- **NATS JetStream Publish Path**: Opt-in at-least-once delivery for the NATS notify and audit targets. A NATS Core publish flushes to the connection without awaiting a broker acknowledgement, so an event can be lost across a broker restart or a reconnect after the send queue has already cleared it. A queued event now clears only after the JetStream `PublishAck`, so bucket notifications survive those interruptions. Off by default and byte-identical to the NATS Core path when disabled.
|
||||
- Three configuration keys per target: `JETSTREAM_ENABLE`, `JETSTREAM_STREAM_NAME`, and `JETSTREAM_ACK_TIMEOUT_SECS`, under the `RUSTFS_NOTIFY_NATS_` and `RUSTFS_AUDIT_NATS_` prefixes
|
||||
- Durable store-and-forward with a stable dedup id sent as the `Nats-Msg-Id` header, so a replay after a crash is collapsed by the server duplicate window
|
||||
|
||||
+37
-11
@@ -109,17 +109,24 @@ affected boundaries and risks. CI still runs its configured repository gates.
|
||||
|
||||
### 🔒 Git Pre-commit Hooks (optional)
|
||||
|
||||
The optional hook uses the checked-in `.pre-commit-config.yaml`. Install [pre-commit](https://pre-commit.com/#installation), then run this from the checkout or a linked worktree:
|
||||
Git hooks are **not** versioned in this repository, so a fresh clone has no
|
||||
active pre-commit hook. If you add your own `.git/hooks/pre-commit` (a good
|
||||
choice is a one-liner that runs `make pre-commit`), you can mark it executable
|
||||
with:
|
||||
|
||||
```bash
|
||||
make setup-hooks
|
||||
```
|
||||
|
||||
The hook runs `cargo fmt --all --check` when staged files include Rust source. It does not compile the workspace or run tests. Fix formatting with `cargo fmt --all`, inspect and stage the result, then commit again.
|
||||
Or manually:
|
||||
|
||||
`pre-commit install` resolves Git's hook directory for linked worktrees and preserves an existing hook in migration mode. If you use `core.hooksPath`, keep that hook manager and integrate `pre-commit run` there; the installer refuses to silently replace that configuration.
|
||||
```bash
|
||||
chmod +x .git/hooks/pre-commit
|
||||
```
|
||||
|
||||
A local hook provides early formatting feedback. With or without it, follow the verification tiers in `AGENTS.md`, run relevant behavioral tests, and satisfy the CI merge gates. `make pre-commit` and `make dev-check` remain explicit broader commands.
|
||||
With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
|
||||
applicable scoped checks, and reserve `make pre-pr` for broad cross-module
|
||||
changes whose impact cannot be bounded by those checks.
|
||||
|
||||
### 📝 Formatting Configuration
|
||||
|
||||
@@ -131,11 +138,31 @@ fn_call_width = 90
|
||||
single_line_let_else_max_width = 100
|
||||
```
|
||||
|
||||
### 🚫 Commit Prevention
|
||||
|
||||
If you set up a pre-commit hook and your code doesn't meet the formatting requirements, the hook will:
|
||||
|
||||
1. **Block the commit** and show clear error messages
|
||||
2. **Provide exact commands** to fix the issues
|
||||
3. **Guide you through** the resolution process
|
||||
|
||||
Example output when formatting fails:
|
||||
|
||||
```
|
||||
❌ Code formatting check failed!
|
||||
💡 Please run 'cargo fmt --all' to format your code before committing.
|
||||
|
||||
🔧 Quick fix:
|
||||
cargo fmt --all
|
||||
git add .
|
||||
git commit
|
||||
```
|
||||
|
||||
### 🔄 Development Workflow
|
||||
|
||||
1. **Make your changes**
|
||||
2. **Format your code**: `make fmt` or `cargo fmt --all`
|
||||
3. **Select relevant checks** using the validation tier in `AGENTS.md`; use `make pre-commit` when its broader fast gate adds useful coverage
|
||||
3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
|
||||
4. **Commit your changes**: `git commit -m "your message"`
|
||||
5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
|
||||
6. **Run applicable scoped checks before opening/updating a PR**; consider
|
||||
@@ -179,12 +206,11 @@ Configure your IDE to:
|
||||
#### Pre-commit hook not running?
|
||||
|
||||
```bash
|
||||
pre-commit validate-config
|
||||
pre-commit run --all-files
|
||||
# Inspect any configured hook manager; do not overwrite it.
|
||||
git config --get core.hooksPath
|
||||
# Install if no separate hook manager is configured.
|
||||
make setup-hooks
|
||||
# Check if hook is executable
|
||||
ls -la .git/hooks/pre-commit
|
||||
|
||||
# Make it executable if needed
|
||||
chmod +x .git/hooks/pre-commit
|
||||
```
|
||||
|
||||
#### Formatting issues?
|
||||
|
||||
Generated
+34
-67
@@ -315,7 +315,7 @@ dependencies = [
|
||||
"strum",
|
||||
"thiserror 2.0.20",
|
||||
"uuid",
|
||||
"zstd 0.13.3",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -508,7 +508,7 @@ dependencies = [
|
||||
"arrow-select",
|
||||
"flatbuffers",
|
||||
"lz4_flex",
|
||||
"zstd 0.13.3",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1679,7 +1679,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1698,7 +1698,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2110,7 +2110,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -2249,8 +2249,8 @@ dependencies = [
|
||||
"liblzma",
|
||||
"lz4",
|
||||
"memchr",
|
||||
"zstd 0.13.3",
|
||||
"zstd-safe 7.3.0",
|
||||
"zstd",
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2580,7 +2580,7 @@ version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -2605,11 +2605,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -3901,7 +3901,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -4067,7 +4067,7 @@ dependencies = [
|
||||
"uuid",
|
||||
"walkdir",
|
||||
"zip",
|
||||
"zstd 0.14.0",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4166,7 +4166,7 @@ dependencies = [
|
||||
"crypto-bigint 0.5.5",
|
||||
"digest 0.10.7",
|
||||
"ff 0.13.1",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"group 0.13.0",
|
||||
"hkdf 0.12.4",
|
||||
"pem-rfc7468 0.7.0",
|
||||
@@ -4499,7 +4499,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
|
||||
dependencies = [
|
||||
"io-lifetimes 2.0.4",
|
||||
"rustix",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4622,9 +4622,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.9"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4637,7 +4637,7 @@ version = "1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -5319,9 +5319,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.25.1"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "846bde0d9600d98434e1aac376977d7718bfe3d2f5312a041b7c59a6a466c51a"
|
||||
checksum = "929b2285d2cd21b2733a7fb6ebc843bb4f83dbd1db0122f5f9ebb9567b1e2613"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5660,7 +5660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5693,7 +5693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
|
||||
dependencies = [
|
||||
"io-lifetimes 3.0.1",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5971,7 +5971,7 @@ dependencies = [
|
||||
"lz4",
|
||||
"snap",
|
||||
"uuid",
|
||||
"zstd 0.13.3",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7115,7 +7115,7 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"base64 0.21.7",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.5.0",
|
||||
@@ -7658,7 +7658,7 @@ dependencies = [
|
||||
"snap",
|
||||
"tokio",
|
||||
"twox-hash",
|
||||
"zstd 0.13.3",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9493,8 +9493,6 @@ dependencies = [
|
||||
"atomic_enum",
|
||||
"aws-config",
|
||||
"aws-sdk-s3",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"axum",
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9503,13 +9501,11 @@ dependencies = [
|
||||
"clap",
|
||||
"const-str",
|
||||
"datafusion",
|
||||
"faster-hex",
|
||||
"flatbuffers",
|
||||
"flate2",
|
||||
"futures",
|
||||
"futures-lite",
|
||||
"futures-util",
|
||||
"google-cloud-auth",
|
||||
"hashbrown 0.17.1",
|
||||
"hex-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9528,7 +9524,6 @@ dependencies = [
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"mime_guess",
|
||||
"moka",
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
"p256 0.14.0",
|
||||
@@ -9621,10 +9616,9 @@ dependencies = [
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
"x509-parser",
|
||||
"xxhash-rust",
|
||||
"zeroize",
|
||||
"zip",
|
||||
"zstd 0.14.0",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10234,7 +10228,7 @@ dependencies = [
|
||||
"thiserror 2.0.20",
|
||||
"walkdir",
|
||||
"zip",
|
||||
"zstd 0.14.0",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10401,7 +10395,7 @@ dependencies = [
|
||||
"tracing-opentelemetry",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"zstd 0.14.0",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10991,7 +10985,7 @@ dependencies = [
|
||||
"transform-stream",
|
||||
"url",
|
||||
"windows",
|
||||
"zstd 0.14.0",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11405,7 +11399,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct 0.2.0",
|
||||
"der 0.7.10",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -12405,7 +12399,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -13657,15 +13651,6 @@ dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.60.2"
|
||||
@@ -13838,7 +13823,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14110,7 +14095,7 @@ dependencies = [
|
||||
"typed-path",
|
||||
"zeroize",
|
||||
"zopfli",
|
||||
"zstd 0.13.3",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14143,16 +14128,7 @@ version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||
dependencies = [
|
||||
"zstd-safe 7.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e"
|
||||
dependencies = [
|
||||
"zstd-safe 8.0.0",
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14164,15 +14140,6 @@ dependencies = [
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "8.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab"
|
||||
dependencies = [
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "2.1.0+zstd.1.5.7"
|
||||
|
||||
+6
-7
@@ -199,10 +199,10 @@ serde_urlencoded = "0.7.1"
|
||||
# matching stable releases are not available yet, while previous stable lines
|
||||
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
|
||||
# releases.
|
||||
aes-gcm = { version = "0.11.1" }
|
||||
argon2 = { version = "0.6.0" }
|
||||
blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
aes-gcm = { version = "=0.11.1" }
|
||||
argon2 = { version = "=0.6.0" }
|
||||
blake2 = "=0.11.0"
|
||||
chacha20poly1305 = { version = "=0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.0.0" }
|
||||
@@ -343,7 +343,7 @@ windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.18" }
|
||||
zip = "8.6.0"
|
||||
zstd = "0.14.0"
|
||||
zstd = "0.13.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
@@ -371,8 +371,7 @@ dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.5.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "=0.25.0", default-features = false }
|
||||
hotpath = { version = "0.25.0", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -130,21 +130,6 @@ Scanner cycle budget controls:
|
||||
- timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
|
||||
- this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
|
||||
|
||||
## Remote tier timeout environment variables
|
||||
|
||||
- `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS`
|
||||
- remote tier TCP connect timeout.
|
||||
- default is `10`.
|
||||
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
|
||||
- `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS`
|
||||
- remote tier request timeout through response headers.
|
||||
- default is `86400` so large transition uploads keep a production-safe budget.
|
||||
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default. Very large values are accepted and act as a correspondingly long budget.
|
||||
- `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS`
|
||||
- maximum idle time between remote tier response-body chunks.
|
||||
- default is `60`; the timer resets only when non-empty body data keeps progressing.
|
||||
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
|
||||
|
||||
## Drive timeout environment variables
|
||||
|
||||
- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
|
||||
|
||||
@@ -137,28 +137,6 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
|
||||
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
|
||||
|
||||
/// Environment variable for remote tier TCP connect timeout in seconds.
|
||||
pub const ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS";
|
||||
/// Default remote tier TCP connect timeout in seconds.
|
||||
pub const DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS: u64 = 10;
|
||||
|
||||
/// Environment variable for the remote tier request timeout in seconds.
|
||||
///
|
||||
/// This bounds upload/download request progress through response headers. The
|
||||
/// default is intentionally large so multi-TiB transition uploads keep their
|
||||
/// previous production budget while black-hole remotes no longer wait forever.
|
||||
pub const ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS";
|
||||
/// Default remote tier request timeout in seconds.
|
||||
pub const DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS: u64 = 24 * 60 * 60;
|
||||
|
||||
/// Environment variable for remote tier response-body idle timeout in seconds.
|
||||
///
|
||||
/// The timer is re-armed on every non-empty response-body chunk, so slow but
|
||||
/// progressing remotes can continue while silent response bodies are cancelled.
|
||||
pub const ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS";
|
||||
/// Default remote tier response-body idle timeout in seconds.
|
||||
pub const DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: u64 = 60;
|
||||
|
||||
/// Request the object-transaction fencing contract used by storage-owned
|
||||
/// cleanup receipts and lock-window optimizations.
|
||||
///
|
||||
@@ -834,16 +812,6 @@ mod remote_version_state_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_tier_timeout_env_names_are_stable() {
|
||||
assert_eq!(super::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS");
|
||||
assert_eq!(super::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS");
|
||||
assert_eq!(
|
||||
super::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
|
||||
"RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_movement_part_checksum_gate_uses_stable_environment_names() {
|
||||
assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");
|
||||
|
||||
@@ -26,7 +26,6 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -172,7 +171,6 @@ the same profile for membership and execution with one nightly worker.
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
@@ -193,9 +191,6 @@ cargo nextest run --profile e2e-smoke -p e2e_test
|
||||
cargo nextest run --profile e2e-full -p e2e_test
|
||||
# Cluster fault nightly lane
|
||||
cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
# 4-node 4-disk distributed lane (S3 / lock / versioning / replication / decommission / chaos / upgrade)
|
||||
# Upgrade cases need RUSTFS_UPGRADE_SOURCE_BINARY; without it they fail closed.
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test
|
||||
# Replication nightly lane; awscurl is required for STS paths
|
||||
cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
# Fixed-port protocol nightly lane
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
fn git(root: &Path, args: &[&str]) -> Option<String> {
|
||||
let output = Command::new("git").args(args).current_dir(root).output().ok()?;
|
||||
output
|
||||
.status
|
||||
.success()
|
||||
.then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned())
|
||||
}
|
||||
|
||||
fn emit(name: &str, value: &str) {
|
||||
let value = if value.contains(['\n', '\r']) { "unknown" } else { value };
|
||||
println!("cargo:rustc-env=RUSTFS_E2E_BUILD_{name}={value}");
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let manifest = std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default();
|
||||
let root = Path::new(&manifest).join("../..");
|
||||
// Cover dependency/common sources as well as this crate. HEAD/ref/index
|
||||
// changes must refresh identity even when no Rust source mtime changes.
|
||||
for path in [
|
||||
"crates",
|
||||
"rustfs",
|
||||
"Cargo.toml",
|
||||
"Cargo.lock",
|
||||
"rust-toolchain.toml",
|
||||
".cargo",
|
||||
".config",
|
||||
] {
|
||||
println!("cargo:rerun-if-changed={}", root.join(path).display());
|
||||
}
|
||||
let mut git_paths = vec!["HEAD".to_owned(), "index".to_owned(), "packed-refs".to_owned()];
|
||||
if let Some(reference) = git(&root, &["symbolic-ref", "-q", "HEAD"]) {
|
||||
git_paths.push(reference);
|
||||
}
|
||||
for path in git_paths {
|
||||
if let Some(path) = git(&root, &["rev-parse", "--git-path", &path]) {
|
||||
let path = Path::new(&path);
|
||||
let path = if path.is_absolute() {
|
||||
path.to_owned()
|
||||
} else {
|
||||
root.join(path)
|
||||
};
|
||||
if path.exists() {
|
||||
println!("cargo:rerun-if-changed={}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
let revision = git(&root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unknown".to_owned());
|
||||
let dirty = git(&root, &["status", "--porcelain", "--untracked-files=normal"]).is_none_or(|status| !status.is_empty());
|
||||
let lock = git(&root, &["hash-object", "Cargo.lock"]).unwrap_or_else(|| "unknown".to_owned());
|
||||
let mut features = std::env::vars()
|
||||
.filter_map(|(key, _)| {
|
||||
key.strip_prefix("CARGO_FEATURE_")
|
||||
.map(|name| name.to_ascii_lowercase().replace('_', "-"))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
features.sort();
|
||||
emit("COMMIT", &revision);
|
||||
emit("DIRTY", if dirty { "true" } else { "false" });
|
||||
emit("LOCK", &lock);
|
||||
emit("FEATURES", &features.join(","));
|
||||
for name in ["TARGET", "PROFILE"] {
|
||||
emit(name, &std::env::var(name).unwrap_or_else(|_| "unknown".to_owned()));
|
||||
}
|
||||
println!("cargo:rerun-if-env-changed=CARGO_ENCODED_RUSTFLAGS");
|
||||
let flags = std::env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default();
|
||||
let flags: String = flags.as_bytes().iter().map(|byte| format!("{byte:02x}")).collect();
|
||||
emit("RUSTFLAGS_HEX", &flags);
|
||||
}
|
||||
@@ -55,20 +55,18 @@ type ChaosResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
/// A successful S3 GET only proves that a quorum can serve an object. Replacement
|
||||
/// tests need this lower-level record to prove that the rebuilt target holds the
|
||||
/// `xl.meta` selected for a specific version and every `part.N` it declares.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct VersionShardCensus {
|
||||
pub version_id: Option<String>,
|
||||
pub has_xl_meta: bool,
|
||||
pub data_dir: Option<String>,
|
||||
pub erasure_index: Option<usize>,
|
||||
pub data_blocks: Option<usize>,
|
||||
pub parity_blocks: Option<usize>,
|
||||
pub expected_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
|
||||
pub inline_data_fingerprint: Option<PartShardFingerprint>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct PartShardFingerprint {
|
||||
pub size: u64,
|
||||
pub sha256: String,
|
||||
@@ -90,15 +88,13 @@ impl VersionShardCensus {
|
||||
&& manifest.is_complete()
|
||||
&& self.data_dir == manifest.data_dir
|
||||
&& self.erasure_index == manifest.erasure_index
|
||||
&& self.data_blocks == manifest.data_blocks
|
||||
&& self.parity_blocks == manifest.parity_blocks
|
||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||
&& self.present_part_fingerprints == manifest.present_part_fingerprints
|
||||
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn sha256_hex(data: &[u8]) -> String {
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
let digest = Sha256::digest(data);
|
||||
digest.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
@@ -317,8 +313,6 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
data_blocks: None,
|
||||
parity_blocks: None,
|
||||
expected_part_numbers: BTreeSet::new(),
|
||||
present_part_fingerprints: BTreeMap::new(),
|
||||
inline_data_fingerprint: None,
|
||||
@@ -366,8 +360,6 @@ pub(crate) fn census_object_version_on_disk(
|
||||
has_xl_meta: true,
|
||||
data_dir,
|
||||
erasure_index,
|
||||
data_blocks: Some(file_info.erasure.data_blocks),
|
||||
parity_blocks: Some(file_info.erasure.parity_blocks),
|
||||
expected_part_numbers,
|
||||
present_part_fingerprints,
|
||||
inline_data_fingerprint,
|
||||
@@ -421,8 +413,6 @@ mod tests {
|
||||
has_xl_meta: true,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
erasure_index: Some(3),
|
||||
data_blocks: Some(2),
|
||||
parity_blocks: Some(2),
|
||||
expected_part_numbers: BTreeSet::from([1]),
|
||||
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
|
||||
inline_data_fingerprint: None,
|
||||
|
||||
@@ -1700,69 +1700,6 @@ impl RustFSTestClusterEnvironment {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a new single-node erasure pool to a stopped multi-pool cluster.
|
||||
///
|
||||
/// Used to simulate pool expansion on localhost: every pool already owns
|
||||
/// exactly one node with `drives_per_node >= 2` (the only multi-pool layout
|
||||
/// the single-host `RUSTFS_VOLUMES` syntax can express). The new node is
|
||||
/// allocated a fresh port and empty drive directories; callers must
|
||||
/// [`Self::start`] afterwards so every process picks up the extended
|
||||
/// volumes argument. Existing data directories are left untouched.
|
||||
pub async fn append_single_node_pool(&mut self) -> Result<usize, Box<dyn std::error::Error + Send + Sync>> {
|
||||
if self.nodes.iter().any(|node| node.process.is_some()) {
|
||||
return Err("stop the cluster before appending a pool".into());
|
||||
}
|
||||
if self.topology.drives_per_node < 2 {
|
||||
return Err(
|
||||
"append_single_node_pool requires drives_per_node >= 2 (the server parser rejects a single-drive ellipses pool)"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
let mut pools = self.topology.normalized_pools();
|
||||
for (pool_idx, nodes) in pools.iter().enumerate() {
|
||||
if nodes.len() != 1 {
|
||||
return Err(format!(
|
||||
"pool {pool_idx} spans {} nodes; append_single_node_pool requires one node per pool",
|
||||
nodes.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
|
||||
let new_idx = self.nodes.len();
|
||||
let port = RustFSTestEnvironment::find_available_port().await?;
|
||||
let address = format!("127.0.0.1:{port}");
|
||||
let data_dirs: Vec<String> = (0..self.topology.drives_per_node)
|
||||
.map(|drive| format!("{}/node{}/drive{}", self.temp_dir, new_idx, drive))
|
||||
.collect();
|
||||
for dir in &data_dirs {
|
||||
fs::create_dir_all(dir).await?;
|
||||
}
|
||||
|
||||
self.nodes.push(ClusterNode {
|
||||
url: format!("http://{address}"),
|
||||
address,
|
||||
data_dir: data_dirs[0].clone(),
|
||||
data_dirs,
|
||||
pool_idx: pools.len(),
|
||||
process: None,
|
||||
});
|
||||
pools.push(vec![new_idx]);
|
||||
self.topology.node_count = self.nodes.len();
|
||||
self.topology.pools = pools;
|
||||
self.node_extra_env.push(Vec::new());
|
||||
self.node_capture_log_paths.push(None);
|
||||
self.volume_proxy_addresses.push(None);
|
||||
|
||||
if !self.extra_env.iter().any(|(key, _)| key == "RUSTFS_UNSAFE_BYPASS_DISK_CHECK") {
|
||||
self.extra_env
|
||||
.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
|
||||
}
|
||||
|
||||
Ok(new_idx)
|
||||
}
|
||||
|
||||
/// Gracefully stop one cluster node and wait for its process to exit.
|
||||
///
|
||||
/// This is intentionally separate from [`Self::stop_node`]: the latter is
|
||||
|
||||
@@ -35,15 +35,11 @@ where
|
||||
{
|
||||
let mut last_usage = DataUsageInfo::default();
|
||||
let mut last_query_error = None;
|
||||
for _ in 0..90 {
|
||||
for _ in 0..45 {
|
||||
match get_data_usage_info(env).await {
|
||||
Ok(usage) => {
|
||||
last_query_error = None;
|
||||
if usage.is_complete_bucket_usage_snapshot()
|
||||
&& usage.usage_snapshot_converged != Some(false)
|
||||
&& usage.buckets_usage.contains_key(bucket)
|
||||
&& predicate(&usage)
|
||||
{
|
||||
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
|
||||
return Ok(usage);
|
||||
}
|
||||
last_usage = usage;
|
||||
|
||||
@@ -1,222 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
|
||||
wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::chaos::{census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{build_test_s3_config, init_logging};
|
||||
use crate::fault_proxy::FaultMode;
|
||||
use aws_sdk_s3::Client;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{Barrier, mpsc};
|
||||
use tokio::time::timeout;
|
||||
|
||||
#[tokio::test]
|
||||
async fn kill_and_restart_node_preserves_objects() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("killnode");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = vec![0x11u8; 128 * 1024];
|
||||
put_object(&dist.client(0)?, &bucket, "keep.bin", body.clone()).await?;
|
||||
|
||||
dist.cluster.stop_node(3)?;
|
||||
retrying_get_equals(&dist.client(0)?, &bucket, "keep.bin", &body, Duration::from_secs(20)).await?;
|
||||
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
assert_object_bytes(&dist.client(3)?, &bucket, "keep.bin", &body).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_cluster_restart_preserves_objects() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("pwr");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = vec![0x44u8; 64 * 1024];
|
||||
put_object(&dist.client(1)?, &bucket, "survive.bin", body.clone()).await?;
|
||||
|
||||
dist.cluster.stop();
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node_idx)?, &bucket, "survive.bin", &body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_drive_replacement_is_physically_healed_without_data_change() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start_with_env(DistLayout::FourByFour, &[("RUSTFS_HEAL_ENABLED", "true")]).await?;
|
||||
let bucket = unique_bucket("baddrive");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = payload_for("fresh-drive/durable.bin", 8 * 1024 * 1024);
|
||||
put_object(&dist.client(1)?, &bucket, "durable.bin", body.clone()).await?;
|
||||
|
||||
let replaced_drive = PathBuf::from(&dist.cluster.nodes[0].data_dirs[0]);
|
||||
let baseline = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
|
||||
assert!(
|
||||
baseline.is_complete(),
|
||||
"replacement target did not hold a complete baseline shard: {baseline:?}"
|
||||
);
|
||||
assert!(
|
||||
!baseline.expected_part_numbers.is_empty(),
|
||||
"replacement witness must use physical part shards: {baseline:?}"
|
||||
);
|
||||
|
||||
dist.cluster.stop_node(0)?;
|
||||
let format_path = replaced_drive.join(".rustfs.sys/format.json");
|
||||
let format = std::fs::read(&format_path)?;
|
||||
let retired_drive = PathBuf::from(format!("{}.retired", replaced_drive.display()));
|
||||
std::fs::rename(&replaced_drive, &retired_drive)?;
|
||||
std::fs::create_dir_all(format_path.parent().ok_or("replacement format path omitted parent")?)?;
|
||||
std::fs::write(&format_path, format)?;
|
||||
let empty = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
|
||||
assert!(!empty.has_xl_meta, "fresh replacement unexpectedly retained object metadata: {empty:?}");
|
||||
|
||||
dist.cluster.start_node(0).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let heal_body =
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[1].url);
|
||||
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(90),
|
||||
|| async {
|
||||
let healed = census_object_version_on_disk(&replaced_drive, &bucket, "durable.bin", None)?;
|
||||
Ok(healed.matches_manifest(&baseline))
|
||||
},
|
||||
"fresh replacement contains the original complete shard manifest",
|
||||
)
|
||||
.await?;
|
||||
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node_idx)?, &bucket, "durable.bin", &body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_gets_survive_peer_node_kill() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("getkill");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let body = payload_for("inflight/steady.bin", 8 * 1024 * 1024);
|
||||
put_object(&dist.client(0)?, &bucket, "steady.bin", body.clone()).await?;
|
||||
|
||||
let live: Vec<_> = (0..3).map(|idx| dist.client(idx)).collect::<Result<Vec<_>, _>>()?;
|
||||
let worker_count = 12;
|
||||
let release = Arc::new(Barrier::new(worker_count + 1));
|
||||
let (started_tx, mut started_rx) = mpsc::unbounded_channel();
|
||||
let mut handles = Vec::new();
|
||||
for idx in 0..worker_count {
|
||||
let client = live[idx % live.len()].clone();
|
||||
let bucket = bucket.clone();
|
||||
let body = body.clone();
|
||||
let release = release.clone();
|
||||
let started_tx = started_tx.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
let response = client.get_object().bucket(&bucket).key("steady.bin").send().await?;
|
||||
if response.content_length() != Some(body.len() as i64) {
|
||||
return Err::<(), Box<dyn std::error::Error + Send + Sync>>(
|
||||
format!("worker {idx} received a wrong content length").into(),
|
||||
);
|
||||
}
|
||||
started_tx.send(idx)?;
|
||||
release.wait().await;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != body.as_slice() {
|
||||
return Err(format!("worker {idx} received corrupted bytes after peer kill").into());
|
||||
}
|
||||
Ok(())
|
||||
}));
|
||||
}
|
||||
drop(started_tx);
|
||||
for _ in 0..worker_count {
|
||||
timeout(Duration::from_secs(30), started_rx.recv())
|
||||
.await?
|
||||
.ok_or("a streaming GET exited before reaching the kill barrier")?;
|
||||
}
|
||||
|
||||
dist.cluster.stop_node(3)?;
|
||||
release.wait().await;
|
||||
for handle in handles {
|
||||
handle.await??;
|
||||
}
|
||||
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
assert_object_bytes(&dist.client(3)?, &bucket, "steady.bin", &body).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn blackholed_node_client_network_preserves_cluster_availability_and_recovers() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let proxy = crate::fault_proxy::FaultProxy::start(dist.cluster.nodes[3].address.parse()?).await?;
|
||||
let proxied_url = format!("http://{}", proxy.local_addr());
|
||||
let proxied_client = Client::from_conf(build_test_s3_config(
|
||||
&proxied_url,
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
None,
|
||||
"distributed-network-chaos",
|
||||
));
|
||||
|
||||
let result: TestResult = async {
|
||||
let bucket = unique_bucket("netfault");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let baseline = payload_for("network/baseline.bin", 1024 * 1024);
|
||||
put_object(&dist.client(0)?, &bucket, "baseline.bin", baseline.clone()).await?;
|
||||
assert_object_bytes(&proxied_client, &bucket, "baseline.bin", &baseline).await?;
|
||||
|
||||
proxy.set_mode(FaultMode::Blackhole);
|
||||
assert_eq!(proxy.mode(), FaultMode::Blackhole);
|
||||
if let Ok(Ok(_)) = timeout(
|
||||
Duration::from_secs(5),
|
||||
proxied_client.get_object().bucket(&bucket).key("baseline.bin").send(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Err("blackholed node endpoint unexpectedly completed a GET".into());
|
||||
}
|
||||
|
||||
let during = payload_for("network/during.bin", 1024 * 1024);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
put_object(&dist.client(1)?, &bucket, "during-blackhole.bin", during.clone()).await?;
|
||||
assert_object_bytes(&dist.client(2)?, &bucket, "baseline.bin", &baseline).await?;
|
||||
assert_object_bytes(&dist.client(0)?, &bucket, "during-blackhole.bin", &during).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
proxy.set_mode(FaultMode::Pass);
|
||||
retrying_get_equals(&proxied_client, &bucket, "during-blackhole.bin", &during, Duration::from_secs(30)).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
proxy.set_mode(FaultMode::Pass);
|
||||
proxy.shutdown().await;
|
||||
result
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for idx in 0..worker_count {
|
||||
let clients = clients.clone();
|
||||
let barrier = barrier.clone();
|
||||
let bucket = bucket.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
let writer = &clients[idx % clients.len()];
|
||||
let reader = &clients[(idx + 1) % clients.len()];
|
||||
let copier = &clients[(idx + 2) % clients.len()];
|
||||
let mut retained = Vec::with_capacity(rounds);
|
||||
for round in 0..rounds {
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
retained.push((copy_key, body));
|
||||
}
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>(retained)
|
||||
}));
|
||||
}
|
||||
|
||||
let mut inventory = Vec::new();
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
|
||||
decommission_status_json, payload_for, put_inventory_retrying, retrying_get_equals, retrying_put, start_decommission,
|
||||
unique_bucket, wait_for_decommission_complete, wait_for_decommission_running_with_progress,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_puts_during_decommission_do_not_lose_baseline_or_new_objects() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("concdecom");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let baseline_client = dist.client(0)?;
|
||||
let inventory = put_inventory_retrying(&baseline_client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
let barrier = Arc::new(Barrier::new(17));
|
||||
let mut handles = Vec::new();
|
||||
for idx in 0..16 {
|
||||
let clients = clients.clone();
|
||||
let barrier = barrier.clone();
|
||||
let bucket = bucket.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
let client = &clients[idx % clients.len()];
|
||||
let key = format!("live/{idx:02}.bin");
|
||||
let body = payload_for(&key, 8 * 1024);
|
||||
retrying_put(client, &bucket, &key, body.clone(), Duration::from_secs(45)).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((key, body))
|
||||
}));
|
||||
}
|
||||
|
||||
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
barrier.wait().await;
|
||||
|
||||
let mut live_objects = Vec::new();
|
||||
for handle in handles {
|
||||
live_objects.push(handle.await??);
|
||||
}
|
||||
let status = decommission_status_json(&dist.cluster).await?;
|
||||
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
|
||||
return Err(format!("decommission did not remain active across concurrent PUTs: {status}").into());
|
||||
}
|
||||
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
|
||||
let checker = dist.client(2)?;
|
||||
assert_inventory(&checker, &bucket, &inventory).await?;
|
||||
for (key, body) in live_objects {
|
||||
retrying_get_equals(&checker, &bucket, &key, &body, Duration::from_secs(30)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, enable_versioning, put_inventory_retrying,
|
||||
sha256_hex, start_decommission, unique_bucket, wait_for_decommission_active, wait_for_decommission_complete,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn decommission_does_not_alter_object_sha256_across_pools() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("integrity");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
enable_versioning(&client, &bucket).await?;
|
||||
let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
|
||||
let before: Vec<(String, String)> = inventory.iter().map(|(key, body)| (key.clone(), sha256_hex(body))).collect();
|
||||
|
||||
let versioned_key = "history/versioned.bin";
|
||||
let version_one = b"historical bytes before data movement".to_vec();
|
||||
let version_two = b"current bytes before data movement".to_vec();
|
||||
let version_one_id = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(versioned_key)
|
||||
.body(ByteStream::from(version_one.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("historical PUT omitted version ID")?
|
||||
.to_string();
|
||||
let version_two_id = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(versioned_key)
|
||||
.body(ByteStream::from(version_two.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("current PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let multipart_key = "multipart/moved.bin";
|
||||
let first_part = vec![0x31; 5 * 1024 * 1024];
|
||||
let second_part = vec![0x72; 1024 * 1024];
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = upload.upload_id().ok_or("movement multipart upload omitted upload ID")?;
|
||||
let uploaded_one = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(first_part.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let uploaded_two = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(second_part.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(uploaded_one.e_tag().ok_or("movement part 1 omitted ETag")?)
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(uploaded_two.e_tag().ok_or("movement part 2 omitted ETag")?)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
|
||||
let after_client = dist.client(2)?;
|
||||
assert_inventory(&after_client, &bucket, &inventory).await?;
|
||||
for (key, expected_hash) in before {
|
||||
let got = after_client.get_object().bucket(&bucket).key(&key).send().await?;
|
||||
let body = got.body.collect().await?.into_bytes();
|
||||
assert_eq!(sha256_hex(body.as_ref()), expected_hash, "checksum changed for {key} after decommission");
|
||||
}
|
||||
for (version_id, expected) in [(&version_one_id, &version_one), (&version_two_id, &version_two)] {
|
||||
let got = after_client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(versioned_key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(got.as_ref(), expected.as_slice(), "version {version_id} changed after decommission");
|
||||
}
|
||||
let mut expected_multipart = first_part;
|
||||
expected_multipart.extend_from_slice(&second_part);
|
||||
let got_multipart = after_client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(multipart_key)
|
||||
.send()
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(
|
||||
sha256_hex(got_multipart.as_ref()),
|
||||
sha256_hex(&expected_multipart),
|
||||
"multipart checksum changed after decommission"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, list_pools_json, put_inventory,
|
||||
put_inventory_retrying, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_active,
|
||||
wait_for_decommission_complete, wait_for_rebalance_active, wait_for_rebalance_complete,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_pool_expand_preserves_objects_then_rebalance() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("expand");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
let inventory = put_inventory(&client, &bucket, 64, 256 * 1024).await?;
|
||||
assert_inventory(&client, &bucket, &inventory).await?;
|
||||
|
||||
for expected_nodes in 2..=4 {
|
||||
let new_node = dist.append_pool_and_restart().await?;
|
||||
assert_eq!(new_node + 1, expected_nodes);
|
||||
assert_inventory(&dist.client(new_node)?, &bucket, &inventory).await?;
|
||||
}
|
||||
assert_eq!(dist.cluster.nodes.len(), 4);
|
||||
|
||||
// Prove that the expanded pool map is durable, and clear any recovery
|
||||
// latch raised while the newly-added pool replicas converged.
|
||||
dist.restart_current_binary_gracefully().await?;
|
||||
|
||||
let after_expand = dist.client(0)?;
|
||||
assert_inventory(&after_expand, &bucket, &inventory).await?;
|
||||
let peer = dist.client(3)?;
|
||||
assert_inventory(&peer, &bucket, &inventory).await?;
|
||||
|
||||
let rebalance_id = start_rebalance(&dist.cluster).await?;
|
||||
wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
|
||||
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
|
||||
assert_inventory(&peer, &bucket, &inventory).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_pool_decommission_moves_objects_without_loss() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("decom");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
let inventory = put_inventory_retrying(&client, &bucket, 96, 128 * 1024, Duration::from_secs(30)).await?;
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
let pools_before = list_pools_json(&dist.cluster).await?;
|
||||
let pool_count = pools_before
|
||||
.as_array()
|
||||
.map(Vec::len)
|
||||
.or_else(|| pools_before.get("pools").and_then(serde_json::Value::as_array).map(Vec::len))
|
||||
.ok_or_else(|| format!("pool list omitted an array: {pools_before}"))?;
|
||||
assert_eq!(pool_count, 4, "expected exactly four pools before decommission: {pools_before}");
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
wait_for_decommission_active(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
|
||||
let after = dist.client(2)?;
|
||||
assert_inventory(&after, &bucket, &inventory).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket, wait_until,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_multipart_and_cross_node_listing_agree() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("extra");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
|
||||
let key = "multipart.bin";
|
||||
let part1 = vec![0x41u8; 5 * 1024 * 1024];
|
||||
let part2 = vec![0x42u8; 5 * 1024 * 1024];
|
||||
let upload = client.create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let uploaded1 = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(part1.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let uploaded2 = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(2)
|
||||
.body(ByteStream::from(part2.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(uploaded1.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(2)
|
||||
.e_tag(uploaded2.e_tag().unwrap_or_default())
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let mut expected = part1;
|
||||
expected.extend_from_slice(&part2);
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
assert_object_bytes(&dist.client(node_idx)?, &bucket, key, &expected).await?;
|
||||
}
|
||||
|
||||
put_object(&client, &bucket, "list/a", b"a".to_vec()).await?;
|
||||
put_object(&dist.client(2)?, &bucket, "list/b", b"b".to_vec()).await?;
|
||||
let mut seen = Vec::new();
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
let listed = dist
|
||||
.client(node_idx)?
|
||||
.list_objects_v2()
|
||||
.bucket(&bucket)
|
||||
.prefix("list/")
|
||||
.send()
|
||||
.await?;
|
||||
let keys: Vec<String> = listed
|
||||
.contents()
|
||||
.iter()
|
||||
.filter_map(|object| object.key().map(str::to_string))
|
||||
.collect();
|
||||
seen.push(keys);
|
||||
}
|
||||
for keys in &seen[1..] {
|
||||
assert_eq!(&seen[0], keys, "list results diverged across nodes: {seen:?}");
|
||||
}
|
||||
|
||||
let got = get_object_bytes(&dist.client(3)?, &bucket, "list/a").await?;
|
||||
assert_eq!(got, b"a");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_list_buckets_agree_across_all_nodes() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("listed");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
put_object(&dist.client(0)?, &bucket, "seed.bin", b"seed".to_vec()).await?;
|
||||
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
let client = dist.client(node_idx)?;
|
||||
let name = bucket.clone();
|
||||
wait_until(
|
||||
Duration::from_secs(20),
|
||||
|| {
|
||||
let client = client.clone();
|
||||
let name = name.clone();
|
||||
async move {
|
||||
let listed = client.list_buckets().send().await?;
|
||||
Ok(listed.buckets().iter().any(|entry| entry.name() == Some(name.as_str())))
|
||||
}
|
||||
},
|
||||
&format!("node {node_idx} lists {bucket}"),
|
||||
)
|
||||
.await?;
|
||||
wait_until(
|
||||
Duration::from_secs(20),
|
||||
|| {
|
||||
let client = dist.client(node_idx).expect("client");
|
||||
let name = bucket.clone();
|
||||
async move { Ok(get_object_bytes(&client, &name, "seed.bin").await.ok() == Some(b"seed".to_vec())) }
|
||||
},
|
||||
&format!("node {node_idx} reads seed.bin"),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,35 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! 4-node 4-drive distributed e2e coverage.
|
||||
//!
|
||||
//! Selected by `[profile.e2e-distributed]` and run from
|
||||
//! `.github/workflows/e2e-distributed.yml`. Excluded from `e2e-full` because
|
||||
//! each case starts four real `rustfs` processes.
|
||||
|
||||
mod chaos_test;
|
||||
mod concurrency_stability_test;
|
||||
mod concurrent_data_movement_test;
|
||||
mod data_integrity_movement_test;
|
||||
mod expand_decommission_rebalance_test;
|
||||
mod extra_test;
|
||||
mod harness;
|
||||
mod object_lock_test;
|
||||
mod observability_test;
|
||||
mod replication_quota_test;
|
||||
mod s3_basic_test;
|
||||
mod s3_during_data_movement_test;
|
||||
mod site_replication_test;
|
||||
mod upgrade_test;
|
||||
mod versioning_test;
|
||||
@@ -1,219 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use crate::object_lock::common::{
|
||||
delete_object_with_bypass, put_object_lock_configuration, put_object_with_legal_hold, put_object_with_retention,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::delete_object::DeleteObjectError;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ObjectLockRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
|
||||
fn delete_denied(error: &SdkError<DeleteObjectError>, context: &str) -> TestResult {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
if code == Some("AccessDenied") {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("{context}: expected AccessDenied, got {error:?}").into())
|
||||
}
|
||||
}
|
||||
|
||||
async fn expect_versioned_delete_denied(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: &str,
|
||||
bypass: bool,
|
||||
context: &str,
|
||||
) -> TestResult {
|
||||
match delete_object_with_bypass(client, bucket, key, Some(version_id), bypass).await {
|
||||
Ok(_) => Err(format!("{context}: DeleteObject of retained version must be denied").into()),
|
||||
Err(error) => delete_denied(error.as_ref(), context),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_object_lock_worm_blocks_delete() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let client = dist.client(0)?;
|
||||
let peer = dist.client(2)?;
|
||||
let bucket = unique_bucket("objlock");
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(&bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let retain_until = Utc::now() + ChronoDuration::days(1);
|
||||
|
||||
let compliance_key = "compliance.bin";
|
||||
let compliance_version = put_object_with_retention(
|
||||
&client,
|
||||
&bucket,
|
||||
compliance_key,
|
||||
b"locked-compliance",
|
||||
ObjectLockRetentionMode::Compliance,
|
||||
retain_until,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Unversioned DELETE is allowed: it only creates a delete marker. WORM
|
||||
// applies to a specific version id.
|
||||
let marker = peer.delete_object().bucket(&bucket).key(compliance_key).send().await?;
|
||||
assert_eq!(
|
||||
marker.delete_marker(),
|
||||
Some(true),
|
||||
"unversioned DELETE on a locked object must create a delete marker"
|
||||
);
|
||||
|
||||
expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, false, "COMPLIANCE without bypass")
|
||||
.await?;
|
||||
expect_versioned_delete_denied(&peer, &bucket, compliance_key, &compliance_version, true, "COMPLIANCE with bypass").await?;
|
||||
|
||||
let governance_key = "governance.bin";
|
||||
let governance_version = put_object_with_retention(
|
||||
&client,
|
||||
&bucket,
|
||||
governance_key,
|
||||
b"locked-governance",
|
||||
ObjectLockRetentionMode::Governance,
|
||||
retain_until,
|
||||
)
|
||||
.await?;
|
||||
|
||||
expect_versioned_delete_denied(&peer, &bucket, governance_key, &governance_version, false, "GOVERNANCE without bypass")
|
||||
.await?;
|
||||
delete_object_with_bypass(&peer, &bucket, governance_key, Some(&governance_version), true).await?;
|
||||
let deleted_governance = peer
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(governance_key)
|
||||
.version_id(&governance_version)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("GOVERNANCE bypass must remove the retained version");
|
||||
assert_eq!(
|
||||
deleted_governance.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"deleted GOVERNANCE version returned an unexpected HEAD result: {deleted_governance:?}"
|
||||
);
|
||||
|
||||
let hold_key = "legal-hold.bin";
|
||||
let hold_version =
|
||||
put_object_with_legal_hold(&client, &bucket, hold_key, b"legal-hold", ObjectLockLegalHoldStatus::On).await?;
|
||||
expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, false, "legal hold without bypass").await?;
|
||||
expect_versioned_delete_denied(&peer, &bucket, hold_key, &hold_version, true, "legal hold with bypass").await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_default_retention_is_visible_and_non_lock_bucket_rejects_configuration() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
let bucket = unique_bucket("default-lock");
|
||||
|
||||
writer
|
||||
.create_bucket()
|
||||
.bucket(&bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
put_object_lock_configuration(&writer, &bucket, ObjectLockRetentionMode::Governance, Some(1), None).await?;
|
||||
|
||||
let key = "default-governance.bin";
|
||||
let put = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"default retention payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let version_id = put.version_id().ok_or("default-retained PUT omitted version ID")?;
|
||||
|
||||
let config = reader.get_object_lock_configuration().bucket(&bucket).send().await?;
|
||||
let default_retention = config
|
||||
.object_lock_configuration()
|
||||
.and_then(|configuration| configuration.rule())
|
||||
.and_then(|rule| rule.default_retention())
|
||||
.ok_or("GetObjectLockConfiguration omitted default retention")?;
|
||||
assert_eq!(default_retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
|
||||
assert_eq!(default_retention.days(), Some(1));
|
||||
|
||||
let retention = reader
|
||||
.get_object_retention()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await?;
|
||||
let retention = retention.retention().ok_or("GetObjectRetention omitted applied retention")?;
|
||||
assert_eq!(retention.mode().map(|mode| mode.as_str()), Some("GOVERNANCE"));
|
||||
let retain_until = retention
|
||||
.retain_until_date()
|
||||
.ok_or("default retention omitted retain-until date")?;
|
||||
assert!(retain_until.secs() > Utc::now().timestamp(), "default retention is not in the future");
|
||||
|
||||
let versioning = reader.get_bucket_versioning().bucket(&bucket).send().await?;
|
||||
assert_eq!(versioning.status().map(|status| status.as_str()), Some("Enabled"));
|
||||
expect_versioned_delete_denied(&reader, &bucket, key, version_id, false, "default GOVERNANCE retention without bypass")
|
||||
.await?;
|
||||
|
||||
let plain_bucket = unique_bucket("no-lock");
|
||||
dist.create_bucket(&plain_bucket).await?;
|
||||
let configuration = ObjectLockConfiguration::builder()
|
||||
.object_lock_enabled(ObjectLockEnabled::Enabled)
|
||||
.rule(
|
||||
ObjectLockRule::builder()
|
||||
.default_retention(
|
||||
DefaultRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Governance)
|
||||
.days(1)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
let error = writer
|
||||
.put_object_lock_configuration()
|
||||
.bucket(&plain_bucket)
|
||||
.object_lock_configuration(configuration)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unversioned bucket must reject Object Lock enablement");
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or("non-lock bucket rejection was not an S3 service error")?;
|
||||
assert_eq!(service_error.code(), Some("InvalidBucketState"), "unexpected error: {error:?}");
|
||||
assert_eq!(
|
||||
service_error.message(),
|
||||
Some("Object Lock configuration cannot be enabled on existing buckets"),
|
||||
"unexpected error: {error:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, cluster_admin_ok, unique_bucket, wait_for_ready};
|
||||
use crate::common::{admin_request, init_logging, local_http_client};
|
||||
use aws_sdk_s3::operation::RequestId;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use bytes::Bytes;
|
||||
use http::Method;
|
||||
use http_body_util::{BodyExt, Empty};
|
||||
use hyper::body::Incoming;
|
||||
use hyper::service::service_fn;
|
||||
use hyper::{Request, Response};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use local_ip_address::local_ip;
|
||||
use rustfs_madmin::metrics::RealtimeMetrics;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use serde_json::Value;
|
||||
use std::convert::Infallible;
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::{Instant, timeout};
|
||||
|
||||
async fn spawn_audit_collector() -> TestResult<(String, mpsc::UnboundedReceiver<Value>, JoinHandle<()>)> {
|
||||
let listener = TcpListener::bind("0.0.0.0:0").await?;
|
||||
let endpoint = format!("http://{}/audit", std::net::SocketAddr::new(local_ip()?, listener.local_addr()?.port()));
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let handle = tokio::spawn(async move {
|
||||
loop {
|
||||
let Ok((stream, _)) = listener.accept().await else {
|
||||
return;
|
||||
};
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
let tx = tx.clone();
|
||||
async move {
|
||||
let method = request.method().clone();
|
||||
if let Ok(body) = request.into_body().collect().await
|
||||
&& method == Method::POST
|
||||
&& let Ok(payload) = serde_json::from_slice::<Value>(&body.to_bytes())
|
||||
{
|
||||
if let Some(records) = payload["Records"].as_array() {
|
||||
for entry in records {
|
||||
let _ = tx.send(entry.clone());
|
||||
}
|
||||
} else {
|
||||
let _ = tx.send(payload);
|
||||
}
|
||||
}
|
||||
Ok::<_, Infallible>(Response::new(Empty::<Bytes>::new()))
|
||||
}
|
||||
});
|
||||
let _ = hyper::server::conn::http1::Builder::new()
|
||||
.serve_connection(TokioIo::new(stream), service)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
});
|
||||
Ok((endpoint, rx, handle))
|
||||
}
|
||||
|
||||
async fn wait_for_audit_entry(
|
||||
rx: &mut mpsc::UnboundedReceiver<Value>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
request_id: &str,
|
||||
) -> TestResult<Value> {
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
let mut seen = Vec::new();
|
||||
loop {
|
||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return Err(format!(
|
||||
"audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
|
||||
seen.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let entry = match timeout(remaining, rx.recv()).await {
|
||||
Ok(Some(entry)) => entry,
|
||||
Ok(None) => return Err("audit collector stopped before the expected entry arrived".into()),
|
||||
Err(_) => {
|
||||
return Err(format!(
|
||||
"audit webhook did not receive PutObject for {bucket}/{key}; received {} other records: {seen:?}",
|
||||
seen.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
};
|
||||
if entry["api"]["name"].as_str() == Some("s3:PutObject")
|
||||
&& entry["api"]["bucket"].as_str() == Some(bucket)
|
||||
&& entry["api"]["object"].as_str() == Some(key)
|
||||
&& entry["requestID"].as_str() == Some(request_id)
|
||||
{
|
||||
return Ok(entry);
|
||||
}
|
||||
if seen.len() < 8 {
|
||||
seen.push(format!(
|
||||
"api={:?} bucket={:?} object={:?} requestID={:?}",
|
||||
entry["api"]["name"].as_str(),
|
||||
entry["api"]["bucket"].as_str(),
|
||||
entry["api"]["object"].as_str(),
|
||||
entry["requestID"].as_str()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent() -> TestResult {
|
||||
init_logging();
|
||||
let (audit_endpoint, mut audit_entries, collector) = spawn_audit_collector().await?;
|
||||
let audit_origin = reqwest::Url::parse(&audit_endpoint)?.origin().ascii_serialization();
|
||||
let audit_env = [
|
||||
("RUSTFS_AUDIT_ENABLE", "true"),
|
||||
("RUSTFS_AUDIT_WEBHOOK_ENABLE_DISTRIBUTED", "on"),
|
||||
("RUSTFS_AUDIT_WEBHOOK_ENDPOINT_DISTRIBUTED", audit_endpoint.as_str()),
|
||||
(ENV_OUTBOUND_ALLOW_ORIGINS, audit_origin.as_str()),
|
||||
];
|
||||
let mut dist = DistCluster::new_stopped_with_env(DistLayout::FourByFour, &audit_env).await?;
|
||||
for node_idx in 0..dist.cluster.nodes.len() {
|
||||
let queue_dir = format!("{}/audit-queue-node-{node_idx}", dist.cluster.temp_dir);
|
||||
tokio::fs::create_dir_all(&queue_dir).await?;
|
||||
dist.cluster
|
||||
.set_node_env(node_idx, "RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_DISTRIBUTED", queue_dir)?;
|
||||
}
|
||||
dist.cluster.start().await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
|
||||
let http = local_http_client();
|
||||
for node in &dist.cluster.nodes {
|
||||
for probe in ["ready", "live"] {
|
||||
let response = http.get(format!("{}/health/{probe}", node.url)).send().await?;
|
||||
assert!(
|
||||
response.status().is_success(),
|
||||
"node {} {probe} probe failed: {}",
|
||||
node.address,
|
||||
response.status()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
|
||||
let info: Value = serde_json::from_str(&info_body)?;
|
||||
let servers = info["info"]["servers"]
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("admin info omitted servers: {info}"))?;
|
||||
assert_eq!(servers.len(), 4, "admin info did not report all four nodes: {info}");
|
||||
|
||||
let storage_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
|
||||
let storage: Value = serde_json::from_str(&storage_body)?;
|
||||
let disks = storage["info"]["disks"]
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("storageinfo omitted disks: {storage}"))?;
|
||||
assert_eq!(disks.len(), 16, "storageinfo did not report all sixteen drives: {storage}");
|
||||
assert!(
|
||||
disks.iter().all(|disk| {
|
||||
disk["state"].as_str().is_some_and(|state| state.eq_ignore_ascii_case("ok"))
|
||||
&& disk["runtimeState"]
|
||||
.as_str()
|
||||
.is_some_and(|state| state.eq_ignore_ascii_case("online"))
|
||||
}),
|
||||
"storageinfo reported a drive that was not healthy and online: {storage}"
|
||||
);
|
||||
|
||||
for (node_idx, node) in dist.cluster.nodes.iter().enumerate() {
|
||||
let (status, metrics_body) = admin_request(
|
||||
&node.url,
|
||||
Method::GET,
|
||||
"/rustfs/admin/v3/metrics?n=1&by-host=true&by-disk=true",
|
||||
None,
|
||||
&dist.cluster.access_key,
|
||||
&dist.cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
assert!(status.is_success(), "node {node_idx} metrics failed: {status} {metrics_body}");
|
||||
let sample: RealtimeMetrics = serde_json::from_str(
|
||||
metrics_body
|
||||
.lines()
|
||||
.next()
|
||||
.ok_or_else(|| format!("node {node_idx} returned empty metrics"))?,
|
||||
)?;
|
||||
assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
|
||||
assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
|
||||
assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
|
||||
}
|
||||
|
||||
let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
|
||||
let targets: Value = serde_json::from_str(&targets_body)?;
|
||||
let configured = targets["audit_endpoints"]
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("audit target list omitted audit_endpoints: {targets}"))?
|
||||
.iter()
|
||||
.any(|target| target["account_id"].as_str() == Some("distributed") && target["service"].as_str() == Some("webhook"));
|
||||
assert!(configured, "configured audit webhook was missing: {targets}");
|
||||
|
||||
let bucket = unique_bucket("audit");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let key = "correlated/audit-object.bin";
|
||||
let put = dist
|
||||
.client(2)?
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"distributed audit payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let request_id = put.request_id().ok_or("PutObject response omitted request ID")?;
|
||||
let audit = wait_for_audit_entry(&mut audit_entries, &bucket, key, request_id).await?;
|
||||
assert_eq!(
|
||||
audit["api"]["status_code"].as_i64(),
|
||||
Some(200),
|
||||
"audit entry did not report success: {audit}"
|
||||
);
|
||||
assert!(
|
||||
!audit.to_string().contains(&dist.cluster.secret_key),
|
||||
"audit entry leaked the root secret key"
|
||||
);
|
||||
|
||||
collector.abort();
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, enable_versioning, put_bucket_replication, put_object, retrying_put, set_bucket_quota,
|
||||
set_remote_target, unique_bucket, wait_for_ready, wait_for_replicated_bytes, wait_until,
|
||||
};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, init_logging};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use std::time::Duration;
|
||||
|
||||
async fn wait_for_replication_status(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
expected: &[&str],
|
||||
timeout: Duration,
|
||||
) -> TestResult {
|
||||
wait_until(
|
||||
timeout,
|
||||
|| async {
|
||||
let head = client.head_object().bucket(bucket).key(key).send().await?;
|
||||
Ok(head
|
||||
.replication_status()
|
||||
.is_some_and(|status| expected.contains(&status.as_str())))
|
||||
},
|
||||
&format!("replication status for {bucket}/{key} in {expected:?}"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_bucket_replication_converges_to_peer_cluster() -> TestResult {
|
||||
init_logging();
|
||||
let (source, mut target) = DistCluster::start_replication_pair().await?;
|
||||
let source_bucket = unique_bucket("replsrc");
|
||||
let target_bucket = unique_bucket("repldst");
|
||||
source.create_bucket(&source_bucket).await?;
|
||||
target.create_bucket(&target_bucket).await?;
|
||||
|
||||
let source_client = source.client(0)?;
|
||||
let target_client = target.client(0)?;
|
||||
enable_versioning(&source_client, &source_bucket).await?;
|
||||
enable_versioning(&target_client, &target_bucket).await?;
|
||||
|
||||
let arn = set_remote_target(&source.cluster, &source_bucket, &target.cluster, &target_bucket).await?;
|
||||
put_bucket_replication(&source.cluster, &source_bucket, &arn).await?;
|
||||
|
||||
let key = "replicated/metadata-and-tags.bin";
|
||||
let body = b"distributed-bucket-replication".to_vec();
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(&source_bucket)
|
||||
.key(key)
|
||||
.metadata("origin", "four-node-source")
|
||||
.tagging("suite=distributed&shape=metadata")
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_replicated_bytes(&target_client, &target_bucket, key, &body, Duration::from_secs(45)).await?;
|
||||
wait_for_replication_status(&source_client, &source_bucket, key, &["COMPLETED"], Duration::from_secs(30)).await?;
|
||||
|
||||
let peer_read = target.client(3)?;
|
||||
wait_for_replicated_bytes(&peer_read, &target_bucket, key, &body, Duration::from_secs(15)).await?;
|
||||
let replica_head = peer_read.head_object().bucket(&target_bucket).key(key).send().await?;
|
||||
assert_eq!(
|
||||
replica_head
|
||||
.metadata()
|
||||
.and_then(|metadata| metadata.get("origin"))
|
||||
.map(String::as_str),
|
||||
Some("four-node-source")
|
||||
);
|
||||
assert_eq!(replica_head.replication_status().map(|status| status.as_str()), Some("REPLICA"));
|
||||
let replica_tags = peer_read.get_object_tagging().bucket(&target_bucket).key(key).send().await?;
|
||||
let tags: std::collections::BTreeMap<_, _> = replica_tags.tag_set().iter().map(|tag| (tag.key(), tag.value())).collect();
|
||||
assert_eq!(tags.get("suite"), Some(&"distributed"));
|
||||
assert_eq!(tags.get("shape"), Some(&"metadata"));
|
||||
|
||||
target.cluster.stop();
|
||||
let outage_key = "replicated/queued-during-target-outage.bin";
|
||||
let outage_body = b"retry-after-target-restart".to_vec();
|
||||
put_object(&source_client, &source_bucket, outage_key, outage_body.clone()).await?;
|
||||
wait_for_replication_status(
|
||||
&source_client,
|
||||
&source_bucket,
|
||||
outage_key,
|
||||
&["PENDING", "FAILED"],
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
|
||||
target.cluster.start().await?;
|
||||
wait_for_ready(&target.cluster).await?;
|
||||
wait_for_replicated_bytes(&target.client(2)?, &target_bucket, outage_key, &outage_body, Duration::from_secs(90)).await?;
|
||||
wait_for_replication_status(&source_client, &source_bucket, outage_key, &["COMPLETED"], Duration::from_secs(45)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_hard_quota_rejects_over_limit_put() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start_with_env(DistLayout::FourByFour, FAST_DATA_USAGE_SCANNER_ENV).await?;
|
||||
let bucket = unique_bucket("quota");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
set_bucket_quota(&dist.cluster, &bucket, 8 * 1024).await?;
|
||||
|
||||
let client = dist.client(1)?;
|
||||
retrying_put(&client, &bucket, "small.bin", vec![0u8; 1024], Duration::from_secs(30)).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
let (status, body) = super::harness::cluster_admin(
|
||||
&dist.cluster,
|
||||
Method::GET,
|
||||
&format!("/rustfs/admin/v3/quota-stats/{bucket}"),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
if !status.is_success() {
|
||||
return Ok(false);
|
||||
}
|
||||
let stats: serde_json::Value =
|
||||
serde_json::from_str(&body).map_err(|error| format!("quota stats returned invalid JSON: {error}: {body}"))?;
|
||||
let usage = stats
|
||||
.get("current_usage")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
.ok_or_else(|| format!("quota stats omitted current_usage: {stats}"))?;
|
||||
Ok(usage >= 1024)
|
||||
},
|
||||
"quota stats observe small object",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let oversized_key = "too-big.bin";
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(oversized_key)
|
||||
.body(vec![0u8; 16 * 1024].into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("hard quota must reject the oversized PUT");
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or("quota rejection was not an S3 service error")?;
|
||||
assert_eq!(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(400),
|
||||
"quota rejection must be HTTP 400: {error:?}"
|
||||
);
|
||||
assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}");
|
||||
assert!(
|
||||
service_error
|
||||
.message()
|
||||
.is_some_and(|message| message.starts_with("Bucket quota exceeded")),
|
||||
"PUT must fail specifically at quota admission: {error:?}"
|
||||
);
|
||||
|
||||
let missing = client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(oversized_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an object rejected by quota must not become visible");
|
||||
assert_eq!(
|
||||
missing.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"quota-rejected object returned an unexpected HEAD result: {missing:?}"
|
||||
);
|
||||
|
||||
let listed = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
assert!(
|
||||
listed.contents().iter().all(|object| object.key() != Some(oversized_key)),
|
||||
"quota-rejected key leaked into ListObjectsV2"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,258 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, get_object_bytes, put_object, unique_bucket};
|
||||
use crate::common::{init_logging, local_http_client};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::presigning::PresigningConfig;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{Delete, MetadataDirective, ObjectIdentifier};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_s3_put_get_head_list_copy_rename_delete_and_presign() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("s3basic");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
let key = "dir/object.bin";
|
||||
let body = vec![0xA5u8; 256 * 1024];
|
||||
put_object(&writer, &bucket, key, body.clone()).await?;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(body.len() as i64));
|
||||
assert_object_bytes(&reader, &bucket, key, &body).await?;
|
||||
|
||||
let ranged = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.range("bytes=0-15")
|
||||
.send()
|
||||
.await?;
|
||||
let ranged_body = ranged.body.collect().await?.into_bytes();
|
||||
assert_eq!(ranged_body.as_ref(), &body[..16]);
|
||||
|
||||
let listed = reader.list_objects_v2().bucket(&bucket).prefix("dir/").send().await?;
|
||||
let keys: Vec<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(keys, vec![key]);
|
||||
|
||||
let copy_key = "dir/object-copy.bin";
|
||||
reader
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(copy_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.metadata_directive(MetadataDirective::Copy)
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(&writer, &bucket, copy_key, &body).await?;
|
||||
|
||||
let moved_key = "dir/object-moved.bin";
|
||||
writer
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(moved_key)
|
||||
.copy_source(format!("{bucket}/{copy_key}"))
|
||||
.send()
|
||||
.await?;
|
||||
writer.delete_object().bucket(&bucket).key(copy_key).send().await?;
|
||||
match writer.head_object().bucket(&bucket).key(copy_key).send().await {
|
||||
Ok(_) => return Err("copied source still present after rename delete".into()),
|
||||
Err(error) if error.as_service_error().is_some_and(|err| err.is_not_found()) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
assert_object_bytes(&reader, &bucket, moved_key, &body).await?;
|
||||
|
||||
let presigned = writer
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.presigned(PresigningConfig::expires_in(Duration::from_secs(120))?)
|
||||
.await?;
|
||||
let response = local_http_client().get(presigned.uri().to_string()).send().await?;
|
||||
assert!(response.status().is_success(), "presigned GET failed: {}", response.status());
|
||||
let presigned_body = response.bytes().await?;
|
||||
assert_eq!(presigned_body.as_ref(), body.as_slice());
|
||||
|
||||
let empty_key = "empty";
|
||||
put_object(&writer, &bucket, empty_key, Vec::new()).await?;
|
||||
let empty = get_object_bytes(&reader, &bucket, empty_key).await?;
|
||||
assert!(empty.is_empty());
|
||||
|
||||
let deleted = writer
|
||||
.delete_objects()
|
||||
.bucket(&bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key(key).build()?)
|
||||
.objects(ObjectIdentifier::builder().key(moved_key).build()?)
|
||||
.objects(ObjectIdentifier::builder().key(empty_key).build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(deleted.errors().is_empty(), "DeleteObjects reported failures: {deleted:?}");
|
||||
assert_eq!(deleted.deleted().len(), 3, "DeleteObjects did not acknowledge every key");
|
||||
|
||||
let remaining = reader.list_objects_v2().bucket(&bucket).send().await?;
|
||||
assert!(remaining.contents().is_empty(), "bucket still has objects after delete");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_s3_metadata_tags_special_keys_pagination_and_multipart_abort() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("s3matrix");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
|
||||
let special_key = "unicode/测试 space+percent%25.txt";
|
||||
let special_body = b"metadata and tagging survive distributed routing".to_vec();
|
||||
let put = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(special_key)
|
||||
.metadata("test-meta", "distributed")
|
||||
.tagging("purpose=compatibility&scope=four-by-four")
|
||||
.body(ByteStream::from(special_body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let etag = put.e_tag().ok_or("PutObject omitted ETag")?.to_string();
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(special_key).send().await?;
|
||||
assert_eq!(
|
||||
head.metadata()
|
||||
.and_then(|metadata| metadata.get("test-meta"))
|
||||
.map(String::as_str),
|
||||
Some("distributed")
|
||||
);
|
||||
assert_eq!(head.e_tag(), Some(etag.as_str()));
|
||||
let tags = reader.get_object_tagging().bucket(&bucket).key(special_key).send().await?;
|
||||
let actual_tags: std::collections::BTreeMap<_, _> = tags
|
||||
.tag_set()
|
||||
.iter()
|
||||
.map(|tag| (tag.key().to_string(), tag.value().to_string()))
|
||||
.collect();
|
||||
assert_eq!(actual_tags.get("purpose").map(String::as_str), Some("compatibility"));
|
||||
assert_eq!(actual_tags.get("scope").map(String::as_str), Some("four-by-four"));
|
||||
|
||||
let conditional = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(special_key)
|
||||
.if_match(&etag)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(conditional.body.collect().await?.into_bytes().as_ref(), special_body.as_slice());
|
||||
let invalid_range = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(special_key)
|
||||
.range("bytes=999999-1000000")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unsatisfiable range must fail");
|
||||
assert_eq!(
|
||||
invalid_range.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("InvalidRange"),
|
||||
"unexpected invalid-range error: {invalid_range:?}"
|
||||
);
|
||||
|
||||
let upload_key = "multipart/aborted.bin";
|
||||
let upload = writer
|
||||
.create_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = upload.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?;
|
||||
writer
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(vec![0x5Au8; 5 * 1024 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
let pending = reader
|
||||
.list_multipart_uploads()
|
||||
.bucket(&bucket)
|
||||
.prefix("multipart/")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(pending.uploads().iter().any(|entry| entry.upload_id() == Some(upload_id)));
|
||||
writer
|
||||
.abort_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
let after_abort = reader
|
||||
.list_multipart_uploads()
|
||||
.bucket(&bucket)
|
||||
.prefix("multipart/")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(after_abort.uploads().iter().all(|entry| entry.upload_id() != Some(upload_id)));
|
||||
let aborted_head = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(upload_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("aborted multipart upload must not create an object");
|
||||
assert_eq!(
|
||||
aborted_head.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"aborted multipart object returned an unexpected HEAD result: {aborted_head:?}"
|
||||
);
|
||||
|
||||
for index in 0..113 {
|
||||
let key = format!("page/{index:04}.txt");
|
||||
put_object(&writer, &bucket, &key, format!("page-{index}").into_bytes()).await?;
|
||||
}
|
||||
let mut token = None;
|
||||
let mut paged_keys = Vec::new();
|
||||
loop {
|
||||
let page = reader
|
||||
.list_objects_v2()
|
||||
.bucket(&bucket)
|
||||
.prefix("page/")
|
||||
.max_keys(37)
|
||||
.set_continuation_token(token.take())
|
||||
.send()
|
||||
.await?;
|
||||
paged_keys.extend(page.contents().iter().filter_map(|object| object.key().map(str::to_string)));
|
||||
if page.is_truncated() != Some(true) {
|
||||
break;
|
||||
}
|
||||
token = Some(
|
||||
page.next_continuation_token()
|
||||
.ok_or("truncated ListObjectsV2 page omitted next continuation token")?
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
assert_eq!(paged_keys.len(), 113);
|
||||
let expected: Vec<_> = (0..113).map(|index| format!("page/{index:04}.txt")).collect();
|
||||
assert_eq!(paged_keys, expected, "pagination lost, duplicated, or reordered keys");
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
|
||||
decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
|
||||
retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
|
||||
wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start(DistLayout::SingleNodeFourDrive).await?;
|
||||
let bucket = unique_bucket("s3move");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = dist.client(0)?;
|
||||
let inventory = put_inventory_retrying(&client, &bucket, 96, 256 * 1024, Duration::from_secs(30)).await?;
|
||||
dist.expand_to_four_pools().await?;
|
||||
|
||||
start_decommission(&dist.cluster, DECOMMISSION_POOL_ID).await?;
|
||||
wait_for_decommission_running_with_progress(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(30)).await?;
|
||||
let live = dist.client(2)?;
|
||||
retrying_put(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-decommission.bin",
|
||||
b"written-while-decommissioning".to_vec(),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
retrying_get_equals(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-decommission.bin",
|
||||
b"written-while-decommissioning",
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
let listed = live.list_objects_v2().bucket(&bucket).send().await?;
|
||||
assert!(
|
||||
listed
|
||||
.contents()
|
||||
.iter()
|
||||
.any(|object| object.key() == Some("during-decommission.bin")),
|
||||
"list during decommission missed the newly written key"
|
||||
);
|
||||
let status = decommission_status_json(&dist.cluster).await?;
|
||||
if !decommission_running_with_progress(&status, DECOMMISSION_POOL_ID)? {
|
||||
return Err(format!("decommission did not remain active across the S3 operations: {status}").into());
|
||||
}
|
||||
|
||||
wait_for_decommission_complete(&dist.cluster, DECOMMISSION_POOL_ID, Duration::from_secs(180)).await?;
|
||||
assert_inventory(&live, &bucket, &inventory).await?;
|
||||
|
||||
let rebalance_id = start_rebalance(&dist.cluster).await?;
|
||||
wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
|
||||
retrying_put(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-rebalance.bin",
|
||||
b"written-while-rebalancing".to_vec(),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
retrying_get_equals(
|
||||
&live,
|
||||
&bucket,
|
||||
"during-rebalance.bin",
|
||||
b"written-while-rebalancing",
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
let status = rebalance_status_json(&dist.cluster).await?;
|
||||
if !rebalance_running_with_progress(&status, &rebalance_id)? {
|
||||
return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
|
||||
}
|
||||
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
|
||||
assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, TestResult, cluster_admin_ok, enable_versioning, put_object, unique_bucket, wait_for_replicated_bytes,
|
||||
wait_until,
|
||||
};
|
||||
use crate::common::{init_logging, signed_request};
|
||||
use http::{Method, StatusCode};
|
||||
use rustfs_madmin::{PeerSite, ReplicateAddStatus, SiteReplicationInfo, SyncStatus};
|
||||
use std::time::Duration;
|
||||
|
||||
async fn site_replication_add(
|
||||
cluster: &crate::common::RustFSTestClusterEnvironment,
|
||||
sites: &[PeerSite],
|
||||
) -> TestResult<ReplicateAddStatus> {
|
||||
let url = format!("{}/rustfs/admin/v3/site-replication/add?replicateILMExpiry=false", cluster.nodes[0].url);
|
||||
let response = signed_request(
|
||||
Method::PUT,
|
||||
&url,
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
Some(serde_json::to_vec(sites)?),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("site replication add failed: {status} {body}").into());
|
||||
}
|
||||
Ok(serde_json::from_slice(&response.bytes().await?)?)
|
||||
}
|
||||
|
||||
async fn site_replication_info(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult<SiteReplicationInfo> {
|
||||
let body = cluster_admin_ok(cluster, Method::GET, "/rustfs/admin/v3/site-replication/info", None).await?;
|
||||
Ok(serde_json::from_str(&body)?)
|
||||
}
|
||||
|
||||
async fn wait_for_site_replication_enabled(cluster: &crate::common::RustFSTestClusterEnvironment) -> TestResult {
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
let info = site_replication_info(cluster).await?;
|
||||
Ok(info.enabled && info.sites.len() == 2 && info.sites.iter().all(|site| site.sync_state == SyncStatus::Enable))
|
||||
},
|
||||
"site replication enabled with two synchronized sites",
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_site_replication_replicates_object_to_peer_site() -> TestResult {
|
||||
init_logging();
|
||||
let (site_a, site_b) = DistCluster::start_replication_pair().await?;
|
||||
let bucket = unique_bucket("siterepl");
|
||||
site_a.create_bucket(&bucket).await?;
|
||||
site_b.create_bucket(&bucket).await?;
|
||||
|
||||
let client_a = site_a.client(0)?;
|
||||
let client_b = site_b.client(0)?;
|
||||
enable_versioning(&client_a, &bucket).await?;
|
||||
enable_versioning(&client_b, &bucket).await?;
|
||||
|
||||
let sites = vec![
|
||||
PeerSite {
|
||||
name: "site-a".to_string(),
|
||||
endpoint: site_a.cluster.nodes[0].url.clone(),
|
||||
access_key: site_a.cluster.access_key.clone(),
|
||||
secret_key: site_a.cluster.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
PeerSite {
|
||||
name: "site-b".to_string(),
|
||||
endpoint: site_b.cluster.nodes[0].url.clone(),
|
||||
access_key: site_b.cluster.access_key.clone(),
|
||||
secret_key: site_b.cluster.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let add_status = site_replication_add(&site_a.cluster, &sites).await?;
|
||||
assert!(
|
||||
add_status.success && add_status.err_detail.is_empty() && add_status.initial_sync_error_message.is_empty(),
|
||||
"site replication add reported failure: {add_status:?}"
|
||||
);
|
||||
wait_for_site_replication_enabled(&site_a.cluster).await?;
|
||||
wait_for_site_replication_enabled(&site_b.cluster).await?;
|
||||
|
||||
let info_a = site_replication_info(&site_a.cluster).await?;
|
||||
let remote = info_a
|
||||
.sites
|
||||
.iter()
|
||||
.find(|site| site.name == "site-b")
|
||||
.ok_or_else(|| format!("site A info omitted the configured site-b peer: {info_a:?}"))?;
|
||||
assert_eq!(remote.endpoint, site_b.cluster.nodes[0].url);
|
||||
let deployment_ids: std::collections::BTreeSet<_> = info_a.sites.iter().map(|site| site.deployment_id.as_str()).collect();
|
||||
assert!(
|
||||
deployment_ids.iter().all(|deployment_id| !deployment_id.is_empty()) && deployment_ids.len() == 2,
|
||||
"site peers must have two distinct non-empty deployment IDs: {info_a:?}"
|
||||
);
|
||||
assert!(info_a.retry_stats.is_none(), "site A has pending replication retries: {info_a:?}");
|
||||
assert!(info_a.pending_operation.is_none(), "site A has a pending operation: {info_a:?}");
|
||||
|
||||
let key = "site-object.bin";
|
||||
let body = b"four-node-site-replication".to_vec();
|
||||
put_object(&client_a, &bucket, key, body.clone()).await?;
|
||||
wait_for_replicated_bytes(&client_b, &bucket, key, &body, Duration::from_secs(60)).await?;
|
||||
|
||||
let peer_b = site_b.client(3)?;
|
||||
wait_for_replicated_bytes(&peer_b, &bucket, key, &body, Duration::from_secs(20)).await?;
|
||||
|
||||
let reverse_key = "reverse/site-object.bin";
|
||||
let reverse_body = b"site-b-to-site-a".to_vec();
|
||||
put_object(&site_b.client(2)?, &bucket, reverse_key, reverse_body.clone()).await?;
|
||||
wait_for_replicated_bytes(&site_a.client(3)?, &bucket, reverse_key, &reverse_body, Duration::from_secs(60)).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! 4-node upgrade coverage for historical objects and IAM AK/SK.
|
||||
//!
|
||||
//! Complements `upgrade_compatibility_test` (single-node SSE/multipart and
|
||||
//! mixed-version listing). This module pins the distributed contract the
|
||||
//! hardware upgrade chain is meant to catch: after a 4-node upgrade, objects
|
||||
//! written on the previous release still read back, and IAM user credentials
|
||||
//! created before the upgrade still authenticate.
|
||||
//!
|
||||
//! Requires `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous
|
||||
//! release. The `e2e-distributed` workflow downloads that binary; a local run
|
||||
//! without it fails closed rather than skipping.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, enable_versioning, get_object_bytes, put_object,
|
||||
unique_bucket, wait_until,
|
||||
};
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const IAM_SECRET: &str = "UpgradeTestSecretKey1";
|
||||
const WRONG_SECRET: &str = "WrongSecretKey000000";
|
||||
const CREDENTIAL_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
struct UpgradeSeed {
|
||||
history_bucket: String,
|
||||
history_key: &'static str,
|
||||
history_body: Vec<u8>,
|
||||
versioned_bucket: String,
|
||||
versioned_key: &'static str,
|
||||
version1: String,
|
||||
version1_body: Vec<u8>,
|
||||
version2: String,
|
||||
version2_body: Vec<u8>,
|
||||
iam_bucket: String,
|
||||
iam_key: &'static str,
|
||||
iam_body: Vec<u8>,
|
||||
iam_user: String,
|
||||
iam_secret: &'static str,
|
||||
}
|
||||
|
||||
fn source_binary() -> TestResult<PathBuf> {
|
||||
let path = std::env::var_os(SOURCE_BINARY_ENV).map(PathBuf::from).ok_or_else(|| {
|
||||
format!(
|
||||
"{SOURCE_BINARY_ENV} must point to the pinned previous release binary (the e2e-distributed workflow downloads it)"
|
||||
)
|
||||
})?;
|
||||
if !path.is_file() {
|
||||
return Err(format!("upgrade source binary does not exist: {}", path.display()).into());
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn capture_upgrade_logs(cluster: &mut DistCluster, label: &str) -> TestResult {
|
||||
let Some(log_dir) = std::env::var_os("RUSTFS_E2E_LOG_DIR") else {
|
||||
return Ok(());
|
||||
};
|
||||
std::fs::create_dir_all(&log_dir)?;
|
||||
for node_idx in 0..cluster.cluster.nodes.len() {
|
||||
let path = Path::new(&log_dir).join(format!("{label}-node-{node_idx}.log"));
|
||||
cluster
|
||||
.cluster
|
||||
.set_node_capture_log_path(node_idx, path.to_string_lossy().into_owned())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn iam_rw_policy(bucket: &str) -> String {
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{bucket}"),
|
||||
format!("arn:aws:s3:::{bucket}/*")
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
async fn create_iam_user(dist: &DistCluster, user: &str, secret: &str, policy_name: &str, bucket: &str) -> TestResult {
|
||||
let url = &dist.cluster.nodes[0].url;
|
||||
let access = &dist.cluster.access_key;
|
||||
let admin_secret = &dist.cluster.secret_key;
|
||||
admin_create_user_via(AdminTransport::Signed, url, access, admin_secret, user, secret).await?;
|
||||
admin_add_canned_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, &iam_rw_policy(bucket)).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, url, access, admin_secret, policy_name, user).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_put(client: &Client, bucket: &str, key: &str, body: Vec<u8>, label: &str) -> TestResult {
|
||||
wait_until(
|
||||
CREDENTIAL_TIMEOUT,
|
||||
|| {
|
||||
let client = client.clone();
|
||||
let bucket = bucket.to_string();
|
||||
let key = key.to_string();
|
||||
let body = body.clone();
|
||||
async move {
|
||||
put_object(&client, &bucket, &key, body).await?;
|
||||
Ok(true)
|
||||
}
|
||||
},
|
||||
label,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn wait_for_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8], label: &str) -> TestResult {
|
||||
wait_until(
|
||||
CREDENTIAL_TIMEOUT,
|
||||
|| {
|
||||
let client = client.clone();
|
||||
let bucket = bucket.to_string();
|
||||
let key = key.to_string();
|
||||
let expected = expected.to_vec();
|
||||
async move {
|
||||
let got = get_object_bytes(&client, &bucket, &key).await?;
|
||||
Ok(got == expected)
|
||||
}
|
||||
},
|
||||
label,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn seed_history_and_iam(dist: &DistCluster) -> TestResult<UpgradeSeed> {
|
||||
let history_bucket = unique_bucket("upg-hist");
|
||||
let versioned_bucket = unique_bucket("upg-ver");
|
||||
let iam_bucket = unique_bucket("upg-iam");
|
||||
dist.create_bucket(&history_bucket).await?;
|
||||
dist.create_bucket(&versioned_bucket).await?;
|
||||
dist.create_bucket(&iam_bucket).await?;
|
||||
|
||||
let root = dist.client(0)?;
|
||||
enable_versioning(&root, &versioned_bucket).await?;
|
||||
|
||||
let history_key = "plain-history.bin";
|
||||
let history_body = b"written by the previous 4-node release".to_vec();
|
||||
put_object(&root, &history_bucket, history_key, history_body.clone()).await?;
|
||||
|
||||
let versioned_key = "versioned-history.txt";
|
||||
let version1_body = b"version-one-before-upgrade".to_vec();
|
||||
let version1 = root
|
||||
.put_object()
|
||||
.bucket(&versioned_bucket)
|
||||
.key(versioned_key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(version1_body.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("first versioned PUT omitted version ID")?
|
||||
.to_string();
|
||||
let version2_body = b"version-two-before-upgrade".to_vec();
|
||||
let version2 = root
|
||||
.put_object()
|
||||
.bucket(&versioned_bucket)
|
||||
.key(versioned_key)
|
||||
.body(aws_sdk_s3::primitives::ByteStream::from(version2_body.clone()))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("second versioned PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let iam_user = format!("upg{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let policy_name = format!("upgpol{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_iam_user(dist, &iam_user, IAM_SECRET, &policy_name, &iam_bucket).await?;
|
||||
|
||||
let iam_key = "iam-history.bin";
|
||||
let iam_body = b"written with pre-upgrade IAM AK/SK".to_vec();
|
||||
let iam_client = dist.client_with_credentials(1, &iam_user, IAM_SECRET)?;
|
||||
wait_for_put(&iam_client, &iam_bucket, iam_key, iam_body.clone(), "IAM user PUT before upgrade").await?;
|
||||
|
||||
Ok(UpgradeSeed {
|
||||
history_bucket,
|
||||
history_key,
|
||||
history_body,
|
||||
versioned_bucket,
|
||||
versioned_key,
|
||||
version1,
|
||||
version1_body,
|
||||
version2,
|
||||
version2_body,
|
||||
iam_bucket,
|
||||
iam_key,
|
||||
iam_body,
|
||||
iam_user,
|
||||
iam_secret: IAM_SECRET,
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context: &str) -> TestResult {
|
||||
let root_a = dist.client(0)?;
|
||||
let root_b = dist.client(3)?;
|
||||
wait_for_bytes(
|
||||
&root_b,
|
||||
&seed.history_bucket,
|
||||
seed.history_key,
|
||||
&seed.history_body,
|
||||
&format!("{context}: root GET historical object"),
|
||||
)
|
||||
.await?;
|
||||
assert_object_bytes(&root_a, &seed.history_bucket, seed.history_key, &seed.history_body).await?;
|
||||
|
||||
let v1 = root_b
|
||||
.get_object()
|
||||
.bucket(&seed.versioned_bucket)
|
||||
.key(seed.versioned_key)
|
||||
.version_id(&seed.version1)
|
||||
.send()
|
||||
.await?;
|
||||
let v1_body = v1.body.collect().await?.into_bytes();
|
||||
if v1_body.as_ref() != seed.version1_body.as_slice() {
|
||||
return Err(format!("{context}: version 1 bytes changed after upgrade").into());
|
||||
}
|
||||
let v2 = root_a
|
||||
.get_object()
|
||||
.bucket(&seed.versioned_bucket)
|
||||
.key(seed.versioned_key)
|
||||
.version_id(&seed.version2)
|
||||
.send()
|
||||
.await?;
|
||||
let v2_body = v2.body.collect().await?.into_bytes();
|
||||
if v2_body.as_ref() != seed.version2_body.as_slice() {
|
||||
return Err(format!("{context}: version 2 bytes changed after upgrade").into());
|
||||
}
|
||||
|
||||
let users = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/list-users", None).await?;
|
||||
if !users.contains(&seed.iam_user) {
|
||||
return Err(format!("{context}: list-users lost IAM user {}: {users}", seed.iam_user).into());
|
||||
}
|
||||
|
||||
let iam_on_upgraded = dist.client_with_credentials(0, &seed.iam_user, seed.iam_secret)?;
|
||||
let iam_on_peer = dist.client_with_credentials(3, &seed.iam_user, seed.iam_secret)?;
|
||||
wait_for_bytes(
|
||||
&iam_on_upgraded,
|
||||
&seed.iam_bucket,
|
||||
seed.iam_key,
|
||||
&seed.iam_body,
|
||||
&format!("{context}: IAM GET historical object on node 0"),
|
||||
)
|
||||
.await?;
|
||||
wait_for_bytes(
|
||||
&iam_on_peer,
|
||||
&seed.iam_bucket,
|
||||
seed.iam_key,
|
||||
&seed.iam_body,
|
||||
&format!("{context}: IAM GET historical object on node 3"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let post_key = format!("after-upgrade-{context}.txt");
|
||||
let post_body = format!("{context}: written with the same IAM AK/SK after upgrade").into_bytes();
|
||||
wait_for_put(
|
||||
&iam_on_peer,
|
||||
&seed.iam_bucket,
|
||||
&post_key,
|
||||
post_body.clone(),
|
||||
&format!("{context}: IAM PUT after upgrade"),
|
||||
)
|
||||
.await?;
|
||||
assert_object_bytes(&iam_on_upgraded, &seed.iam_bucket, &post_key, &post_body).await?;
|
||||
|
||||
let bad = dist.client_with_credentials(1, &seed.iam_user, WRONG_SECRET)?;
|
||||
match bad.get_object().bucket(&seed.iam_bucket).key(seed.iam_key).send().await {
|
||||
Ok(_) => return Err(format!("{context}: wrong secret must not read the IAM object").into()),
|
||||
Err(error) => {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
let rejected = code == Some("SignatureDoesNotMatch")
|
||||
|| code == Some("InvalidAccessKeyId")
|
||||
|| code == Some("AccessDenied")
|
||||
|| code == Some("InvalidArgument")
|
||||
|| error.raw_response().is_some_and(|response| response.status().as_u16() == 403);
|
||||
if !rejected {
|
||||
return Err(format!("{context}: wrong secret failed with unexpected error {error:?}").into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let post_root_key = format!("root-after-{context}.bin");
|
||||
let post_root_body = format!("{context}: root write after upgrade").into_bytes();
|
||||
put_object(&root_a, &seed.history_bucket, &post_root_key, post_root_body.clone()).await?;
|
||||
assert_object_bytes(&root_b, &seed.history_bucket, &post_root_key, &post_root_body).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_direct_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
capture_upgrade_logs(&mut dist, "direct-upgrade")?;
|
||||
dist.start_from_binary(&previous).await?;
|
||||
|
||||
let seed = seed_history_and_iam(&dist).await?;
|
||||
dist.restart_with_current_binary().await?;
|
||||
assert_history_and_iam(&dist, &seed, "direct").await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
|
||||
dist.start_from_binary(&previous).await?;
|
||||
|
||||
let seed = seed_history_and_iam(&dist).await?;
|
||||
|
||||
dist.replace_node_with_current_binary(0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
dist.replace_node_with_current_binary(node_idx).await?;
|
||||
}
|
||||
assert_history_and_iam(&dist, &seed, "one-previous-node").await?;
|
||||
|
||||
dist.replace_node_with_current_binary(3).await?;
|
||||
assert_history_and_iam(&dist, &seed, "homogeneous-current").await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, enable_versioning, get_object_bytes, put_object, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_four_drive_versioning_put_list_get_delete_marker() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("version");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
enable_versioning(&writer, &bucket).await?;
|
||||
|
||||
let key = "versioned.txt";
|
||||
let v1_id = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(b"v1".to_vec().into())
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("v1 PUT omitted version ID")?
|
||||
.to_string();
|
||||
let v2_id = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(b"v2".to_vec().into())
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("v2 PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
|
||||
let matching_versions: Vec<_> = versions
|
||||
.versions()
|
||||
.iter()
|
||||
.filter(|version| version.key() == Some(key))
|
||||
.collect();
|
||||
assert_eq!(matching_versions.len(), 2, "fresh key must have exactly two versions: {versions:?}");
|
||||
assert!(versions.delete_markers().is_empty(), "fresh key unexpectedly has a delete marker");
|
||||
assert!(
|
||||
matching_versions
|
||||
.iter()
|
||||
.any(|version| version.version_id() == Some(v1_id.as_str()) && version.is_latest() != Some(true)),
|
||||
"v1 was not the historical version: {versions:?}"
|
||||
);
|
||||
assert!(
|
||||
matching_versions
|
||||
.iter()
|
||||
.any(|version| version.version_id() == Some(v2_id.as_str()) && version.is_latest() == Some(true)),
|
||||
"v2 was not the latest version: {versions:?}"
|
||||
);
|
||||
|
||||
let latest = get_object_bytes(&reader, &bucket, key).await?;
|
||||
assert_eq!(latest, b"v2");
|
||||
|
||||
let older = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
|
||||
let older_body = older.body.collect().await?.into_bytes();
|
||||
assert_eq!(older_body.as_ref(), b"v1");
|
||||
|
||||
let deleted = writer.delete_object().bucket(&bucket).key(key).send().await?;
|
||||
assert_eq!(deleted.delete_marker(), Some(true));
|
||||
let marker_id = deleted.version_id().ok_or("DeleteObject omitted delete-marker version ID")?;
|
||||
let after_delete = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
|
||||
let matching_markers: Vec<_> = after_delete
|
||||
.delete_markers()
|
||||
.iter()
|
||||
.filter(|marker| marker.key() == Some(key))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
matching_markers.len(),
|
||||
1,
|
||||
"delete marker missing or duplicated after current-version delete: {after_delete:?}"
|
||||
);
|
||||
assert!(
|
||||
matching_markers[0].version_id() == Some(marker_id) && matching_markers[0].is_latest() == Some(true),
|
||||
"DeleteObject response and ListObjectVersions disagree about the marker: {after_delete:?}"
|
||||
);
|
||||
|
||||
let latest_after_delete = reader.get_object().bucket(&bucket).key(key).send().await;
|
||||
match latest_after_delete {
|
||||
Ok(_) => return Err("current version should be a delete marker".into()),
|
||||
Err(error)
|
||||
if error
|
||||
.as_service_error()
|
||||
.and_then(ProvideErrorMetadata::code)
|
||||
.is_some_and(|code| code == "NoSuchKey" || code == "NotFound") => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
|
||||
let restored = reader.get_object().bucket(&bucket).key(key).version_id(&v1_id).send().await?;
|
||||
let restored_body = restored.body.collect().await?.into_bytes();
|
||||
assert_eq!(restored_body.as_ref(), b"v1");
|
||||
|
||||
writer
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.version_id(marker_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"v2");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_versioning_suspension_keeps_one_null_version_and_history() -> TestResult {
|
||||
init_logging();
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("suspend");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let writer = dist.client(0)?;
|
||||
let reader = dist.client(3)?;
|
||||
enable_versioning(&writer, &bucket).await?;
|
||||
|
||||
let key = "suspended.txt";
|
||||
let original = writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(b"enabled-history".to_vec().into())
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("enabled PUT omitted version ID")?
|
||||
.to_string();
|
||||
writer
|
||||
.put_bucket_versioning()
|
||||
.bucket(&bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Suspended)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
put_object(&writer, &bucket, key, b"null-one".to_vec()).await?;
|
||||
put_object(&writer, &bucket, key, b"null-two".to_vec()).await?;
|
||||
assert_eq!(get_object_bytes(&reader, &bucket, key).await?, b"null-two");
|
||||
|
||||
let versions = reader.list_object_versions().bucket(&bucket).prefix(key).send().await?;
|
||||
let matching: Vec<_> = versions
|
||||
.versions()
|
||||
.iter()
|
||||
.filter(|version| version.key() == Some(key))
|
||||
.collect();
|
||||
assert!(matching.iter().any(|version| version.version_id() == Some(original.as_str())));
|
||||
let null_version_count = matching
|
||||
.iter()
|
||||
.filter(|version| {
|
||||
matches!(
|
||||
version.version_id(),
|
||||
None | Some("") | Some("null") | Some("00000000-0000-0000-0000-000000000000")
|
||||
)
|
||||
})
|
||||
.count();
|
||||
assert_eq!(null_version_count, 1, "suspended overwrites must keep one null version: {versions:?}");
|
||||
|
||||
let historical = reader
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.version_id(&original)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(historical.body.collect().await?.into_bytes().as_ref(), b"enabled-history");
|
||||
Ok(())
|
||||
}
|
||||
@@ -16,18 +16,15 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, sha256_hex, signed_admin_post};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{
|
||||
FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging,
|
||||
rustfs_binary_path,
|
||||
};
|
||||
use crate::storage_api::RUSTFS_META_BUCKET;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
@@ -37,76 +34,6 @@ mod tests {
|
||||
|
||||
const POOL_METADATA_OBJECT: &str = "pool.bin";
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EvidenceBuild {
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RestartEvidenceRun {
|
||||
schema: u32,
|
||||
run_id: String,
|
||||
source_revision: String,
|
||||
test_build: serde_json::Value,
|
||||
binary: EvidenceBuild,
|
||||
test_binary: EvidenceBuild,
|
||||
}
|
||||
|
||||
fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let mut file = std::fs::File::open(path)?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file.read(&mut buffer)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..read]);
|
||||
}
|
||||
Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
|
||||
}
|
||||
|
||||
fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
|
||||
let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
|
||||
return Ok(None);
|
||||
};
|
||||
let directory = PathBuf::from(directory);
|
||||
let receipt = directory.join("run.json");
|
||||
if receipt.metadata()?.len() > 1024 * 1024 {
|
||||
return Err("oversized scanner/heal execution receipt".into());
|
||||
}
|
||||
let run: RestartEvidenceRun = serde_json::from_slice(&std::fs::read(receipt)?)?;
|
||||
if run.schema != 1 || run.run_id.len() != 32 || run.source_revision.len() != 40 {
|
||||
return Err("invalid scanner/heal execution identity".into());
|
||||
}
|
||||
let built = compiled_test_identity();
|
||||
for key in ["source_revision", "dirty", "lock_blob", "features"] {
|
||||
assert_eq!(built[key], run.test_build[key], "compiled test identity differs for {key}");
|
||||
}
|
||||
assert_eq!(file_sha256(binary)?, run.binary.sha256, "server binary must match the run receipt");
|
||||
assert_eq!(
|
||||
file_sha256(&std::env::current_exe()?)?,
|
||||
run.test_binary.sha256,
|
||||
"test executable must match the run receipt"
|
||||
);
|
||||
if directory.join("background-target-restart.json").exists() {
|
||||
return Err("scanner/heal oracle already exists; create a new execution receipt".into());
|
||||
}
|
||||
Ok(Some((directory, run)))
|
||||
}
|
||||
|
||||
fn compiled_test_identity() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"source_revision": env!("RUSTFS_E2E_BUILD_COMMIT"),
|
||||
"dirty": env!("RUSTFS_E2E_BUILD_DIRTY") != "false",
|
||||
"lock_blob": env!("RUSTFS_E2E_BUILD_LOCK"),
|
||||
"features": env!("RUSTFS_E2E_BUILD_FEATURES"),
|
||||
"target": env!("RUSTFS_E2E_BUILD_TARGET"),
|
||||
"profile": env!("RUSTFS_E2E_BUILD_PROFILE"),
|
||||
"rustflags_hex": env!("RUSTFS_E2E_BUILD_RUSTFLAGS_HEX"),
|
||||
})
|
||||
}
|
||||
|
||||
struct TcpPortBlackhole {
|
||||
port: u16,
|
||||
comment: String,
|
||||
@@ -268,9 +195,8 @@ mod tests {
|
||||
clients: &[aws_sdk_s3::Client],
|
||||
bucket: &str,
|
||||
expected_keys: &HashSet<String>,
|
||||
) -> Result<Vec<Vec<String>>, Box<dyn Error + Send + Sync>> {
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
const PAGE_SIZE: i32 = 10;
|
||||
let mut node_listings = Vec::with_capacity(clients.len());
|
||||
for (node_index, client) in clients.iter().enumerate() {
|
||||
let mut listed_keys = Vec::new();
|
||||
let mut continuation_token = None;
|
||||
@@ -317,10 +243,8 @@ mod tests {
|
||||
&listed_key_set, expected_keys,
|
||||
"node {node_index} did not expose the complete recovered namespace"
|
||||
);
|
||||
listed_keys.sort();
|
||||
node_listings.push(listed_keys);
|
||||
}
|
||||
Ok(node_listings)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn heal_task_status_diagnostic(body: &str) -> String {
|
||||
@@ -884,13 +808,6 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
|
||||
restart_evidence_run(&server_binary)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut evidence_objects = Vec::new();
|
||||
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
||||
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
||||
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
|
||||
@@ -938,7 +855,7 @@ mod tests {
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
}
|
||||
cluster.start_with_binary(&server_binary).await?;
|
||||
cluster.start().await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
@@ -1079,7 +996,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
cluster.start_node_from_binary(1, &server_binary).await?;
|
||||
cluster.start_node(1).await?;
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(60);
|
||||
@@ -1357,7 +1274,7 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
cluster.start_node(interruption_node).await?;
|
||||
if interruption_node == 0 {
|
||||
let target = cluster.nodes[1]
|
||||
.process
|
||||
@@ -1456,7 +1373,7 @@ mod tests {
|
||||
.map(|manifest| manifest.key.clone())
|
||||
.collect::<HashSet<_>>();
|
||||
assert!(expected_keys.insert(outage_key.to_string()));
|
||||
let node_listings = assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
assert_all_nodes_list_exact_keys(&clients, bucket, &expected_keys).await?;
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
for expected in &expected_manifests {
|
||||
@@ -1464,31 +1381,11 @@ mod tests {
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": expected.key, "version_id": expected.shard_census.version_id,
|
||||
"expected_bytes": expected_body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(&expected_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": expected.shard_census,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?,
|
||||
}));
|
||||
}
|
||||
}
|
||||
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
|
||||
if evidence_run.is_some() {
|
||||
evidence_objects.push(serde_json::json!({
|
||||
"key": outage_key, "version_id": null,
|
||||
"expected_bytes": expected_outage_body.len(), "actual_bytes": actual.len(),
|
||||
"expected_sha256": sha256_hex(&expected_outage_body),
|
||||
"actual_sha256": sha256_hex(&actual),
|
||||
"expected_physical": null,
|
||||
"physical": census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?,
|
||||
}));
|
||||
}
|
||||
|
||||
let terminal_deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
@@ -1535,31 +1432,6 @@ mod tests {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
|
||||
if let Some((directory, run)) = evidence_run {
|
||||
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
|
||||
assert_ne!(target_pid, restarted_pid, "target must be a new process");
|
||||
assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": "background-target-restart", "evidence": "process-restart",
|
||||
"run_id": run.run_id, "source_revision": run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
|
||||
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
if data.len() > 1024 * 1024 {
|
||||
return Err("scanner/heal oracle exceeds the 1 MiB artifact budget".into());
|
||||
}
|
||||
let mut output = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(directory.join("background-target-restart.json"))?;
|
||||
output.write_all(&data)?;
|
||||
output.sync_all()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -378,11 +378,6 @@ mod bucket_stats_regression_test;
|
||||
#[cfg(test)]
|
||||
mod distributed_startup_regression_test;
|
||||
|
||||
// 4-node / 4-disk distributed Actions suite (S3, lock, versioning, replication,
|
||||
// quota, observability, expand/decommission/rebalance, site replication, chaos).
|
||||
#[cfg(test)]
|
||||
mod distributed;
|
||||
|
||||
// P1 regression: tier/ILM transition (rustfs#5218, #5130, #5011, #4826, #5024)
|
||||
#[cfg(test)]
|
||||
mod tier_transition_regression_test;
|
||||
|
||||
@@ -20,10 +20,9 @@
|
||||
//! journal (`count_requests`) carries the assertion in every one of them.
|
||||
|
||||
use super::common::{BoxError, OdmTestEnv, RawResponse, SeedObject, start_configured_env};
|
||||
use crate::fake_s3_target::{FaultAction, Operation};
|
||||
use crate::fake_s3_target::Operation;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use bytes::Bytes;
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use std::time::Duration;
|
||||
|
||||
type TestResult = Result<(), BoxError>;
|
||||
@@ -146,38 +145,14 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
|
||||
.await?;
|
||||
|
||||
let body = payload(128 * 1024);
|
||||
let blocker = "queue/blocker.bin";
|
||||
env.seed_source(SOURCE_BUCKET, &[SeedObject::new(blocker, body.clone())]);
|
||||
// The one-chunk range completes immediately; its full background pull
|
||||
// occupies the only slot while the remaining requests fill the queue.
|
||||
env.source.inject_for_key(
|
||||
Operation::GetObject,
|
||||
blocker,
|
||||
FaultAction::SlowSendBody {
|
||||
chunk_bytes: 1024,
|
||||
delay: Duration::from_millis(100),
|
||||
},
|
||||
2,
|
||||
);
|
||||
let response = env
|
||||
.raw_object_request(http::Method::GET, bucket, blocker, &[("range", "bytes=0-1023")])
|
||||
.await?;
|
||||
assert_eq!(response.status, 206);
|
||||
assert_eq!(response.body, body.slice(0..1024));
|
||||
env.wait_for_status_counter(bucket, "/inflight_pulls", 1, SETTLE).await?;
|
||||
|
||||
let keys: Vec<String> = (0..REQUESTS).map(|index| format!("queue/object-{index:03}.bin")).collect();
|
||||
let seeds: Vec<SeedObject> = keys.iter().map(|key| SeedObject::new(key.clone(), body.clone())).collect();
|
||||
env.seed_source(SOURCE_BUCKET, &seeds);
|
||||
|
||||
// Bound source connections below the fixture's limit while still
|
||||
// submitting all 100 requests to the eight-slot background queue.
|
||||
let responses: Vec<RawResponse> = futures::stream::iter(
|
||||
let responses: Vec<RawResponse> = futures::future::try_join_all(
|
||||
keys.iter()
|
||||
.map(|key| env.raw_object_request(http::Method::GET, bucket, key, &[("range", "bytes=0-1023")])),
|
||||
)
|
||||
.buffered(16)
|
||||
.try_collect()
|
||||
.await?;
|
||||
for (key, response) in keys.iter().zip(&responses) {
|
||||
assert_eq!(response.status, 206, "{key}: {}", String::from_utf8_lossy(&response.body));
|
||||
@@ -193,15 +168,6 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
|
||||
.wait_for_status_counter(bucket, "/counters/pull_failures_total/queue_full", 1, SETTLE)
|
||||
.await?;
|
||||
assert!(queue_full > 0, "a 100-deep burst must overflow an 8-slot queue");
|
||||
let queue_full = usize::try_from(queue_full)?;
|
||||
assert!(queue_full <= REQUESTS);
|
||||
env.wait_for_status_counter(
|
||||
bucket,
|
||||
"/counters/pulled_objects_total/background",
|
||||
u64::try_from(REQUESTS + 1 - queue_full)?,
|
||||
SETTLE,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let ranged_reads: usize = keys.iter().map(|key| source_get_count(&env, key)).sum();
|
||||
assert!(
|
||||
@@ -209,6 +175,9 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
|
||||
"every reader is served from the source: {ranged_reads} GETs for {REQUESTS} readers"
|
||||
);
|
||||
let dropped = keys.iter().filter(|key| source_get_count(&env, key) == 1).count();
|
||||
assert_eq!(dropped, queue_full, "only overflowed keys remain without a background GET");
|
||||
assert!(
|
||||
dropped > 0,
|
||||
"the overflowed keys are the ones with no backfill GET, but every key got one"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -265,13 +265,16 @@ async fn list_through_rejects_a_tampered_continuation_token() -> TestResult {
|
||||
let decoded = String::from_utf8(base64_simd::STANDARD.decode_to_vec(token.as_bytes())?)?;
|
||||
assert!(decoded.contains("\"t\":\"odm-list\""), "the merged token is an envelope: {decoded}");
|
||||
|
||||
let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":3").as_bytes());
|
||||
assert_ne!(tampered, token, "the test must change the token version");
|
||||
let query = serde_urlencoded::to_string([("continuation-token", tampered.as_str())])?;
|
||||
let rejected = env.raw_list_objects_v2(bucket, &query).await?;
|
||||
let error_body = String::from_utf8_lossy(&rejected.body);
|
||||
assert_eq!(rejected.status, 400, "a bumped token version is a client error: {}", error_body);
|
||||
assert!(error_body.contains("<Code>InvalidArgument</Code>"), "{error_body}");
|
||||
let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":2").as_bytes());
|
||||
let rejected = env
|
||||
.raw_list_objects_v2(bucket, &format!("continuation-token={tampered}"))
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rejected.status,
|
||||
400,
|
||||
"a bumped token version is a client error: {}",
|
||||
String::from_utf8_lossy(&rejected.body)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -12,34 +12,21 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{
|
||||
RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
|
||||
};
|
||||
use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
|
||||
use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject};
|
||||
use crate::replication_extension_test::{
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
|
||||
};
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketLifecycleConfiguration, BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DefaultRetention,
|
||||
ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, ObjectLockConfiguration, ObjectLockEnabled,
|
||||
ObjectLockRetentionMode, ObjectLockRule, PublicAccessBlockConfiguration, ServerSideEncryption, ServerSideEncryptionByDefault,
|
||||
ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Tag, Tagging, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration,
|
||||
};
|
||||
use http::{Method, StatusCode};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::{Instant, sleep};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
|
||||
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
|
||||
const PLAIN_BUCKET: &str = "upgrade-plain-data";
|
||||
@@ -53,32 +40,6 @@ const MULTIPART_UPLOADS_PER_WORKER: usize = 16;
|
||||
// comfortably covers that window plus CI scheduling jitter.
|
||||
const LISTING_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
// Bucket-configuration upgrade/rollback scenarios (rustfs#7172, #7183, #7089).
|
||||
const CONFIG_PLAIN_BUCKET: &str = "upgrade-config-plain";
|
||||
const CONFIG_ENCRYPTED_BUCKET: &str = "upgrade-config-encrypted";
|
||||
const CONFIG_REPLICATED_BUCKET: &str = "upgrade-config-replicated";
|
||||
const CONFIG_LOCKED_BUCKET: &str = "upgrade-config-locked";
|
||||
const CONFIG_REPLICA_BUCKET: &str = "upgrade-config-replica";
|
||||
const ROLLBACK_BUCKET: &str = "rollback-config-data";
|
||||
const ROLLBACK_REPLICA_BUCKET: &str = "rollback-config-replica";
|
||||
const BUCKET_QUOTA_BYTES: u64 = 64 * 1024 * 1024;
|
||||
const LIFECYCLE_RULE_ID: &str = "upgrade-expire-logs";
|
||||
const LIFECYCLE_PREFIX: &str = "logs/";
|
||||
const LIFECYCLE_DAYS: i32 = 30;
|
||||
const BUCKET_TAG_KEY: &str = "owner";
|
||||
const BUCKET_TAG_VALUE: &str = "upgrade-compatibility";
|
||||
const OBJECT_LOCK_DAYS: i32 = 1;
|
||||
// `set-bucket-quota` answers 503 until the scanner has made the bucket's usage
|
||||
// authoritative; the quota test uses the same 30s budget.
|
||||
const QUOTA_READINESS_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
// Quota admission fails closed while a freshly started server has neither
|
||||
// authoritative usage nor a persisted degraded baseline for the bucket
|
||||
// (rustfs#5716), so a write to a quota-enabled bucket is retryable-503 for that
|
||||
// window. It is a restart property, not an upgrade property — the same window
|
||||
// opens on the very first start — so the write assertions ride it out instead
|
||||
// of treating it as an upgrade failure.
|
||||
const QUOTA_ADMISSION_WARMUP_TIMEOUT: Duration = Duration::from_secs(90);
|
||||
|
||||
fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let path = std::env::var_os(SOURCE_BINARY_ENV)
|
||||
.map(PathBuf::from)
|
||||
@@ -279,93 +240,6 @@ async fn exercise_mixed_cluster(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Pins the published old writer's limitation and the supported recovery
|
||||
/// procedure. This is not a promise that mixed-version ODM is supported.
|
||||
/// Replace the loss assertion when ODM gains independent persistence;
|
||||
/// preserving configuration across rc.5 writes is then an improvement.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
|
||||
async fn rc5_rollback_requires_restoring_odm_configuration() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let version = tokio::process::Command::new(&previous_binary)
|
||||
.arg("--version")
|
||||
.output()
|
||||
.await?;
|
||||
assert!(version.status.success(), "previous binary must report its version");
|
||||
assert!(
|
||||
String::from_utf8(version.stdout)?.contains(RC5_COMMIT),
|
||||
"this compatibility scenario requires the published rc.5 writer"
|
||||
);
|
||||
let mut env = OdmTestEnv::start().await?;
|
||||
let bucket = "odm-rc5-rollback";
|
||||
let source_bucket = "odm-rc5-source";
|
||||
env.source.create_bucket_with_mode(source_bucket, BucketMode::Unversioned);
|
||||
env.seed_source(
|
||||
source_bucket,
|
||||
&[SeedObject::new(
|
||||
"source-only",
|
||||
bytes::Bytes::from_static(b"source read after recovery"),
|
||||
)],
|
||||
);
|
||||
env.rustfs.create_test_bucket(bucket).await?;
|
||||
let saved_config = env.fake_source_spec(source_bucket);
|
||||
assert_eq!(env.configure_source(bucket, &saved_config).await?.status, 200);
|
||||
let before = env.get_config(bucket).await?;
|
||||
assert_eq!(before.status, 200);
|
||||
let expected_config = before
|
||||
.json()?
|
||||
.get("config")
|
||||
.cloned()
|
||||
.ok_or("configuration response omitted config")?;
|
||||
env.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("local")
|
||||
.body(ByteStream::from_static(b"local data survives rollback"))
|
||||
.send()
|
||||
.await?;
|
||||
env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
|
||||
let restarted = env.get_config(bucket).await?;
|
||||
assert_eq!(restarted.status, 200, "a current writer preserves ODM across restart");
|
||||
assert_eq!(restarted.json()?.get("config"), Some(&expected_config));
|
||||
|
||||
restart_from_binary(&mut env.rustfs, &previous_binary, &[]).await?;
|
||||
env.client
|
||||
.put_bucket_tagging()
|
||||
.bucket(bucket)
|
||||
.tagging(
|
||||
Tagging::builder()
|
||||
.tag_set(Tag::builder().key("writer").value("rc5").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
|
||||
let missing = env.get_config(bucket).await?;
|
||||
assert_eq!(missing.status, 404, "rc.5 rewrites metadata without ODM keys");
|
||||
assert!(missing.body.contains("NoSuchConfiguration"));
|
||||
assert_eq!(read_object(&env.client, bucket, "local", None).await?.1, b"local data survives rollback");
|
||||
let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
|
||||
assert!(tags.tag_set().iter().any(|tag| tag.key() == "writer" && tag.value() == "rc5"));
|
||||
|
||||
assert_eq!(
|
||||
env.configure_source(bucket, &saved_config).await?.status,
|
||||
200,
|
||||
"restore from saved full configuration"
|
||||
);
|
||||
env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
|
||||
let restored = env.get_config(bucket).await?;
|
||||
assert_eq!(restored.status, 200, "restored ODM configuration persists");
|
||||
assert_eq!(restored.json()?.get("config"), Some(&expected_config));
|
||||
env.wait_until_source_consulted(bucket).await?;
|
||||
assert_eq!(
|
||||
read_object(&env.client, bucket, "source-only", None).await?.1,
|
||||
b"source read after recovery"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
|
||||
@@ -555,653 +429,3 @@ async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestRes
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Child-process environment shared by both bucket-configuration scenarios.
|
||||
///
|
||||
/// The replication target is an in-process fake bound to `127.0.0.1`, which
|
||||
/// `set-remote-target` rejects as an SSRF risk without the loopback opt-in, and
|
||||
/// the proxy bypass keeps a developer's `HTTP_PROXY` from intercepting the
|
||||
/// server's outbound health check.
|
||||
fn bucket_config_server_env() -> Vec<(&'static str, &'static str)> {
|
||||
let mut env = vec![
|
||||
(SSE_MASTER_KEY_ENV, SSE_MASTER_KEY),
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
// Shorten the scanner cycle so the bucket's usage becomes authoritative
|
||||
// in seconds; both `set-bucket-quota` and quota admission block on it.
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
|
||||
];
|
||||
env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env.extend(replication_fast_env());
|
||||
env
|
||||
}
|
||||
|
||||
/// Restart `env` in place on the same data directory using an explicit binary.
|
||||
///
|
||||
/// [`RustFSTestEnvironment::restart_server_preserving_data`] always relaunches
|
||||
/// the workspace build, which is the upgrade direction only. The rollback
|
||||
/// scenario needs the reverse: stop the current build and bring the pinned
|
||||
/// previous release up on the metadata that build just wrote.
|
||||
async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, server_env: &[(&str, &str)]) -> TestResult {
|
||||
env.stop_server();
|
||||
env.start_rustfs_server_from_binary(binary, vec![], server_env).await
|
||||
}
|
||||
|
||||
async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
|
||||
let path = format!("/rustfs/admin/v3/quota/{bucket}");
|
||||
let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
|
||||
let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
|
||||
loop {
|
||||
let (status, response) =
|
||||
admin_request(&env.url, Method::PUT, &path, Some(body.clone()), &env.access_key, &env.secret_key).await?;
|
||||
if status.is_success() {
|
||||
return Ok(());
|
||||
}
|
||||
if status != StatusCode::SERVICE_UNAVAILABLE || Instant::now() >= deadline {
|
||||
return Err(format!("setting the quota of {bucket} failed: {status} {response}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// PUT into a quota-enabled bucket, riding out the post-start quota-admission
|
||||
/// warm-up described on [`QUOTA_ADMISSION_WARMUP_TIMEOUT`].
|
||||
///
|
||||
/// Only `ServiceUnavailable` is retried: any other failure, and a warm-up that
|
||||
/// never ends, is a genuine regression and surfaces as an error.
|
||||
async fn put_object_through_quota_warmup(client: &Client, bucket: &str, key: &str, body: &'static [u8]) -> TestResult {
|
||||
let deadline = Instant::now() + QUOTA_ADMISSION_WARMUP_TIMEOUT;
|
||||
loop {
|
||||
let result = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.send()
|
||||
.await;
|
||||
let error = match result {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) => error,
|
||||
};
|
||||
let retryable = error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("ServiceUnavailable");
|
||||
if !retryable || Instant::now() >= deadline {
|
||||
return Err(format!("PUT {bucket}/{key} failed after the quota warm-up window: {error}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_bucket_quota(env: &RustFSTestEnvironment, bucket: &str) -> Result<Option<u64>, BoxError> {
|
||||
let path = format!("/rustfs/admin/v3/quota/{bucket}");
|
||||
let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
|
||||
if status != StatusCode::OK {
|
||||
return Err(format!("reading the quota of {bucket} failed: {status} {response}").into());
|
||||
}
|
||||
let quota: serde_json::Value = serde_json::from_str(&response)?;
|
||||
Ok(quota.get("quota").and_then(serde_json::Value::as_u64))
|
||||
}
|
||||
|
||||
/// `GET /rustfs/admin/v3/list-remote-targets?bucket=...`.
|
||||
///
|
||||
/// Returns an error for any non-200, because rustfs#7172 made this endpoint
|
||||
/// fail closed on a `bucket-targets.json` blob the running build cannot parse.
|
||||
/// An upgrade that misreads a blob written by the previous release therefore
|
||||
/// shows up here as an error, and a silently dropped target shows up as an
|
||||
/// empty list — the caller must distinguish the two.
|
||||
async fn list_remote_targets(env: &RustFSTestEnvironment, bucket: &str) -> Result<Vec<serde_json::Value>, BoxError> {
|
||||
let path = format!("/rustfs/admin/v3/list-remote-targets?bucket={}", urlencoding::encode(bucket));
|
||||
let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
|
||||
if status != StatusCode::OK {
|
||||
return Err(format!("list-remote-targets for {bucket} failed: {status} {response}").into());
|
||||
}
|
||||
Ok(serde_json::from_str(&response)?)
|
||||
}
|
||||
|
||||
/// Assert that `bucket` still carries exactly the replication target `arn`.
|
||||
async fn assert_remote_target_preserved(env: &RustFSTestEnvironment, bucket: &str, arn: &str, context: &str) -> TestResult {
|
||||
let targets = list_remote_targets(env, bucket).await?;
|
||||
assert_eq!(
|
||||
targets.len(),
|
||||
1,
|
||||
"{context}: list-remote-targets must still report the single configured target, got {targets:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
targets[0].get("arn").and_then(serde_json::Value::as_str),
|
||||
Some(arn),
|
||||
"{context}: the target ARN changed across the restart: {targets:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Configure a replication target on `bucket` pointing at the in-process fake,
|
||||
/// then attach an enabled replication rule for it. Returns the target ARN.
|
||||
async fn configure_replication(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
target: &FakeS3Target,
|
||||
target_bucket: &str,
|
||||
) -> Result<String, BoxError> {
|
||||
let arn = set_replication_target_with_options(
|
||||
env,
|
||||
bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(env, bucket, &arn).await?;
|
||||
Ok(arn)
|
||||
}
|
||||
|
||||
async fn put_default_sse_s3_encryption(client: &Client, bucket: &str) -> TestResult {
|
||||
let configuration = ServerSideEncryptionConfiguration::builder()
|
||||
.rules(
|
||||
ServerSideEncryptionRule::builder()
|
||||
.apply_server_side_encryption_by_default(
|
||||
ServerSideEncryptionByDefault::builder()
|
||||
.sse_algorithm(ServerSideEncryption::Aes256)
|
||||
.build()?,
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build()?;
|
||||
client
|
||||
.put_bucket_encryption()
|
||||
.bucket(bucket)
|
||||
.server_side_encryption_configuration(configuration)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_default_sse_s3_encryption(client: &Client, bucket: &str, context: &str) -> TestResult {
|
||||
let response = client.get_bucket_encryption().bucket(bucket).send().await?;
|
||||
let rules = response
|
||||
.server_side_encryption_configuration()
|
||||
.ok_or("GetBucketEncryption omitted the configuration")?
|
||||
.rules();
|
||||
assert_eq!(rules.len(), 1, "{context}: expected exactly one encryption rule, got {rules:?}");
|
||||
assert_eq!(
|
||||
rules[0]
|
||||
.apply_server_side_encryption_by_default()
|
||||
.map(ServerSideEncryptionByDefault::sse_algorithm),
|
||||
Some(&ServerSideEncryption::Aes256),
|
||||
"{context}: the default encryption algorithm changed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_bucket_tag(client: &Client, bucket: &str) -> TestResult {
|
||||
let tagging = Tagging::builder()
|
||||
.tag_set(Tag::builder().key(BUCKET_TAG_KEY).value(BUCKET_TAG_VALUE).build()?)
|
||||
.build()?;
|
||||
client.put_bucket_tagging().bucket(bucket).tagging(tagging).send().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_bucket_tag(client: &Client, bucket: &str, context: &str) -> TestResult {
|
||||
let tags = client.get_bucket_tagging().bucket(bucket).send().await?;
|
||||
let tag_set = tags.tag_set();
|
||||
assert_eq!(tag_set.len(), 1, "{context}: expected exactly one bucket tag, got {tag_set:?}");
|
||||
assert_eq!(tag_set[0].key(), BUCKET_TAG_KEY, "{context}: bucket tag key changed");
|
||||
assert_eq!(tag_set[0].value(), BUCKET_TAG_VALUE, "{context}: bucket tag value changed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_versioning_enabled(client: &Client, bucket: &str, context: &str) -> TestResult {
|
||||
let versioning = client.get_bucket_versioning().bucket(bucket).send().await?;
|
||||
assert_eq!(
|
||||
versioning.status(),
|
||||
Some(&BucketVersioningStatus::Enabled),
|
||||
"{context}: versioning is no longer Enabled on {bucket}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn bucket_policy_document(bucket: &str) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Sid": "UpgradePublicRead",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": ["*"] },
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/public/*")]
|
||||
}]
|
||||
})
|
||||
}
|
||||
|
||||
/// `GET .../on-demand-migration/{bucket}/status`.
|
||||
///
|
||||
/// The migration module defaults on from rustfs#7089, so a bucket that never
|
||||
/// configured a source must still answer `configured: false` rather than
|
||||
/// engaging the migration path.
|
||||
async fn assert_migration_not_configured(env: &RustFSTestEnvironment, bucket: &str) -> TestResult {
|
||||
let path = format!("/rustfs/admin/v3/on-demand-migration/{bucket}/status");
|
||||
let (status, response) = admin_request(&env.url, Method::GET, &path, None, &env.access_key, &env.secret_key).await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
StatusCode::OK,
|
||||
"the migration status endpoint must answer for an unconfigured bucket: {status} {response}"
|
||||
);
|
||||
let body: serde_json::Value = serde_json::from_str(&response)?;
|
||||
assert_eq!(
|
||||
body.get("configured"),
|
||||
Some(&serde_json::Value::Bool(false)),
|
||||
"a bucket upgraded from the previous release must not look migration-configured: {body}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A GET for a key that was never written must be a plain `NoSuchKey`.
|
||||
///
|
||||
/// With the migration module on by default this is the cheap proof that an
|
||||
/// unconfigured bucket never consults a source: any migration engagement would
|
||||
/// surface as a different status or error code here.
|
||||
async fn assert_missing_key_is_no_such_key(client: &Client, bucket: &str, key: &str) -> TestResult {
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a key that was never written must not be readable");
|
||||
assert_eq!(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"a missing key must stay a 404 on a bucket with no migration configuration"
|
||||
);
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"a missing key must stay NoSuchKey on a bucket with no migration configuration"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bucket configuration written by the pinned previous release must survive an
|
||||
/// upgrade to the current build unchanged, and must keep working.
|
||||
///
|
||||
/// This pins the three on-disk surfaces the on-demand-migration series moved:
|
||||
///
|
||||
/// * `BucketMetadata` grew two msgpack keys (encoded map length 44 -> 46), so
|
||||
/// every configuration read below decodes a 44-key blob on 46-key code.
|
||||
/// * rustfs#7172 made an unreadable `bucket-targets.json` / encryption /
|
||||
/// public-access-block / quota blob "present but unreadable" instead of
|
||||
/// silently defaulting, and made `list-remote-targets` fail closed on it. A
|
||||
/// replication target configured by the old release must therefore still be
|
||||
/// *listed*, not dropped and not an error.
|
||||
/// * rustfs#7183 made the object write path refuse a PUT when the bucket's
|
||||
/// encryption configuration cannot be read, so a misparsed SSE config would
|
||||
/// turn every PUT to that bucket into a 500.
|
||||
///
|
||||
/// Not covered on purpose: on-demand-migration configuration itself, which the
|
||||
/// previous release has no public API for — the reverse direction is asserted
|
||||
/// instead (an upgraded bucket reports `configured: false`).
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
|
||||
// In-process: the fake target outlives both server processes, so the
|
||||
// replication target stays reachable across the upgrade.
|
||||
let replication_target = FakeS3Target::start().await?;
|
||||
replication_target.create_bucket(CONFIG_REPLICA_BUCKET);
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let server_env = bucket_config_server_env();
|
||||
env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env)
|
||||
.await?;
|
||||
let old_client = env.create_s3_client();
|
||||
|
||||
env.create_test_bucket(CONFIG_PLAIN_BUCKET).await?;
|
||||
env.create_test_bucket(CONFIG_ENCRYPTED_BUCKET).await?;
|
||||
env.create_test_bucket(CONFIG_REPLICATED_BUCKET).await?;
|
||||
old_client
|
||||
.create_bucket()
|
||||
.bucket(CONFIG_LOCKED_BUCKET)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Plain bucket: policy, tags, lifecycle, quota.
|
||||
let policy = bucket_policy_document(CONFIG_PLAIN_BUCKET);
|
||||
old_client
|
||||
.put_bucket_policy()
|
||||
.bucket(CONFIG_PLAIN_BUCKET)
|
||||
.policy(policy.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
put_bucket_tag(&old_client, CONFIG_PLAIN_BUCKET).await?;
|
||||
old_client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(CONFIG_PLAIN_BUCKET)
|
||||
.lifecycle_configuration(
|
||||
BucketLifecycleConfiguration::builder()
|
||||
.rules(
|
||||
LifecycleRule::builder()
|
||||
.id(LIFECYCLE_RULE_ID)
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.filter(LifecycleRuleFilter::builder().prefix(LIFECYCLE_PREFIX).build())
|
||||
.expiration(LifecycleExpiration::builder().days(LIFECYCLE_DAYS).build())
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
set_bucket_quota(&env, CONFIG_PLAIN_BUCKET, BUCKET_QUOTA_BYTES).await?;
|
||||
|
||||
// Encrypted bucket: SSE-S3 default encryption plus a fully restrictive
|
||||
// public access block, both of which rustfs#7172 now fails closed on.
|
||||
put_default_sse_s3_encryption(&old_client, CONFIG_ENCRYPTED_BUCKET).await?;
|
||||
old_client
|
||||
.put_public_access_block()
|
||||
.bucket(CONFIG_ENCRYPTED_BUCKET)
|
||||
.public_access_block_configuration(
|
||||
PublicAccessBlockConfiguration::builder()
|
||||
.block_public_acls(true)
|
||||
.ignore_public_acls(true)
|
||||
.block_public_policy(true)
|
||||
.restrict_public_buckets(true)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Replicated bucket: versioning, a validated remote target, a rule.
|
||||
enable_versioning(&old_client, CONFIG_REPLICATED_BUCKET).await?;
|
||||
let target_arn = configure_replication(&env, CONFIG_REPLICATED_BUCKET, &replication_target, CONFIG_REPLICA_BUCKET).await?;
|
||||
assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "before the upgrade").await?;
|
||||
|
||||
// Object-lock bucket: a default GOVERNANCE retention on a fresh bucket.
|
||||
old_client
|
||||
.put_object_lock_configuration()
|
||||
.bucket(CONFIG_LOCKED_BUCKET)
|
||||
.object_lock_configuration(
|
||||
ObjectLockConfiguration::builder()
|
||||
.object_lock_enabled(ObjectLockEnabled::Enabled)
|
||||
.rule(
|
||||
ObjectLockRule::builder()
|
||||
.default_retention(
|
||||
DefaultRetention::builder()
|
||||
.mode(ObjectLockRetentionMode::Governance)
|
||||
.days(OBJECT_LOCK_DAYS)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let plain_key = "plain/written-by-previous";
|
||||
let plain_bytes = b"plain object written by the previous RustFS release";
|
||||
put_object_through_quota_warmup(&old_client, CONFIG_PLAIN_BUCKET, plain_key, plain_bytes).await?;
|
||||
|
||||
let encrypted_key = "encrypted/written-by-previous";
|
||||
let encrypted_bytes = b"default-encrypted object written by the previous RustFS release";
|
||||
old_client
|
||||
.put_object()
|
||||
.bucket(CONFIG_ENCRYPTED_BUCKET)
|
||||
.key(encrypted_key)
|
||||
.body(ByteStream::from_static(encrypted_bytes))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
read_object(&old_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None)
|
||||
.await?
|
||||
.0,
|
||||
Some(ServerSideEncryption::Aes256),
|
||||
"the previous release must apply the bucket default encryption it just accepted"
|
||||
);
|
||||
|
||||
// The multipart object lives in the default-encrypted bucket so the
|
||||
// upgraded build has to reassemble parts *and* re-derive the object key.
|
||||
let multipart_key = "encrypted/multipart-written-by-previous";
|
||||
let multipart_parts = vec![vec![b'm'; 5 * 1024 * 1024], b"final multipart bytes".to_vec()];
|
||||
let multipart_bytes = multipart_parts.concat();
|
||||
write_multipart(&old_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, &multipart_parts).await?;
|
||||
|
||||
let versioned_key = "versioned/written-by-previous";
|
||||
let versioned_bytes = b"versioned object written by the previous RustFS release";
|
||||
let versioned_id = old_client
|
||||
.put_object()
|
||||
.bucket(CONFIG_REPLICATED_BUCKET)
|
||||
.key(versioned_key)
|
||||
.body(ByteStream::from_static(versioned_bytes))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("versioned PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
env.restart_server_preserving_data(vec![], &server_env).await?;
|
||||
let new_client = env.create_s3_client();
|
||||
|
||||
// Every configuration must read back unchanged on the upgraded build.
|
||||
let upgraded_policy = new_client.get_bucket_policy().bucket(CONFIG_PLAIN_BUCKET).send().await?;
|
||||
let upgraded_policy: serde_json::Value =
|
||||
serde_json::from_str(upgraded_policy.policy().ok_or("GetBucketPolicy omitted the document")?)?;
|
||||
assert_eq!(upgraded_policy, policy, "the bucket policy changed across the upgrade");
|
||||
assert_bucket_tag(&new_client, CONFIG_PLAIN_BUCKET, "after the upgrade").await?;
|
||||
|
||||
let lifecycle = new_client
|
||||
.get_bucket_lifecycle_configuration()
|
||||
.bucket(CONFIG_PLAIN_BUCKET)
|
||||
.send()
|
||||
.await?;
|
||||
let rules = lifecycle.rules();
|
||||
assert_eq!(rules.len(), 1, "the lifecycle rule count changed across the upgrade: {rules:?}");
|
||||
assert_eq!(rules[0].id(), Some(LIFECYCLE_RULE_ID));
|
||||
assert_eq!(rules[0].status(), &ExpirationStatus::Enabled);
|
||||
assert_eq!(
|
||||
rules[0].expiration().and_then(LifecycleExpiration::days),
|
||||
Some(LIFECYCLE_DAYS),
|
||||
"the lifecycle expiration changed across the upgrade"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_bucket_quota(&env, CONFIG_PLAIN_BUCKET).await?,
|
||||
Some(BUCKET_QUOTA_BYTES),
|
||||
"the bucket quota changed across the upgrade"
|
||||
);
|
||||
|
||||
assert_default_sse_s3_encryption(&new_client, CONFIG_ENCRYPTED_BUCKET, "after the upgrade").await?;
|
||||
let public_access_block = new_client
|
||||
.get_public_access_block()
|
||||
.bucket(CONFIG_ENCRYPTED_BUCKET)
|
||||
.send()
|
||||
.await?;
|
||||
let public_access_block = public_access_block
|
||||
.public_access_block_configuration()
|
||||
.ok_or("GetPublicAccessBlock omitted the configuration")?;
|
||||
assert_eq!(public_access_block.block_public_acls(), Some(true));
|
||||
assert_eq!(public_access_block.ignore_public_acls(), Some(true));
|
||||
assert_eq!(public_access_block.block_public_policy(), Some(true));
|
||||
assert_eq!(public_access_block.restrict_public_buckets(), Some(true));
|
||||
|
||||
assert_versioning_enabled(&new_client, CONFIG_REPLICATED_BUCKET, "after the upgrade").await?;
|
||||
// rustfs#7172: neither an empty list nor an error is acceptable here.
|
||||
assert_remote_target_preserved(&env, CONFIG_REPLICATED_BUCKET, &target_arn, "after the upgrade").await?;
|
||||
let replication = new_client
|
||||
.get_bucket_replication()
|
||||
.bucket(CONFIG_REPLICATED_BUCKET)
|
||||
.send()
|
||||
.await?;
|
||||
let replication_rules = replication
|
||||
.replication_configuration()
|
||||
.ok_or("GetBucketReplication omitted the configuration")?
|
||||
.rules();
|
||||
assert_eq!(
|
||||
replication_rules.len(),
|
||||
1,
|
||||
"the replication rule count changed across the upgrade: {replication_rules:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
replication_rules[0].destination().map(|destination| destination.bucket()),
|
||||
Some(target_arn.as_str()),
|
||||
"the replication rule no longer points at the configured target"
|
||||
);
|
||||
|
||||
let object_lock = new_client
|
||||
.get_object_lock_configuration()
|
||||
.bucket(CONFIG_LOCKED_BUCKET)
|
||||
.send()
|
||||
.await?;
|
||||
let object_lock = object_lock
|
||||
.object_lock_configuration()
|
||||
.ok_or("GetObjectLockConfiguration omitted the configuration")?;
|
||||
assert_eq!(object_lock.object_lock_enabled(), Some(&ObjectLockEnabled::Enabled));
|
||||
let retention = object_lock
|
||||
.rule()
|
||||
.and_then(ObjectLockRule::default_retention)
|
||||
.ok_or("the object lock configuration lost its default retention")?;
|
||||
assert_eq!(retention.mode(), Some(&ObjectLockRetentionMode::Governance));
|
||||
assert_eq!(retention.days(), Some(OBJECT_LOCK_DAYS));
|
||||
|
||||
// rustfs#7183: a PUT into the default-encrypted bucket must still succeed
|
||||
// and still come back encrypted.
|
||||
let post_upgrade_encrypted_key = "encrypted/written-after-upgrade";
|
||||
let post_upgrade_encrypted_bytes = b"default-encrypted object written by the current RustFS build";
|
||||
new_client
|
||||
.put_object()
|
||||
.bucket(CONFIG_ENCRYPTED_BUCKET)
|
||||
.key(post_upgrade_encrypted_key)
|
||||
.body(ByteStream::from_static(post_upgrade_encrypted_bytes))
|
||||
.send()
|
||||
.await?;
|
||||
let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, post_upgrade_encrypted_key, None).await?;
|
||||
assert_eq!(
|
||||
encryption,
|
||||
Some(ServerSideEncryption::Aes256),
|
||||
"a PUT after the upgrade lost the bucket default encryption"
|
||||
);
|
||||
assert_eq!(body, post_upgrade_encrypted_bytes);
|
||||
|
||||
let post_upgrade_plain_key = "plain/written-after-upgrade";
|
||||
let post_upgrade_plain_bytes = b"plain object written by the current RustFS build";
|
||||
put_object_through_quota_warmup(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, post_upgrade_plain_bytes).await?;
|
||||
let (encryption, body) = read_object(&new_client, CONFIG_PLAIN_BUCKET, post_upgrade_plain_key, None).await?;
|
||||
assert_eq!(encryption, None, "a bucket without default encryption must not encrypt a PUT");
|
||||
assert_eq!(body, post_upgrade_plain_bytes);
|
||||
|
||||
// Every object written by the previous release reads back byte-identical.
|
||||
assert_eq!(read_object(&new_client, CONFIG_PLAIN_BUCKET, plain_key, None).await?.1, plain_bytes);
|
||||
let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, encrypted_key, None).await?;
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, encrypted_bytes);
|
||||
let (encryption, body) = read_object(&new_client, CONFIG_ENCRYPTED_BUCKET, multipart_key, None).await?;
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, multipart_bytes, "the multipart object did not survive the upgrade");
|
||||
assert_eq!(
|
||||
read_object(&new_client, CONFIG_REPLICATED_BUCKET, versioned_key, Some(&versioned_id))
|
||||
.await?
|
||||
.1,
|
||||
versioned_bytes
|
||||
);
|
||||
|
||||
// rustfs#7089: the migration module is on by default, but a bucket that
|
||||
// never configured a source behaves exactly as before.
|
||||
assert_migration_not_configured(&env, CONFIG_PLAIN_BUCKET).await?;
|
||||
assert_missing_key_is_no_such_key(&new_client, CONFIG_PLAIN_BUCKET, "plain/never-written").await?;
|
||||
|
||||
replication_target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rolling back to the pinned previous release must still read the bucket
|
||||
/// metadata the current build wrote.
|
||||
///
|
||||
/// This is the other half of the `BucketMetadata` 44 -> 46 key change: the
|
||||
/// current build writes a 46-key msgpack map with `OnDemandMigrationConfigJSON`
|
||||
/// and `OnDemandMigrationConfigUpdatedAt`, and the previous release's decoder
|
||||
/// has to skip those two unknown keys instead of failing the whole blob. If it
|
||||
/// did not, every configuration read below would come back empty or error and
|
||||
/// the rollback would silently discard the bucket's configuration.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
|
||||
let replication_target = FakeS3Target::start().await?;
|
||||
replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET);
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let server_env = bucket_config_server_env();
|
||||
env.start_rustfs_server_with_env(vec![], &server_env).await?;
|
||||
let new_client = env.create_s3_client();
|
||||
|
||||
env.create_test_bucket(ROLLBACK_BUCKET).await?;
|
||||
enable_versioning(&new_client, ROLLBACK_BUCKET).await?;
|
||||
put_default_sse_s3_encryption(&new_client, ROLLBACK_BUCKET).await?;
|
||||
put_bucket_tag(&new_client, ROLLBACK_BUCKET).await?;
|
||||
let target_arn = configure_replication(&env, ROLLBACK_BUCKET, &replication_target, ROLLBACK_REPLICA_BUCKET).await?;
|
||||
assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "before the rollback").await?;
|
||||
|
||||
let single_key = "rollback/single";
|
||||
let single_bytes = b"single-part object written by the current RustFS build";
|
||||
let single_version = new_client
|
||||
.put_object()
|
||||
.bucket(ROLLBACK_BUCKET)
|
||||
.key(single_key)
|
||||
.body(ByteStream::from_static(single_bytes))
|
||||
.send()
|
||||
.await?
|
||||
.version_id()
|
||||
.ok_or("versioned PUT omitted version ID")?
|
||||
.to_string();
|
||||
|
||||
let multipart_key = "rollback/multipart";
|
||||
let multipart_parts = vec![vec![b'r'; 5 * 1024 * 1024], b"final rollback bytes".to_vec()];
|
||||
let multipart_bytes = multipart_parts.concat();
|
||||
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
|
||||
|
||||
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
|
||||
let old_client = env.create_s3_client();
|
||||
|
||||
assert_versioning_enabled(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
|
||||
assert_default_sse_s3_encryption(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
|
||||
assert_bucket_tag(&old_client, ROLLBACK_BUCKET, "after the rollback").await?;
|
||||
assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "after the rollback").await?;
|
||||
|
||||
let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, single_key, Some(&single_version)).await?;
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, single_bytes);
|
||||
let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, multipart_key, None).await?;
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
|
||||
|
||||
// A PUT on the rolled-back release must still honour the encryption
|
||||
// configuration it decoded out of the current build's metadata blob.
|
||||
let post_rollback_key = "rollback/written-after-rollback";
|
||||
let post_rollback_bytes = b"object written by the previous RustFS release after the rollback";
|
||||
old_client
|
||||
.put_object()
|
||||
.bucket(ROLLBACK_BUCKET)
|
||||
.key(post_rollback_key)
|
||||
.body(ByteStream::from_static(post_rollback_bytes))
|
||||
.send()
|
||||
.await?;
|
||||
let (encryption, body) = read_object(&old_client, ROLLBACK_BUCKET, post_rollback_key, None).await?;
|
||||
assert_eq!(
|
||||
encryption,
|
||||
Some(ServerSideEncryption::Aes256),
|
||||
"the rolled-back release lost the bucket default encryption"
|
||||
);
|
||||
assert_eq!(body, post_rollback_bytes);
|
||||
|
||||
replication_target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -31,7 +31,6 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
gcs = ["dep:google-cloud-storage", "dep:google-cloud-auth"]
|
||||
# Compiles the controlled list-objects namespace-journal chaos injector into a
|
||||
# production binary (it is always available to tests). Off by default so the
|
||||
# RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal
|
||||
@@ -213,8 +212,8 @@ aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
|
||||
parking_lot = { workspace = true }
|
||||
base64-simd.workspace = true
|
||||
serde_urlencoded.workspace = true
|
||||
google-cloud-storage = { workspace = true, optional = true }
|
||||
google-cloud-auth = { workspace = true, optional = true }
|
||||
google-cloud-storage = { workspace = true }
|
||||
google-cloud-auth = { workspace = true }
|
||||
faster-hex = { workspace = true }
|
||||
ratelimit = { workspace = true }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
|
||||
@@ -32,7 +32,7 @@ pub mod bucket {
|
||||
pub mod bucket_target_sys {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
SsecPassthroughCapability, TargetClient, UnreadableTargetsPolicy, append_version_id_query,
|
||||
SsecPassthroughCapability, TargetClient, append_version_id_query,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -69,13 +69,6 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod recovery_control {
|
||||
pub use crate::bucket::lifecycle::recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControlPage, IlmRecoveryControlView, IlmRecoveryProtocol,
|
||||
inspect_recovery_control, list_recovery_controls,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod transition_transaction {
|
||||
pub use crate::bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
|
||||
@@ -96,9 +89,8 @@ pub mod bucket {
|
||||
#[allow(clippy::module_inception)]
|
||||
pub mod lifecycle {
|
||||
pub use crate::bucket::lifecycle::lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
|
||||
ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
|
||||
object_opts_from_object_info,
|
||||
Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate,
|
||||
TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time, object_opts_from_object_info,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -154,23 +146,67 @@ pub mod bucket {
|
||||
};
|
||||
}
|
||||
|
||||
pub mod on_demand_migration {
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
ApplyOutcome, BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION,
|
||||
Breaker, BreakerState, BreakerTransition, BreakerVerdict, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, GaugeGuard,
|
||||
LastSourceError, LatencyBucketSnapshot, NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache, OdmBucketSnapshot, OdmLookup,
|
||||
OdmOp, OdmOutcome, OdmStateError, OdmStats, OdmStatsSnapshot, OnDemandMigrationSys, PullError, PullFailureReason,
|
||||
PullFollower, PullLeader, PullOutcome, PullPath, PullResult, PullSlot, SOURCE_LATENCY_BUCKET_BOUNDS_MS,
|
||||
SourceLatencySnapshot, source_client_spec,
|
||||
};
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
|
||||
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy,
|
||||
SourceConfig, SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
|
||||
};
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
EnqueueOutcome, LocalObject, MAX_MULTIPART_PARTS, OdmWriteBack, PULL_MAX_RETRIES, PULL_RETRY_BASE_DELAYS,
|
||||
PullCompletion, PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody,
|
||||
WriteBackError, WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with,
|
||||
idle_guarded_body,
|
||||
};
|
||||
pub use crate::bucket::on_demand_migration::{
|
||||
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
|
||||
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
|
||||
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
pub mod backfill {
|
||||
pub use crate::bucket::on_demand_migration::backfill::{
|
||||
BACKFILL_CHECKPOINT_FILE, BACKFILL_CHECKPOINT_FORMAT_VERSION, BACKFILL_FAILED_KEYS_CAPACITY, BACKFILL_LEASE,
|
||||
BACKFILL_LEASE_LOCK_PREFIX, BACKFILL_LIST_PAGE_SIZE, BACKFILL_RECOVERY_INTERVAL, BACKFILL_SAVE_EVERY_KEYS,
|
||||
BACKFILL_SAVE_INTERVAL, BackfillCheckpoint, BackfillContext, BackfillContextFactory, BackfillError,
|
||||
BackfillLastError, BackfillOwner, BackfillRecoveryStats, BackfillRequest, BackfillRunner, BackfillState,
|
||||
BucketBackfillContext, LocalBackfillObject, PriorityPullPermits, PullPermit, PullPriority, SkipExisting,
|
||||
StoredCheckpoint, SysBackfillContexts, global_backfill_runner, install_global_backfill_runner, key_hash,
|
||||
read_checkpoint, run_backfill_recovery_loop, spawn_backfill_recovery_loop,
|
||||
};
|
||||
}
|
||||
pub mod source_client {
|
||||
pub use crate::bucket::on_demand_migration::source_client::{
|
||||
SourceClient, SourceClientSpec, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage,
|
||||
SourceProbe, SourceProvider, SourceSse, SourceTimeouts, USER_AGENT_SUFFIX, is_multipart_etag, range_header_value,
|
||||
resolve_path_style,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
pub mod metadata_sys {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
|
||||
pub use crate::bucket::metadata_sys::{
|
||||
BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
|
||||
ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
|
||||
acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
|
||||
capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
|
||||
get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
|
||||
get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
|
||||
get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
|
||||
get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
|
||||
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
|
||||
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
|
||||
update, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation,
|
||||
update_quota_if_incarnation, update_under_transaction_lock,
|
||||
get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
|
||||
get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
|
||||
reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
|
||||
update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
|
||||
update_under_transaction_lock,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::bucket::metadata_sys::{ConfigWriteLockProbe, test_support};
|
||||
}
|
||||
|
||||
pub mod migration {
|
||||
@@ -213,7 +249,7 @@ pub mod bucket {
|
||||
pub mod remote_s3_client {
|
||||
pub use crate::bucket::remote_s3_client::{
|
||||
PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_client,
|
||||
build_remote_s3_config, validate_remote_endpoint, validate_target_ca_pem,
|
||||
validate_remote_endpoint,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -459,9 +495,9 @@ pub mod object {
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
|
||||
ScannerPublicationCommitState, StreamConsumer, WriteCompletion, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
ScannerPublicationCommitState, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
|
||||
register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
|
||||
unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
@@ -525,12 +561,6 @@ pub mod set_disk {
|
||||
pub mod test_util {
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
/// Keep a namespace commit pending until the returned owner is dropped.
|
||||
#[must_use]
|
||||
pub fn hold_namespace_commit(store: &crate::store::ECStore) -> impl Send + Sync {
|
||||
store.ctx.begin_namespace_commit()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -369,26 +369,6 @@ struct SsecPassthroughRecord {
|
||||
recorded_at: Instant,
|
||||
}
|
||||
|
||||
/// What a target write does when the bucket's persisted target set exists but
|
||||
/// cannot be decoded.
|
||||
///
|
||||
/// `docs/architecture/remote-credential-sealing-adr.md` forbids rewriting a
|
||||
/// configuration that could not be fully read, because re-serializing a
|
||||
/// partial in-memory view is the one mechanism by which a configured target
|
||||
/// really disappears. That rule guards against an *unintentional* overwrite,
|
||||
/// so an operator who names the hazard keeps a repair path
|
||||
/// (rustfs/backlog#2309); everything that does not name it stays refused.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum UnreadableTargetsPolicy {
|
||||
/// Refuse the write with [`BucketTargetError::BucketRemoteTargetsUnreadable`].
|
||||
#[default]
|
||||
FailClosed,
|
||||
/// Discard the unreadable set; the target being written becomes the whole
|
||||
/// configuration. Reachable only from an admin request that asked for it
|
||||
/// explicitly, and audited by the caller.
|
||||
Replace,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BucketTargetSys {
|
||||
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
|
||||
@@ -811,45 +791,20 @@ impl BucketTargetSys {
|
||||
bucket: &str,
|
||||
target: &BucketTarget,
|
||||
update: bool,
|
||||
unreadable_policy: UnreadableTargetsPolicy,
|
||||
) -> Result<BucketTargets, BucketTargetError> {
|
||||
self.validate_target(bucket, target).await?;
|
||||
|
||||
let mut bucket_targets = self.targets_base_for_write(bucket, unreadable_policy).await?;
|
||||
let mut bucket_targets = match self.list_bucket_targets(bucket).await {
|
||||
Ok(targets) => targets,
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => BucketTargets::default(),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
Self::upsert_target_entry(&mut bucket_targets.targets, target, update)?;
|
||||
|
||||
Ok(bucket_targets)
|
||||
}
|
||||
|
||||
/// The persisted target set a write merges into.
|
||||
///
|
||||
/// An absent configuration starts from the empty set. An unreadable one is
|
||||
/// refused, because re-serializing a partial view of a set this node could
|
||||
/// not decode is how a configured target disappears for good — unless the
|
||||
/// caller carries the operator's explicit
|
||||
/// [`UnreadableTargetsPolicy::Replace`] opt-in, which discards it
|
||||
/// deliberately (rustfs/backlog#2309).
|
||||
async fn targets_base_for_write(
|
||||
&self,
|
||||
bucket: &str,
|
||||
unreadable_policy: UnreadableTargetsPolicy,
|
||||
) -> Result<BucketTargets, BucketTargetError> {
|
||||
match self.list_bucket_targets(bucket).await {
|
||||
Ok(targets) => Ok(targets),
|
||||
Err(BucketTargetError::BucketRemoteTargetNotFound { .. }) => Ok(BucketTargets::default()),
|
||||
// The opt-in discards only a set this node genuinely cannot read.
|
||||
// A readable set still merges through the arm above, so the policy
|
||||
// can never drop a target that was visible here.
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
if unreadable_policy == UnreadableTargetsPolicy::Replace =>
|
||||
{
|
||||
Ok(BucketTargets::default())
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn validate_target(&self, bucket: &str, target: &BucketTarget) -> Result<(), BucketTargetError> {
|
||||
if !target.target_type.is_valid() {
|
||||
return Err(BucketTargetError::BucketRemoteArnTypeInvalid {
|
||||
@@ -4358,75 +4313,4 @@ mod tests {
|
||||
let window = LastMinuteLatency::new();
|
||||
assert_eq!(window.get_total().avg, Duration::from_secs(0));
|
||||
}
|
||||
|
||||
fn repair_target(bucket: &str, id: &str) -> BucketTarget {
|
||||
BucketTarget {
|
||||
source_bucket: bucket.to_string(),
|
||||
endpoint: "remote.example.com".to_string(),
|
||||
target_bucket: "remote".to_string(),
|
||||
arn: format!("arn:rustfs:replication:us-east-1:{bucket}:{id}"),
|
||||
target_type: BucketTargetType::ReplicationService,
|
||||
region: "us-east-1".to_string(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2309: after rustfs/rustfs#7172 an undecodable
|
||||
/// `bucket-targets.json` left the bucket with no API repair path at all.
|
||||
/// The refusal is the default and stays the default; the operator's
|
||||
/// explicit opt-in is the only thing that discards the set, and it starts
|
||||
/// the replacement from empty rather than from a partial view of bytes
|
||||
/// this node never decoded.
|
||||
#[tokio::test]
|
||||
async fn an_unreadable_target_set_is_replaced_only_with_the_explicit_opt_in() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let bucket = "targets-repair-opt-in";
|
||||
sys.mark_targets_unreadable(bucket).await;
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
sys.targets_base_for_write(bucket, UnreadableTargetsPolicy::FailClosed).await,
|
||||
Err(BucketTargetError::BucketRemoteTargetsUnreadable { .. })
|
||||
),
|
||||
"without the opt-in an unreadable target set must still refuse the write"
|
||||
);
|
||||
assert_eq!(
|
||||
UnreadableTargetsPolicy::default(),
|
||||
UnreadableTargetsPolicy::FailClosed,
|
||||
"a caller that says nothing must get the refusal"
|
||||
);
|
||||
|
||||
let base = sys
|
||||
.targets_base_for_write(bucket, UnreadableTargetsPolicy::Replace)
|
||||
.await
|
||||
.expect("the explicit opt-in must let an operator replace an unreadable set");
|
||||
assert!(
|
||||
base.is_empty(),
|
||||
"the replacement must start from an empty set, never from a partial decode"
|
||||
);
|
||||
}
|
||||
|
||||
/// The opt-in is not a wipe switch. On a set this node can read, both
|
||||
/// policies take the same merge path, so a stray `replace-unreadable=true`
|
||||
/// cannot drop a visible target — which is what makes the flag safe to
|
||||
/// repeat in an operator's repair script.
|
||||
#[tokio::test]
|
||||
async fn the_opt_in_never_discards_a_readable_target_set() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let bucket = "targets-repair-readable";
|
||||
let existing = repair_target(bucket, "keep");
|
||||
sys.targets_map
|
||||
.write()
|
||||
.await
|
||||
.insert(bucket.to_string(), vec![existing.clone()]);
|
||||
|
||||
for policy in [UnreadableTargetsPolicy::FailClosed, UnreadableTargetsPolicy::Replace] {
|
||||
let base = sys
|
||||
.targets_base_for_write(bucket, policy)
|
||||
.await
|
||||
.expect("a readable target set must be readable under either policy");
|
||||
assert_eq!(base.targets.len(), 1, "{policy:?} must keep the persisted target");
|
||||
assert_eq!(base.targets[0].arn, existing.arn, "{policy:?} must not rewrite the persisted target");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,18 +150,6 @@ static XXHASH_SEED: u64 = 0;
|
||||
static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
struct FreeVersionPostRemoteDeleteTestBarrier {
|
||||
arrived: Notify,
|
||||
release: Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER: Arc<FreeVersionPostRemoteDeleteTestBarrier>;
|
||||
}
|
||||
|
||||
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -922,11 +910,6 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
|
||||
})??;
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
if let Ok(barrier) = FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER.try_with(Arc::clone) {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
if !free_version_cleanup_fences_current(&topology_generation, &api, &bucket_guard, &object_guards, &lease, cancel, deadline) {
|
||||
// Remote DELETE is idempotent, but a changed fence makes the local
|
||||
// outcome ambiguous. Keep every marker for a fully fenced retry.
|
||||
@@ -5848,7 +5831,7 @@ mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::test_util::register_mock_tier;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::tier::{TIER_DRIVER_TEST_FACTORY, TierConfigMgr, TierDriverTestFactory};
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::warm_backend::{TransitionCandidateProbe, WarmBackend as _};
|
||||
use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
@@ -7847,119 +7830,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_remove_waits_for_inflight_free_version_local_commit() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("tier-remove-free-version-{}", Uuid::new_v4());
|
||||
let object = "free-version";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let (backend, identity_hex) = register_recovery_mock_tier(&ecstore).await;
|
||||
let tier_manager = ecstore.tier_config_mgr();
|
||||
{
|
||||
let manager = tier_manager.read().await;
|
||||
manager
|
||||
.save_tiering_config(Arc::clone(&ecstore))
|
||||
.await
|
||||
.expect("mock tier configuration should persist before removal");
|
||||
}
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, object, None, Some(identity_hex)).await;
|
||||
let page = list_tier_free_versions(Arc::clone(&ecstore), 1, None, None, CancellationToken::new())
|
||||
.await
|
||||
.expect("seeded free version should be listed");
|
||||
let oi = page
|
||||
.items
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("seeded free version should be recoverable");
|
||||
|
||||
backend
|
||||
.set_put_remote_version(Some(oi.transitioned_object.version_id.clone()))
|
||||
.await;
|
||||
let seed_lease = TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM")
|
||||
.await
|
||||
.expect("mock tier lease should be available");
|
||||
seed_lease
|
||||
.put(&oi.transitioned_object.name, ReaderImpl::Body(Bytes::from_static(b"body")), 4)
|
||||
.await
|
||||
.expect("remote free-version tuple should be seeded");
|
||||
drop(seed_lease);
|
||||
|
||||
let barrier = Arc::new(super::FreeVersionPostRemoteDeleteTestBarrier::default());
|
||||
let cleanup_barrier = Arc::clone(&barrier);
|
||||
let cleanup_store = Arc::clone(&ecstore);
|
||||
let cleanup_oi = oi.clone();
|
||||
let cleanup = tokio::spawn(async move {
|
||||
super::FREE_VERSION_POST_REMOTE_DELETE_TEST_BARRIER
|
||||
.scope(cleanup_barrier, async move {
|
||||
super::cleanup_free_version_exact(cleanup_store, &cleanup_oi, &CancellationToken::new()).await
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(30), barrier.arrived.notified())
|
||||
.await
|
||||
.expect("free-version cleanup should pause after the remote delete");
|
||||
assert!(!backend.contains(&oi.transitioned_object.name).await);
|
||||
|
||||
let remove_manager = Arc::clone(&tier_manager);
|
||||
let remove_store = Arc::clone(&ecstore);
|
||||
let remove_backend = backend.clone();
|
||||
let remove_driver_factory: TierDriverTestFactory = Arc::new(move |_| Ok(Box::new(remove_backend.clone())));
|
||||
let mut remove = tokio::spawn(async move {
|
||||
TIER_DRIVER_TEST_FACTORY
|
||||
.scope(
|
||||
remove_driver_factory,
|
||||
TierConfigMgr::remove_and_save(&remove_manager, remove_store, "WARM", true),
|
||||
)
|
||||
.await
|
||||
});
|
||||
let prepared = tokio::time::timeout(StdDuration::from_secs(30), async {
|
||||
loop {
|
||||
match TierConfigMgr::acquire_operation_lease(&tier_manager, "WARM").await {
|
||||
Ok(lease) => drop(lease),
|
||||
Err(err) if TierConfigMgr::operation_lease_blocked_by_mutation(&err) => break,
|
||||
Err(err) => panic!("tier remove should only block new operations while cleanup is paused: {err}"),
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
});
|
||||
tokio::select! {
|
||||
prepared = prepared => {
|
||||
prepared.expect("tier remove should install its prepared admission fence");
|
||||
}
|
||||
result = &mut remove => {
|
||||
panic!("tier remove finished before installing its prepared admission fence: {result:?}");
|
||||
}
|
||||
}
|
||||
assert!(!remove.is_finished(), "tier remove must wait for the leased local cleanup commit");
|
||||
|
||||
barrier.release.notify_one();
|
||||
tokio::time::timeout(StdDuration::from_secs(30), cleanup)
|
||||
.await
|
||||
.expect("free-version cleanup should finish after release")
|
||||
.expect("free-version cleanup task should join")
|
||||
.expect("free-version cleanup should keep its generation current");
|
||||
tokio::time::timeout(StdDuration::from_secs(30), remove)
|
||||
.await
|
||||
.expect("tier remove should finish after local cleanup")
|
||||
.expect("tier remove task should join")
|
||||
.expect("tier remove should pass its fresh authoritative proof");
|
||||
|
||||
assert!(!tier_manager.read().await.is_tier_valid("WARM"));
|
||||
for disk_path in &disk_paths {
|
||||
assert!(
|
||||
!fs::try_exists(disk_path.join(&bucket).join(object))
|
||||
.await
|
||||
.expect("post-removal free-version path check should succeed")
|
||||
);
|
||||
}
|
||||
ecstore
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("empty free-version test bucket should be removed");
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
|
||||
@@ -15,9 +15,9 @@
|
||||
use crate::object_api::ObjectInfo;
|
||||
|
||||
pub use rustfs_lifecycle::{
|
||||
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
|
||||
RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
|
||||
expected_expiry_time, expiration_action_has_valid_target,
|
||||
Event, ExpirationOptions, IlmAction, Lifecycle, LifecycleCalculate, ObjectOpts, RuleValidate, TRANSITION_COMPLETE,
|
||||
TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due, expected_expiry_time,
|
||||
expiration_action_has_valid_target,
|
||||
};
|
||||
|
||||
pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {
|
||||
|
||||
@@ -22,7 +22,7 @@ use super::{
|
||||
bucket_lifecycle_ops::{
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunReport, decode_manual_transition_continuation_token,
|
||||
},
|
||||
manual_transition_job, recovery_control, tier_delete_journal, transition_transaction,
|
||||
manual_transition_job, tier_delete_journal, transition_transaction,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::services::tier::tier_probe_intent;
|
||||
@@ -41,7 +41,6 @@ pub(crate) enum DurableIlmRecordKind {
|
||||
ManualTransitionScope,
|
||||
ManualTransitionTask,
|
||||
ManualTransitionWorkerResult,
|
||||
RecoveryControl,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -106,14 +105,8 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
|
||||
max_record_size: manual_transition_job::MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE,
|
||||
kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
|
||||
};
|
||||
pub(crate) const RECOVERY_CONTROL_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "recovery-control",
|
||||
prefix: recovery_control::ILM_RECOVERY_CONTROL_PREFIX,
|
||||
max_record_size: recovery_control::MAX_ILM_RECOVERY_CONTROL_SIZE,
|
||||
kind: DurableIlmRecordKind::RecoveryControl,
|
||||
};
|
||||
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
|
||||
TIER_DELETE_JOURNAL_NAMESPACE,
|
||||
TIER_DELETE_JOURNAL_V6_NAMESPACE,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
|
||||
@@ -123,7 +116,6 @@ pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 10] = [
|
||||
MANUAL_TRANSITION_SCOPE_NAMESPACE,
|
||||
MANUAL_TRANSITION_TASK_NAMESPACE,
|
||||
MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
|
||||
RECOVERY_CONTROL_NAMESPACE,
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -249,18 +241,6 @@ pub(crate) enum DurableIlmRecordCheckpoint {
|
||||
ManualTransitionWorkerResult {
|
||||
content_sha256: String,
|
||||
},
|
||||
RecoveryControl {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
source_generation_sha256: String,
|
||||
first_seen_at_unix_nanos: i64,
|
||||
revision: u64,
|
||||
classification: recovery_control::IlmRecoveryClassification,
|
||||
attempt_count: u64,
|
||||
consecutive_failure_count: u32,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
owner_fence_sha256: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl DurableIlmRecordCheckpoint {
|
||||
@@ -274,8 +254,7 @@ impl DurableIlmRecordCheckpoint {
|
||||
| Self::ManualTransitionJob { content_sha256, .. }
|
||||
| Self::ManualTransitionScope { content_sha256, .. }
|
||||
| Self::ManualTransitionTask { content_sha256 }
|
||||
| Self::ManualTransitionWorkerResult { content_sha256 }
|
||||
| Self::RecoveryControl { content_sha256, .. } => content_sha256,
|
||||
| Self::ManualTransitionWorkerResult { content_sha256 } => content_sha256,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -549,51 +528,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
..
|
||||
},
|
||||
) => previous_identity == next_identity && next_updated_at > previous_updated_at,
|
||||
(
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: previous_identity,
|
||||
source_generation_sha256: previous_generation,
|
||||
first_seen_at_unix_nanos: previous_first_seen,
|
||||
revision: previous_revision,
|
||||
classification: previous_classification,
|
||||
attempt_count: previous_attempts,
|
||||
consecutive_failure_count: previous_failures,
|
||||
owner_fence_sha256: previous_owner,
|
||||
..
|
||||
},
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: next_identity,
|
||||
source_generation_sha256: next_generation,
|
||||
first_seen_at_unix_nanos: next_first_seen,
|
||||
revision: next_revision,
|
||||
classification: next_classification,
|
||||
attempt_count: next_attempts,
|
||||
consecutive_failure_count: next_failures,
|
||||
owner_fence_sha256: next_owner,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
let adjacent = previous_identity == next_identity
|
||||
&& previous_first_seen == next_first_seen
|
||||
&& previous_revision.checked_add(1) == Some(*next_revision);
|
||||
let claim = next_owner.is_some()
|
||||
&& *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& *next_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& previous_attempts.checked_add(1) == Some(*next_attempts)
|
||||
&& previous_failures == next_failures;
|
||||
let source_refresh = previous_owner.is_some()
|
||||
&& previous_owner == next_owner
|
||||
&& *previous_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& *next_classification == recovery_control::IlmRecoveryClassification::Retrying
|
||||
&& previous_attempts == next_attempts
|
||||
&& previous_failures == next_failures
|
||||
&& previous_generation != next_generation;
|
||||
let completion = previous_owner.is_some()
|
||||
&& next_owner.is_none()
|
||||
&& previous_generation == next_generation
|
||||
&& previous_attempts == next_attempts;
|
||||
adjacent && (claim || source_refresh || completion)
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
|
||||
@@ -619,14 +553,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if let Self::RecoveryControl { classification, .. } = terminal
|
||||
&& !matches!(
|
||||
classification,
|
||||
recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self == terminal || self.validate_successor(terminal).is_ok() {
|
||||
return true;
|
||||
}
|
||||
@@ -726,32 +652,6 @@ impl DurableIlmRecordCheckpoint {
|
||||
.is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
|
||||
&& (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
|
||||
}
|
||||
(
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: previous_identity,
|
||||
source_generation_sha256: previous_generation,
|
||||
first_seen_at_unix_nanos: previous_first_seen,
|
||||
revision: previous_revision,
|
||||
attempt_count: previous_attempts,
|
||||
..
|
||||
},
|
||||
Self::RecoveryControl {
|
||||
identity_sha256: terminal_identity,
|
||||
source_generation_sha256: terminal_generation,
|
||||
first_seen_at_unix_nanos: terminal_first_seen,
|
||||
revision: terminal_revision,
|
||||
attempt_count: terminal_attempts,
|
||||
classification:
|
||||
recovery_control::IlmRecoveryClassification::Terminal | recovery_control::IlmRecoveryClassification::Abandoned,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
previous_identity == terminal_identity
|
||||
&& (previous_generation == terminal_generation || terminal_attempts > previous_attempts)
|
||||
&& previous_first_seen == terminal_first_seen
|
||||
&& terminal_revision > previous_revision
|
||||
&& terminal_attempts >= previous_attempts
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -1319,35 +1219,6 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::RecoveryControl => {
|
||||
let (protocol, control_id) = recovery_control::recovery_control_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
let control =
|
||||
recovery_control::IlmRecoveryControl::decode(&control_id, data).map_err(|err| Error::other(err.to_string()))?;
|
||||
let canonical = recovery_control::recovery_control_record_object_name(protocol, &control_id)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
if canonical != path || control.identity.protocol != protocol {
|
||||
return Err(Error::other("ILM recovery control path is not canonical"));
|
||||
}
|
||||
let identity_sha256 = checkpoint_hash(&control.identity)?;
|
||||
let source_generation_sha256 = checkpoint_hash(&control.observed_source_generation)?;
|
||||
let owner_fence_sha256 = control.owner.as_ref().map(checkpoint_hash).transpose()?;
|
||||
(
|
||||
"control_id",
|
||||
control_id,
|
||||
DurableIlmRecordCheckpoint::RecoveryControl {
|
||||
content_sha256,
|
||||
identity_sha256,
|
||||
source_generation_sha256,
|
||||
first_seen_at_unix_nanos: control.first_seen_at_unix_nanos,
|
||||
revision: control.revision,
|
||||
classification: control.classification,
|
||||
attempt_count: control.attempt_count,
|
||||
consecutive_failure_count: control.consecutive_failure_count,
|
||||
owner_fence_sha256,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::ManualTransitionJob => {
|
||||
let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
@@ -1541,94 +1412,6 @@ mod tests {
|
||||
.checkpoint
|
||||
}
|
||||
|
||||
fn recovery_control_fixture() -> recovery_control::IlmRecoveryControl {
|
||||
let source_path = "ilm/transition-transactions/records/12/34/1234567890abcdef1234567890abcdef.json";
|
||||
let generation = recovery_control::IlmRecoverySourceGeneration::new(
|
||||
transition_transaction::TRANSITION_TRANSACTION_SCHEMA,
|
||||
"source-etag",
|
||||
"a".repeat(64),
|
||||
vec![recovery_control::IlmRecoverySourceCopy {
|
||||
authority: "pool-0/set-0".to_string(),
|
||||
canonical_path: source_path.to_string(),
|
||||
etag: "source-etag".to_string(),
|
||||
encoded_len: 128,
|
||||
content_sha256: "a".repeat(64),
|
||||
}],
|
||||
)
|
||||
.expect("source generation should build");
|
||||
recovery_control::IlmRecoveryControl::new(
|
||||
recovery_control::IlmRecoveryControlIdentity {
|
||||
protocol: recovery_control::IlmRecoveryProtocol::TransitionTransaction,
|
||||
canonical_source_path: source_path.to_string(),
|
||||
stable_operation_identity: "12345678-90ab-cdef-1234-567890abcdef".to_string(),
|
||||
record_class: "transition_transaction_v1".to_string(),
|
||||
},
|
||||
generation,
|
||||
recovery_control::IlmRecoveryClassification::Retrying,
|
||||
1_000_000_000,
|
||||
recovery_control::IlmRecoveryErrorCode::None,
|
||||
)
|
||||
.expect("recovery control should build")
|
||||
}
|
||||
|
||||
fn recovery_control_checkpoint(control: &recovery_control::IlmRecoveryControl) -> DurableIlmRecordCheckpoint {
|
||||
let control_id = control.identity.source_operation_digest().expect("control id should derive");
|
||||
let path = recovery_control::recovery_control_record_object_name(control.identity.protocol, &control_id)
|
||||
.expect("control path should build");
|
||||
let encoded = control.encode().expect("control should encode");
|
||||
let namespace = classify_durable_ilm_record(&path)
|
||||
.expect("recovery control namespace should classify")
|
||||
.expect("recovery control should be durable");
|
||||
assert_eq!(namespace, &RECOVERY_CONTROL_NAMESPACE);
|
||||
validate_durable_ilm_record(&path, &encoded)
|
||||
.expect("recovery control should validate")
|
||||
.checkpoint
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_control_checkpoint_tracks_claim_retry_and_terminal_generations() {
|
||||
let initial_control = recovery_control_fixture();
|
||||
let initial = recovery_control_checkpoint(&initial_control);
|
||||
|
||||
let mut claimed_control = initial_control;
|
||||
let mut advanced_generation = claimed_control.observed_source_generation.clone();
|
||||
advanced_generation.source_schema = "rustfs-transition-transaction-v2".to_string();
|
||||
claimed_control
|
||||
.claim_for_source_generation("node-a", Uuid::new_v4(), 2_000_000_000, 300_000_000_000, advanced_generation)
|
||||
.expect("control should claim");
|
||||
let claimed = recovery_control_checkpoint(&claimed_control);
|
||||
initial.validate_successor(&claimed).expect("claim should advance receipt");
|
||||
|
||||
let mut retry_control = claimed_control;
|
||||
retry_control
|
||||
.record_retryable_failure(3_000_000_000, recovery_control::IlmRecoveryErrorCode::BackendTimeout)
|
||||
.expect("retry should persist");
|
||||
let retry = recovery_control_checkpoint(&retry_control);
|
||||
claimed.validate_successor(&retry).expect("retry should advance receipt");
|
||||
|
||||
let ready_at = retry_control
|
||||
.next_attempt_at_unix_nanos
|
||||
.expect("retry deadline should persist");
|
||||
let mut terminal_control = retry_control;
|
||||
terminal_control
|
||||
.claim("node-b", Uuid::new_v4(), ready_at, 300_000_000_000)
|
||||
.expect("retry should claim");
|
||||
let reclaimed = recovery_control_checkpoint(&terminal_control);
|
||||
retry.validate_successor(&reclaimed).expect("reclaim should advance receipt");
|
||||
terminal_control
|
||||
.finish_attempt(
|
||||
recovery_control::IlmRecoveryClassification::Terminal,
|
||||
recovery_control::IlmRecoveryErrorCode::None,
|
||||
)
|
||||
.expect("control should terminate");
|
||||
let terminal = recovery_control_checkpoint(&terminal_control);
|
||||
reclaimed
|
||||
.validate_successor(&terminal)
|
||||
.expect("terminal state should advance receipt");
|
||||
assert!(initial.is_predecessor_of_terminal(&terminal));
|
||||
assert!(!initial.is_predecessor_of_terminal(&retry));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
|
||||
let initial_intent = tier_probe_intent_fixture();
|
||||
|
||||
@@ -24,7 +24,6 @@ pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, g
|
||||
mod object_handlers_common;
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
pub mod recovery_control;
|
||||
mod replication_sink;
|
||||
pub mod rule;
|
||||
mod runtime_boundary;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -575,7 +575,7 @@ pub(crate) async fn delete_confirmed_transition_candidate_exact_with_lease_idemp
|
||||
#[cfg(test)]
|
||||
static CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn delete_confirmed_transition_candidate_exact_with_manager_and_identity(
|
||||
obj_name: &str,
|
||||
rv_id: &str,
|
||||
@@ -706,16 +706,15 @@ pub(crate) fn transitioned_delete_journal_entry_for_source(
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
#[cfg(feature = "test-util")]
|
||||
use super::delete_confirmed_transition_candidate_exact_with_manager_and_identity;
|
||||
use rustfs_s3_client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
use super::{
|
||||
CONFIRMED_TRANSITION_EMPTY_GUARD_DISPATCHES, ERR_REMOTE_DELETE_BREAKER_OPEN, ERR_REMOTE_DELETE_LIMITER_CLOSED, Jentry,
|
||||
RemoteDeleteBreaker, RemoteTierDeleteOutcome, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
delete_object_from_remote_tier_idempotent, delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
is_remote_tier_not_found_error, is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook,
|
||||
should_record_remote_delete_failure, transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity, delete_object_from_remote_tier_idempotent,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity, is_remote_tier_not_found_error,
|
||||
is_signer_header_error, lifecycle, set_remote_tier_delete_test_hook, should_record_remote_delete_failure,
|
||||
transitioned_delete_journal_entry, transitioned_force_delete_journal_entry,
|
||||
};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use rustfs_filemeta::TransitionVersionState;
|
||||
|
||||
@@ -23,11 +23,6 @@ use uuid::Uuid;
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::lifecycle::recovery_control::{
|
||||
IlmRecoveryClassification, IlmRecoveryControl, IlmRecoveryControlIdentity, IlmRecoveryErrorCode, IlmRecoveryProtocol,
|
||||
ObservedIlmRecoveryControl, load_recovery_control, observe_recovery_source, recovery_control_record_object_name,
|
||||
save_recovery_control_if_absent, save_recovery_control_if_current,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
@@ -49,7 +44,6 @@ const EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY: &str = "lifecycle_transit
|
||||
pub const DEFAULT_TRANSITION_TRANSACTION_RECOVERY_LIMIT: usize = 1_000;
|
||||
const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const TRANSITION_RECOVERY_CONTROL_LEASE_NANOS: i64 = 15 * 60 * 1_000_000_000;
|
||||
pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
|
||||
pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
|
||||
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
|
||||
@@ -743,11 +737,9 @@ pub enum TransitionTransactionRecoveryOutcome {
|
||||
RemoteCandidateDeleted,
|
||||
RecordDeleted,
|
||||
Retained,
|
||||
RetainedAmbiguous(IlmRecoveryErrorCode),
|
||||
OperatorRequired(IlmRecoveryErrorCode),
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
struct TransitionRecoveryClaimBarrierState {
|
||||
transaction_id: Uuid,
|
||||
@@ -755,17 +747,17 @@ struct TransitionRecoveryClaimBarrierState {
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
pub(crate) struct TransitionRecoveryClaimBarrier {
|
||||
state: Arc<TransitionRecoveryClaimBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
static TRANSITION_RECOVERY_CLAIM_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<TransitionRecoveryClaimBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
impl TransitionRecoveryClaimBarrier {
|
||||
pub(crate) fn install(transaction_id: Uuid) -> Self {
|
||||
let state = Arc::new(TransitionRecoveryClaimBarrierState {
|
||||
@@ -796,7 +788,7 @@ impl TransitionRecoveryClaimBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
impl Drop for TransitionRecoveryClaimBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -810,7 +802,7 @@ impl Drop for TransitionRecoveryClaimBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
|
||||
let barrier = TRANSITION_RECOVERY_CLAIM_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -825,80 +817,6 @@ async fn pause_before_transition_recovery_claim(transaction_id: Uuid) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
#[derive(Default)]
|
||||
struct TransitionRecoveryTerminalBarrierState {
|
||||
transaction_id: Uuid,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) struct TransitionRecoveryTerminalBarrier {
|
||||
state: Arc<TransitionRecoveryTerminalBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
static TRANSITION_RECOVERY_TERMINAL_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<TransitionRecoveryTerminalBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl TransitionRecoveryTerminalBarrier {
|
||||
pub(crate) fn install(transaction_id: Uuid) -> Self {
|
||||
let state = Arc::new(TransitionRecoveryTerminalBarrierState {
|
||||
transaction_id,
|
||||
..Default::default()
|
||||
});
|
||||
let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition recovery terminal barrier mutex should not poison");
|
||||
assert!(
|
||||
slot.is_none(),
|
||||
"transition recovery terminal barrier must be installed by one test at a time"
|
||||
);
|
||||
*slot = Some(Arc::clone(&state));
|
||||
drop(slot);
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("transition recovery should persist terminal control before source cleanup");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
impl Drop for TransitionRecoveryTerminalBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = TRANSITION_RECOVERY_TERMINAL_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition recovery terminal barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
async fn pause_after_transition_recovery_terminal(transaction_id: Uuid) {
|
||||
let barrier = TRANSITION_RECOVERY_TERMINAL_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("transition recovery terminal barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.transaction_id == transaction_id)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TransitionOperatorProbe {
|
||||
@@ -1102,35 +1020,17 @@ fn transition_transaction_id_from_record_object_name(object: &str) -> Result<Uui
|
||||
let suffix = object
|
||||
.strip_prefix(&prefix)
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong prefix"))?;
|
||||
let mut parts = suffix.split('/');
|
||||
let shard_a = parts
|
||||
let file_name = suffix
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
|
||||
let shard_b = parts
|
||||
.next()
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
|
||||
let file_name = parts
|
||||
.next()
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path is incomplete"))?;
|
||||
if parts.next().is_some() {
|
||||
return Err(TransitionTransactionError::Corrupt("transaction record path is not canonical"));
|
||||
}
|
||||
let transaction_key = file_name
|
||||
.strip_suffix(".json")
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path has wrong suffix"))?;
|
||||
if transaction_key.len() != 32
|
||||
|| !transaction_key
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
|| shard_a != &transaction_key[..2]
|
||||
|| shard_b != &transaction_key[2..4]
|
||||
{
|
||||
if transaction_key.len() != 32 || !transaction_key.bytes().all(|byte| byte.is_ascii_hexdigit()) {
|
||||
return Err(TransitionTransactionError::Corrupt("transaction record path has invalid transaction id"));
|
||||
}
|
||||
Uuid::parse_str(transaction_key)
|
||||
.ok()
|
||||
.filter(|transaction_id| !transaction_id.is_nil())
|
||||
.ok_or(TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
|
||||
Uuid::parse_str(transaction_key).map_err(|_| TransitionTransactionError::Corrupt("transaction record path has invalid uuid"))
|
||||
}
|
||||
|
||||
pub async fn process_transition_transaction_record(
|
||||
@@ -1155,27 +1055,6 @@ async fn process_transition_transaction_record_at(
|
||||
) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
|
||||
let record_name =
|
||||
transition_transaction_record_object_name(observed.transaction_id).map_err(transition_transaction_store_error)?;
|
||||
let now_unix_nanos =
|
||||
i64::try_from(now_unix_nanos).map_err(|_| Error::other("transition transaction recovery timestamp does not fit i64"))?;
|
||||
let recovery_control_identity = transition_recovery_control_identity(observed, &record_name);
|
||||
let recovery_control_id = recovery_control_identity
|
||||
.source_operation_digest()
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
let control_record_name =
|
||||
recovery_control_record_object_name(IlmRecoveryProtocol::TransitionTransaction, &recovery_control_id)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
let control_lock = if transition_state_needs_recovery_control(observed, now_unix_nanos) {
|
||||
Some(
|
||||
api.new_ns_lock(RUSTFS_META_BUCKET, &format!("{control_record_name}.recovery-lock"))
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _control_guard = match &control_lock {
|
||||
Some(lock) => Some(lock.get_write_lock(crate::set_disk::get_lock_acquire_timeout()).await?),
|
||||
None => None,
|
||||
};
|
||||
// The synthetic key avoids nesting the recovery lock with the config
|
||||
// object's own I/O lock. Holding it across the bounded source proof and
|
||||
// remote DELETE elects one destructive recovery worker across nodes.
|
||||
@@ -1194,400 +1073,55 @@ async fn process_transition_transaction_record_at(
|
||||
return Ok(TransitionTransactionRecoveryOutcome::Retained);
|
||||
}
|
||||
|
||||
let mut recovery_control = if transition_state_needs_recovery_control(¤t, now_unix_nanos) {
|
||||
if cleanup_terminal_transition_recovery_control(
|
||||
api.clone(),
|
||||
¤t,
|
||||
&record_name,
|
||||
&recovery_control_identity,
|
||||
&recovery_control_id,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
return Ok(TransitionTransactionRecoveryOutcome::RecordDeleted);
|
||||
}
|
||||
match claim_transition_recovery_control(
|
||||
api.clone(),
|
||||
¤t,
|
||||
&record_name,
|
||||
recovery_control_identity,
|
||||
&recovery_control_id,
|
||||
now_unix_nanos,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
Some(control) => Some(control),
|
||||
None => return Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let recovery = match current.state {
|
||||
match current.state {
|
||||
TransitionTransactionState::Uploaded => {
|
||||
if transition_transaction_ownership_is_active(¤t, i128::from(now_unix_nanos)) {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
} else {
|
||||
let mut cleanup = current.clone();
|
||||
cleanup
|
||||
.mark_cleanup_pending(
|
||||
current.fence(),
|
||||
TransitionCleanupProof {
|
||||
transaction_id: current.transaction_id,
|
||||
write_id: current.write_id,
|
||||
remote_object: current.remote_object.clone(),
|
||||
remote_version: current.remote_version.clone(),
|
||||
backend_fingerprint: current.backend_fingerprint,
|
||||
decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
|
||||
},
|
||||
)
|
||||
.map_err(transition_transaction_store_error)?;
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pause_before_transition_recovery_claim(current.transaction_id).await;
|
||||
match save_transition_transaction_record_if_current(api.clone(), ¤t, &cleanup).await {
|
||||
Ok(()) => recover_cleanup_pending(api.clone(), &cleanup).await,
|
||||
Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
if transition_transaction_ownership_is_active(¤t, now_unix_nanos) {
|
||||
return Ok(TransitionTransactionRecoveryOutcome::Retained);
|
||||
}
|
||||
let mut cleanup = current.clone();
|
||||
cleanup
|
||||
.mark_cleanup_pending(
|
||||
current.fence(),
|
||||
TransitionCleanupProof {
|
||||
transaction_id: current.transaction_id,
|
||||
write_id: current.write_id,
|
||||
remote_object: current.remote_object.clone(),
|
||||
remote_version: current.remote_version.clone(),
|
||||
backend_fingerprint: current.backend_fingerprint,
|
||||
decision: TransitionCleanupDecision::UploadAbortedBeforeLocalCommit,
|
||||
},
|
||||
)
|
||||
.map_err(transition_transaction_store_error)?;
|
||||
#[cfg(test)]
|
||||
pause_before_transition_recovery_claim(current.transaction_id).await;
|
||||
match save_transition_transaction_record_if_current(api.clone(), ¤t, &cleanup).await {
|
||||
Ok(()) => recover_cleanup_pending(api, &cleanup).await,
|
||||
Err(Error::PreconditionFailed) | Err(Error::ConfigNotFound) => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
TransitionTransactionState::CleanupPending => recover_cleanup_pending(api.clone(), ¤t).await,
|
||||
TransitionTransactionState::CleanupPending => recover_cleanup_pending(api, ¤t).await,
|
||||
TransitionTransactionState::LocalCommitStarted => match local_commit_matches_transaction(api.clone(), ¤t).await {
|
||||
Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
|
||||
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
|
||||
IlmRecoveryErrorCode::LocalCommitAmbiguous,
|
||||
)),
|
||||
Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(
|
||||
IlmRecoveryErrorCode::LocalCommitAmbiguous,
|
||||
)),
|
||||
Ok(true) => {
|
||||
delete_transition_transaction_record(api, ¤t).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
Err(err) if transition_source_is_missing(&err) => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
Err(err) => Err(err),
|
||||
},
|
||||
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
|
||||
delete_transition_transaction_record(api, ¤t).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
TransitionTransactionState::UploadOutcomeUnknown => {
|
||||
if transition_transaction_ownership_is_active(¤t, i128::from(now_unix_nanos)) {
|
||||
if transition_transaction_ownership_is_active(¤t, now_unix_nanos) {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
} else {
|
||||
recover_unknown_upload_outcome(api.clone(), ¤t).await
|
||||
recover_unknown_upload_outcome(api, ¤t).await
|
||||
}
|
||||
}
|
||||
TransitionTransactionState::UploadStarted => {
|
||||
if transition_transaction_ownership_is_active(¤t, i128::from(now_unix_nanos)) {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
} else {
|
||||
Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
IlmRecoveryErrorCode::RemoteVersionUnknown,
|
||||
))
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(mut control) = recovery_control.take() {
|
||||
let source_to_delete = if matches!(
|
||||
recovery,
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted
|
||||
| TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
) {
|
||||
let refreshed =
|
||||
refresh_transition_recovery_control_source(api.clone(), control, &record_name, current.transaction_id).await?;
|
||||
control = refreshed.0;
|
||||
refreshed.1
|
||||
} else {
|
||||
None
|
||||
};
|
||||
persist_transition_recovery_result(api.clone(), control, &recovery, now_unix_nanos).await?;
|
||||
if let Some(source) = source_to_delete {
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pause_after_transition_recovery_terminal(source.transaction_id).await;
|
||||
delete_transition_transaction_record(api, &source).await?;
|
||||
}
|
||||
} else if matches!(
|
||||
recovery,
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
) {
|
||||
delete_transition_transaction_record(api, ¤t).await?;
|
||||
}
|
||||
recovery
|
||||
}
|
||||
|
||||
fn transition_recovery_control_identity(transaction: &TransitionTransaction, record_name: &str) -> IlmRecoveryControlIdentity {
|
||||
IlmRecoveryControlIdentity {
|
||||
protocol: IlmRecoveryProtocol::TransitionTransaction,
|
||||
canonical_source_path: record_name.to_string(),
|
||||
stable_operation_identity: transaction.transaction_id.to_string(),
|
||||
record_class: "transition_transaction_v1".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn transition_recovery_control_id(transaction: &TransitionTransaction) -> Result<String> {
|
||||
let record_name = transition_transaction_record_object_name(transaction.transaction_id)?;
|
||||
transition_recovery_control_identity(transaction, &record_name)
|
||||
.source_operation_digest()
|
||||
.map_err(|_| TransitionTransactionError::Corrupt("transition recovery control identity is invalid"))
|
||||
}
|
||||
|
||||
fn transition_state_needs_recovery_control(transaction: &TransitionTransaction, now_unix_nanos: i64) -> bool {
|
||||
now_unix_nanos >= transaction.not_after_unix_nanos
|
||||
&& !matches!(
|
||||
transaction.state,
|
||||
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed
|
||||
)
|
||||
}
|
||||
|
||||
async fn cleanup_terminal_transition_recovery_control(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
record_name: &str,
|
||||
identity: &IlmRecoveryControlIdentity,
|
||||
control_id: &str,
|
||||
) -> EcstoreResult<bool> {
|
||||
let observed = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
|
||||
Ok(observed) => observed,
|
||||
Err(Error::ConfigNotFound) => return Ok(false),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if observed.control.classification != IlmRecoveryClassification::Terminal {
|
||||
return Ok(false);
|
||||
}
|
||||
let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
|
||||
let exact_source = source.is_consistent()
|
||||
&& source.generation == observed.control.observed_source_generation
|
||||
&& source.canonical_data.as_deref().is_some_and(|data| {
|
||||
TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|decoded| decoded == *transaction)
|
||||
});
|
||||
if observed.control.identity != *identity || !exact_source {
|
||||
return Ok(false);
|
||||
}
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn claim_transition_recovery_control(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
record_name: &str,
|
||||
identity: IlmRecoveryControlIdentity,
|
||||
control_id: &str,
|
||||
now_unix_nanos: i64,
|
||||
) -> EcstoreResult<Option<ObservedIlmRecoveryControl>> {
|
||||
let existing = match load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await {
|
||||
Ok(control) => Some(control),
|
||||
Err(Error::ConfigNotFound) => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if let Some(observed) = existing.as_ref() {
|
||||
if observed.control.identity != identity {
|
||||
return Ok(None);
|
||||
}
|
||||
if observed
|
||||
.control
|
||||
.owner
|
||||
.as_ref()
|
||||
.is_some_and(|owner| owner.lease_expires_at_unix_nanos <= now_unix_nanos)
|
||||
{
|
||||
let mut expired = observed.control.clone();
|
||||
expired
|
||||
.record_expired_attempt(now_unix_nanos)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api, observed, &expired).await?;
|
||||
return Ok(None);
|
||||
}
|
||||
if !observed.control.should_attempt_at(now_unix_nanos) {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
let source = match observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await {
|
||||
Ok(source) => source,
|
||||
Err(err) => {
|
||||
if let Some(observed) = existing {
|
||||
persist_transition_recovery_source_failure(api, observed, now_unix_nanos).await?;
|
||||
return Ok(None);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let source_matches = source.is_consistent()
|
||||
&& source.canonical_data.as_deref().is_some_and(|data| {
|
||||
TransitionTransaction::decode(transaction.transaction_id, data).is_ok_and(|observed| observed == *transaction)
|
||||
});
|
||||
let source_error = if source_matches {
|
||||
IlmRecoveryErrorCode::None
|
||||
} else if source.canonical_data.is_some() {
|
||||
IlmRecoveryErrorCode::SourceGenerationChanged
|
||||
} else {
|
||||
IlmRecoveryErrorCode::SourceDivergent
|
||||
};
|
||||
|
||||
let mut observed = match existing {
|
||||
Some(control) => control,
|
||||
None => {
|
||||
let candidate = IlmRecoveryControl::new(
|
||||
identity.clone(),
|
||||
source.generation.clone(),
|
||||
if source_matches {
|
||||
IlmRecoveryClassification::Retrying
|
||||
} else {
|
||||
IlmRecoveryClassification::Corrupt
|
||||
},
|
||||
now_unix_nanos,
|
||||
source_error,
|
||||
)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
match save_recovery_control_if_absent(api.clone(), &candidate).await {
|
||||
Ok(()) | Err(Error::PreconditionFailed) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?
|
||||
}
|
||||
};
|
||||
if observed.control.identity != identity || !observed.control.should_attempt_at(now_unix_nanos) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut claimed = observed.control.clone();
|
||||
claimed
|
||||
.claim_for_source_generation(
|
||||
api.id.to_string(),
|
||||
Uuid::new_v4(),
|
||||
now_unix_nanos,
|
||||
TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
|
||||
source.generation,
|
||||
)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
|
||||
observed = load_recovery_control(api.clone(), IlmRecoveryProtocol::TransitionTransaction, control_id).await?;
|
||||
if observed.control != claimed {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
if !source_matches {
|
||||
let mut corrupt = observed.control.clone();
|
||||
corrupt
|
||||
.finish_attempt(IlmRecoveryClassification::Corrupt, source_error)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api, &observed, &corrupt).await?;
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(observed))
|
||||
}
|
||||
|
||||
async fn persist_transition_recovery_source_failure(
|
||||
api: Arc<ECStore>,
|
||||
observed: ObservedIlmRecoveryControl,
|
||||
now_unix_nanos: i64,
|
||||
) -> EcstoreResult<()> {
|
||||
let mut claimed = observed.control.clone();
|
||||
claimed
|
||||
.claim(
|
||||
api.id.to_string(),
|
||||
Uuid::new_v4(),
|
||||
now_unix_nanos,
|
||||
TRANSITION_RECOVERY_CONTROL_LEASE_NANOS,
|
||||
)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api.clone(), &observed, &claimed).await?;
|
||||
let claimed = load_recovery_control(
|
||||
api.clone(),
|
||||
IlmRecoveryProtocol::TransitionTransaction,
|
||||
&claimed
|
||||
.identity
|
||||
.source_operation_digest()
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
)
|
||||
.await?;
|
||||
let mut failed = claimed.control.clone();
|
||||
failed
|
||||
.record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceUnavailable)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api, &claimed, &failed).await
|
||||
}
|
||||
|
||||
async fn refresh_transition_recovery_control_source(
|
||||
api: Arc<ECStore>,
|
||||
mut observed: ObservedIlmRecoveryControl,
|
||||
record_name: &str,
|
||||
transaction_id: Uuid,
|
||||
) -> EcstoreResult<(ObservedIlmRecoveryControl, Option<TransitionTransaction>)> {
|
||||
let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
|
||||
Ok(transaction) => transaction,
|
||||
Err(Error::ConfigNotFound) => return Ok((observed, None)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let source = observe_recovery_source(api.clone(), record_name, TRANSITION_TRANSACTION_SCHEMA).await?;
|
||||
let exact_source = source.is_consistent()
|
||||
&& source
|
||||
.canonical_data
|
||||
.as_deref()
|
||||
.is_some_and(|data| TransitionTransaction::decode(transaction_id, data).is_ok_and(|decoded| decoded == transaction));
|
||||
if !exact_source {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
if observed.control.observed_source_generation != source.generation {
|
||||
let mut refreshed = observed.control.clone();
|
||||
refreshed
|
||||
.refresh_owned_source_generation(source.generation)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
save_recovery_control_if_current(api.clone(), &observed, &refreshed).await?;
|
||||
observed = load_recovery_control(
|
||||
api,
|
||||
IlmRecoveryProtocol::TransitionTransaction,
|
||||
&refreshed
|
||||
.identity
|
||||
.source_operation_digest()
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
)
|
||||
.await?;
|
||||
if observed.control != refreshed {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
Ok((observed, Some(transaction)))
|
||||
}
|
||||
|
||||
async fn persist_transition_recovery_result(
|
||||
api: Arc<ECStore>,
|
||||
observed: ObservedIlmRecoveryControl,
|
||||
recovery: &EcstoreResult<TransitionTransactionRecoveryOutcome>,
|
||||
now_unix_nanos: i64,
|
||||
) -> EcstoreResult<()> {
|
||||
let mut next = observed.control.clone();
|
||||
match recovery {
|
||||
Ok(
|
||||
TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted | TransitionTransactionRecoveryOutcome::RecordDeleted,
|
||||
) => next
|
||||
.finish_attempt(IlmRecoveryClassification::Terminal, IlmRecoveryErrorCode::None)
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained) => next
|
||||
.record_retryable_failure(now_unix_nanos, IlmRecoveryErrorCode::SourceGenerationChanged)
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(code)) => next
|
||||
.finish_attempt(IlmRecoveryClassification::RetainedAmbiguous, *code)
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
Ok(TransitionTransactionRecoveryOutcome::OperatorRequired(code)) => next
|
||||
.finish_attempt(IlmRecoveryClassification::OperatorRequired, *code)
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
Err(err) => next
|
||||
.record_retryable_failure(now_unix_nanos, transition_recovery_error_code(err))
|
||||
.map_err(|err| Error::other(err.to_string()))?,
|
||||
}
|
||||
save_recovery_control_if_current(api, &observed, &next).await
|
||||
}
|
||||
|
||||
fn transition_recovery_error_code(err: &Error) -> IlmRecoveryErrorCode {
|
||||
match err {
|
||||
Error::PreconditionFailed => IlmRecoveryErrorCode::CasConflict,
|
||||
Error::ConfigNotFound
|
||||
| Error::FileNotFound
|
||||
| Error::FileVersionNotFound
|
||||
| Error::ObjectNotFound(_, _)
|
||||
| Error::VersionNotFound(_, _, _)
|
||||
| Error::BucketNotFound(_) => IlmRecoveryErrorCode::SourceUnavailable,
|
||||
Error::SlowDown => IlmRecoveryErrorCode::BackendThrottled,
|
||||
_ => IlmRecoveryErrorCode::Unknown,
|
||||
TransitionTransactionState::UploadStarted => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1600,7 +1134,10 @@ async fn recover_cleanup_pending(
|
||||
transaction: &TransitionTransaction,
|
||||
) -> EcstoreResult<TransitionTransactionRecoveryOutcome> {
|
||||
match local_commit_matches_transaction(api.clone(), transaction).await {
|
||||
Ok(true) => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
|
||||
Ok(true) => {
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
Ok(false) => delete_unreferenced_transition_candidate(api, transaction).await,
|
||||
Err(err) if transition_source_is_missing(&err) => delete_unreferenced_transition_candidate(api, transaction).await,
|
||||
Err(err) => Err(err),
|
||||
@@ -1620,6 +1157,7 @@ async fn delete_unreferenced_transition_candidate(
|
||||
return Ok(TransitionTransactionRecoveryOutcome::Retained);
|
||||
}
|
||||
delete_transition_remote_candidate(api.clone(), ¤t).await?;
|
||||
delete_transition_transaction_record(api, ¤t).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
|
||||
@@ -1640,26 +1178,24 @@ async fn recover_unknown_upload_outcome(
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
TransitionCandidateProbe::Missing => Ok(TransitionTransactionRecoveryOutcome::RecordDeleted),
|
||||
TransitionCandidateProbe::Missing => {
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
TransitionCandidateProbe::UnversionedPresent => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::unversioned()).await
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id)
|
||||
if Uuid::parse_str(&version_id).is_ok_and(|version_id| version_id.is_nil()) =>
|
||||
{
|
||||
Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
IlmRecoveryErrorCode::RemoteVersionUnknown,
|
||||
))
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => {
|
||||
cleanup_recovered_unknown_upload_candidate(api, transaction, TransitionRemoteVersion::versioned(version_id)).await
|
||||
}
|
||||
TransitionCandidateProbe::Ambiguous => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
IlmRecoveryErrorCode::RemoteProbeAmbiguous,
|
||||
)),
|
||||
TransitionCandidateProbe::Unsupported => Ok(TransitionTransactionRecoveryOutcome::RetainedAmbiguous(
|
||||
IlmRecoveryErrorCode::RemoteProbeUnsupported,
|
||||
)),
|
||||
TransitionCandidateProbe::Ambiguous | TransitionCandidateProbe::Unsupported => {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1753,7 +1289,7 @@ pub async fn recover_transition_transaction_records(
|
||||
recover_transition_transaction_records_with_now(api, limit, marker, None).await
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn recover_transition_transaction_records_at(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
@@ -1787,11 +1323,6 @@ async fn recover_transition_transaction_records_with_now(
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
if list.is_truncated && list.next_continuation_token.is_none() {
|
||||
return Err(Error::other(
|
||||
"transition transaction recovery returned a truncated page without a continuation marker",
|
||||
));
|
||||
}
|
||||
|
||||
let mut stats = TransitionTransactionRecoveryStats {
|
||||
scanned: 0,
|
||||
@@ -1850,11 +1381,7 @@ async fn recover_transition_transaction_records_with_now(
|
||||
) => {
|
||||
stats.recovered += 1;
|
||||
}
|
||||
Ok(
|
||||
TransitionTransactionRecoveryOutcome::Retained
|
||||
| TransitionTransactionRecoveryOutcome::RetainedAmbiguous(_)
|
||||
| TransitionTransactionRecoveryOutcome::OperatorRequired(_),
|
||||
) => {
|
||||
Ok(TransitionTransactionRecoveryOutcome::Retained) => {
|
||||
stats.retained += 1;
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_TRANSITION_TRANSACTION_RECOVERY,
|
||||
@@ -1982,74 +1509,11 @@ fn state_requires_known_remote_version(state: TransitionTransactionState) -> boo
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use super::*;
|
||||
|
||||
const BACKEND_FINGERPRINT: [u8; 32] = [7; 32];
|
||||
|
||||
struct RecoveryAttemptDropGuard(Arc<AtomicBool>);
|
||||
|
||||
impl Drop for RecoveryAttemptDropGuard {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
async fn pending_recovery_attempt(started: Arc<tokio::sync::Notify>, dropped: Arc<AtomicBool>) -> EcstoreResult<()> {
|
||||
let _drop_guard = RecoveryAttemptDropGuard(dropped);
|
||||
started.notify_one();
|
||||
std::future::pending().await
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn transition_recovery_timeout_and_cancellation_drop_inflight_attempts() {
|
||||
let timeout_started = Arc::new(tokio::sync::Notify::new());
|
||||
let timeout_dropped = Arc::new(AtomicBool::new(false));
|
||||
let timeout_task = tokio::spawn({
|
||||
let started = Arc::clone(&timeout_started);
|
||||
let dropped = Arc::clone(&timeout_dropped);
|
||||
async move {
|
||||
await_transition_transaction_recovery(
|
||||
&CancellationToken::new(),
|
||||
TRANSITION_TRANSACTION_RECOVERY_TIMEOUT,
|
||||
pending_recovery_attempt(started, dropped),
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
timeout_started.notified().await;
|
||||
tokio::time::advance(TRANSITION_TRANSACTION_RECOVERY_TIMEOUT).await;
|
||||
let timed_out = timeout_task.await.expect("timeout wrapper task should join");
|
||||
assert!(matches!(timed_out, Some(Err(_))), "outer timeout should fail the recovery pass");
|
||||
assert!(timeout_dropped.load(Ordering::SeqCst), "outer timeout must drop its in-flight attempt");
|
||||
|
||||
let cancel_token = CancellationToken::new();
|
||||
let cancel_started = Arc::new(tokio::sync::Notify::new());
|
||||
let cancel_dropped = Arc::new(AtomicBool::new(false));
|
||||
let cancel_task = tokio::spawn({
|
||||
let cancel_token = cancel_token.clone();
|
||||
let started = Arc::clone(&cancel_started);
|
||||
let dropped = Arc::clone(&cancel_dropped);
|
||||
async move {
|
||||
await_transition_transaction_recovery(
|
||||
&cancel_token,
|
||||
TRANSITION_TRANSACTION_RECOVERY_TIMEOUT,
|
||||
pending_recovery_attempt(started, dropped),
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
cancel_started.notified().await;
|
||||
cancel_token.cancel();
|
||||
let cancelled = cancel_task.await.expect("cancellation wrapper task should join");
|
||||
assert!(cancelled.is_none(), "outer cancellation should stop the recovery loop");
|
||||
assert!(
|
||||
cancel_dropped.load(Ordering::SeqCst),
|
||||
"outer cancellation must drop its in-flight attempt"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct MemoryTransactionStore {
|
||||
records: HashMap<Uuid, Vec<u8>>,
|
||||
@@ -2504,19 +1968,5 @@ mod tests {
|
||||
transition_transaction_record_object_name(Uuid::nil()),
|
||||
Err(TransitionTransactionError::Corrupt("transaction_id is nil"))
|
||||
));
|
||||
assert_eq!(
|
||||
transition_transaction_id_from_record_object_name(&object).expect("canonical record path should parse"),
|
||||
transaction_id
|
||||
);
|
||||
for malformed in [
|
||||
object.to_ascii_uppercase(),
|
||||
object.replace("/aa/aa/", "/ff/aa/"),
|
||||
object.replace("/aa/aa/", "/aa/aa/extra/"),
|
||||
] {
|
||||
assert!(matches!(
|
||||
transition_transaction_id_from_record_object_name(&malformed),
|
||||
Err(TransitionTransactionError::Corrupt(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -489,17 +489,28 @@ impl BucketMetadata {
|
||||
!self.bucket_targets_config_json.is_empty() && self.bucket_target_config.is_none()
|
||||
}
|
||||
|
||||
/// Opaque application-owned configuration with its persisted update time.
|
||||
/// Empty bytes mean absent or cleared; decoding belongs to the consumer.
|
||||
pub fn on_demand_migration_config(&self) -> Option<(&[u8], OffsetDateTime)> {
|
||||
(!self.on_demand_migration_config_json.is_empty()).then_some((
|
||||
self.on_demand_migration_config_json.as_slice(),
|
||||
self.on_demand_migration_config_updated_at,
|
||||
))
|
||||
/// Parsed per-bucket durability override, if a valid one is stored.
|
||||
///
|
||||
/// Absent/empty/unparsable payloads all mean "no override" (the bucket
|
||||
/// follows the global durability mode); a parse failure is logged so a
|
||||
/// corrupted entry cannot silently change fsync behavior.
|
||||
/// Parsed on-demand migration config, if one is stored.
|
||||
///
|
||||
/// `Ok(None)` means no config (absent or cleared). A stored payload that
|
||||
/// does not parse is an error, never a default: the runtime must not
|
||||
/// pull from a source it cannot describe.
|
||||
pub fn on_demand_migration_config(
|
||||
&self,
|
||||
) -> std::result::Result<
|
||||
Option<super::on_demand_migration::OnDemandMigrationConfig>,
|
||||
super::on_demand_migration::OnDemandMigrationConfigError,
|
||||
> {
|
||||
if self.on_demand_migration_config_json.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
super::on_demand_migration::OnDemandMigrationConfig::from_json(&self.on_demand_migration_config_json).map(Some)
|
||||
}
|
||||
|
||||
/// Parsed per-bucket durability override, if a valid one is stored.
|
||||
/// Invalid payloads follow the global mode after logging a parse failure.
|
||||
pub fn durability_config(&self) -> Option<super::durability::BucketDurabilityConfig> {
|
||||
if self.durability_config_json.is_empty() {
|
||||
return None;
|
||||
@@ -905,6 +916,13 @@ impl BucketMetadata {
|
||||
self.durability_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_ON_DEMAND_MIGRATION_CONFIG => {
|
||||
// Structural check only (shape, unknown fields); the
|
||||
// deployment-relative rules run in the admin handler with a
|
||||
// `ValidationContext`. A blob this build cannot read must not
|
||||
// be persisted for every later reader to trip over.
|
||||
if !data.is_empty() {
|
||||
super::on_demand_migration::OnDemandMigrationConfig::from_json(&data).map_err(Error::other)?;
|
||||
}
|
||||
self.on_demand_migration_config_json = data;
|
||||
self.on_demand_migration_config_updated_at = updated;
|
||||
}
|
||||
@@ -1611,34 +1629,6 @@ mod test {
|
||||
assert!(bm.bucket_target_config.is_none());
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2309: the MinIO-origin `.metadata.bin` this repository
|
||||
/// already carries as a compatibility fixture stores
|
||||
/// `BucketTargetsConfigJSON` as a bare JSON array, which `BucketTargets`
|
||||
/// (a `{"targets":[…]}` struct with no array fallback) cannot decode. The
|
||||
/// bytes below are the exact payload the fixture in
|
||||
/// `metadata_test.rs::TEST_BUCKET_METADATA_HEX` decodes to, so if RustFS
|
||||
/// ever grows the array-shaped compatibility parse, this test is where the
|
||||
/// upgrade break is pinned and where the decision has to be recorded.
|
||||
#[test]
|
||||
fn minio_array_shaped_bucket_targets_are_unreadable() {
|
||||
let minio_array = br#"[{"endpoint":"http://target.example.com","targetBucket":"tb","region":"us-east-1"}]"#.to_vec();
|
||||
let mut bm = BucketMetadata::new("minio-array-targets");
|
||||
bm.bucket_targets_config_json = minio_array.clone();
|
||||
|
||||
bm.parse_all_configs()
|
||||
.expect("a MinIO-shaped targets blob must not fail the whole metadata load");
|
||||
|
||||
assert!(
|
||||
bm.bucket_targets_unreadable(),
|
||||
"an array-shaped MinIO targets blob is unreadable, not an empty target set"
|
||||
);
|
||||
assert!(bm.bucket_target_config.is_none());
|
||||
assert_eq!(
|
||||
bm.bucket_targets_config_json, minio_array,
|
||||
"the raw MinIO bytes must survive so the configuration stays recoverable"
|
||||
);
|
||||
}
|
||||
|
||||
/// The invariant every branch of `parse_all_configs` shares: a stored but
|
||||
/// undecodable payload keeps its raw bytes and leaves the typed field
|
||||
/// `None`, so no branch fabricates a value. What a reader may then do with
|
||||
@@ -1988,30 +1978,51 @@ mod test {
|
||||
|
||||
const ODM_JSON: &[u8] = br#"{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
|
||||
|
||||
/// The metadata codec preserves application-owned bytes and timestamps.
|
||||
/// rustfs/backlog#2148: the on-demand migration config is a RustFS
|
||||
/// extension entry that round-trips through `update_config` and the
|
||||
/// msgpack codec, clears on delete, and never parses corruption into a
|
||||
/// default.
|
||||
#[test]
|
||||
fn on_demand_migration_config_round_trips_and_tracks_updates() {
|
||||
use crate::bucket::on_demand_migration::{OnDemandMigrationConfig, OnDemandMigrationConfigError};
|
||||
|
||||
let mut bm = BucketMetadata::new("odm-bucket");
|
||||
assert_eq!(bm.on_demand_migration_config(), None, "fresh metadata carries no config");
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(None), "fresh metadata carries no config");
|
||||
|
||||
let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.expect("opaque config is accepted");
|
||||
let stamped = bm.on_demand_migration_config_updated_at;
|
||||
assert_ne!(stamped, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(bm.on_demand_migration_config(), Some((ODM_JSON, stamped)));
|
||||
.expect("valid config is accepted");
|
||||
assert_ne!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(Some(expected.clone())));
|
||||
|
||||
let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
|
||||
assert_eq!(back.on_demand_migration_config_json, bm.on_demand_migration_config_json);
|
||||
assert_eq!(back.on_demand_migration_config_updated_at.unix_timestamp(), stamped.unix_timestamp());
|
||||
assert_eq!(
|
||||
back.on_demand_migration_config_updated_at.unix_timestamp(),
|
||||
bm.on_demand_migration_config_updated_at.unix_timestamp()
|
||||
);
|
||||
assert_eq!(back.on_demand_migration_config(), Ok(Some(expected)));
|
||||
|
||||
// A blob this build cannot read is rejected at the write boundary
|
||||
// rather than persisted for every reader to trip over.
|
||||
let before = bm.on_demand_migration_config_json.clone();
|
||||
assert!(
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec())
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(bm.on_demand_migration_config_json, before, "a rejected update leaves the blob untouched");
|
||||
|
||||
// Delete clears the entry.
|
||||
let stamped = bm.on_demand_migration_config_updated_at;
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, Vec::new()).unwrap();
|
||||
assert!(bm.on_demand_migration_config_json.is_empty());
|
||||
assert_eq!(bm.on_demand_migration_config(), None);
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(None));
|
||||
assert!(bm.on_demand_migration_config_updated_at >= stamped);
|
||||
bm.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, b"not-json".to_vec())
|
||||
.unwrap();
|
||||
let back = BucketMetadata::unmarshal(&bm.marshal_msg().unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
back.on_demand_migration_config_json, b"not-json",
|
||||
"metadata must not reinterpret application bytes"
|
||||
);
|
||||
|
||||
// Corruption that bypassed `update_config` (disk, another writer)
|
||||
// is a typed error, never a default.
|
||||
bm.on_demand_migration_config_json = b"not-json".to_vec();
|
||||
assert!(matches!(bm.on_demand_migration_config(), Err(OnDemandMigrationConfigError::Malformed(_))));
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2148: a `.metadata.bin` written before the on-demand
|
||||
@@ -2023,7 +2034,7 @@ mod test {
|
||||
let mut bm = BucketMetadata::unmarshal(&blob[4..]).expect("unmarshal MinIO bucket metadata");
|
||||
assert!(bm.on_demand_migration_config_json.is_empty());
|
||||
assert_eq!(bm.on_demand_migration_config_updated_at, OffsetDateTime::UNIX_EPOCH);
|
||||
assert_eq!(bm.on_demand_migration_config(), None);
|
||||
assert_eq!(bm.on_demand_migration_config(), Ok(None));
|
||||
|
||||
bm.default_timestamps();
|
||||
assert_ne!(bm.created, OffsetDateTime::UNIX_EPOCH, "fixture must carry a real creation time");
|
||||
|
||||
@@ -19,6 +19,7 @@ use super::quota::BucketQuota;
|
||||
use super::target::BucketTargets;
|
||||
use crate::bucket::bucket_target_sys::BucketTargetSys;
|
||||
use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_parse_with_presence};
|
||||
use crate::bucket::on_demand_migration::{ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig};
|
||||
use crate::bucket::utils::is_meta_bucketname;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
|
||||
@@ -48,11 +49,6 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{error, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Opaque bucket configuration notifications for application-owned services.
|
||||
/// `None` withdraws a configuration; consumers validate nonempty bytes.
|
||||
pub type BucketConfigPublishHook = Box<dyn Fn(&str, &str, Option<(&[u8], OffsetDateTime, Uuid)>) + Send + Sync>;
|
||||
pub static BUCKET_CONFIG_PUBLISH_HOOK: std::sync::OnceLock<BucketConfigPublishHook> = std::sync::OnceLock::new();
|
||||
|
||||
const BUCKET_METADATA_REFRESH_INTERVAL: Duration = Duration::from_secs(15 * 60);
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
@@ -399,21 +395,39 @@ fn clear_bucket_durability(bucket: &str) {
|
||||
crate::disk::local::bucket_durability::set(bucket, None);
|
||||
}
|
||||
|
||||
/// Publish application-owned bytes on every cache install path.
|
||||
/// Publish the bucket's on-demand migration config (or its absence) to the
|
||||
/// runtime registered in `ON_DEMAND_MIGRATION_CONFIG_HOOK`.
|
||||
///
|
||||
/// Called from the same five cache-install paths as
|
||||
/// [`sync_bucket_durability`]. A stored payload this build cannot parse is
|
||||
/// published as `None`: the runtime must stop pulling for that bucket rather
|
||||
/// than keep an older config or guess.
|
||||
fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
|
||||
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
|
||||
hook(
|
||||
bucket,
|
||||
super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG,
|
||||
bm.on_demand_migration_config()
|
||||
.map(|(bytes, stamp)| (bytes, stamp, bm.bucket_incarnation_id)),
|
||||
);
|
||||
let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() else {
|
||||
return;
|
||||
};
|
||||
match bm.on_demand_migration_config() {
|
||||
Ok(config) => hook(bucket, config.as_ref()),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = "bucket_metadata_parse_failed",
|
||||
component = "ecstore",
|
||||
subsystem = "bucket_metadata",
|
||||
bucket = %bucket,
|
||||
config = "on_demand_migration",
|
||||
error = %err,
|
||||
"Failed to parse bucket metadata config"
|
||||
);
|
||||
hook(bucket, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Withdraw a bucket's on-demand migration config when its metadata leaves
|
||||
/// the cache.
|
||||
fn clear_on_demand_migration(bucket: &str) {
|
||||
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
|
||||
hook(bucket, super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, None);
|
||||
if let Some(hook) = ON_DEMAND_MIGRATION_CONFIG_HOOK.get() {
|
||||
hook(bucket, None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1035,21 +1049,15 @@ pub async fn get_durability_config(
|
||||
}
|
||||
|
||||
/// The bucket's on-demand migration config with its update time, or
|
||||
/// `Ok(None)` when the bucket has none. Bytes are opaque to the metadata owner.
|
||||
pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
|
||||
/// `Ok(None)` when the bucket has none. A stored payload that does not parse
|
||||
/// is a typed error (`OnDemandMigrationConfigError` inside `Error::Io`).
|
||||
pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
|
||||
bucket_meta_sys.get_on_demand_migration_config(bucket).await
|
||||
}
|
||||
|
||||
/// Resolve opaque configuration from the store's own metadata system.
|
||||
pub async fn get_on_demand_migration_config_in(api: &ECStore, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
|
||||
let sys = bucket_metadata_sys_of(&api.ctx)?;
|
||||
let lock = sys.read().await;
|
||||
lock.get_on_demand_migration_config(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -2571,27 +2579,29 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
/// See [`get_on_demand_migration_config`].
|
||||
pub async fn get_on_demand_migration_config(&self, bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
|
||||
pub async fn get_on_demand_migration_config(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<Option<(OnDemandMigrationConfig, OffsetDateTime)>> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
Ok(bm
|
||||
.on_demand_migration_config()
|
||||
.map(|(bytes, updated_at)| (bytes.to_vec(), updated_at)))
|
||||
let config = bm.on_demand_migration_config().map_err(Error::other)?;
|
||||
Ok(config.map(|config| (config, bm.on_demand_migration_config_updated_at)))
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only fixture shared with sibling modules (e.g. the quota checker
|
||||
/// tests): a 4-disk `ECStore` on an isolated instance context, so tests
|
||||
/// exercising the metadata system never touch ambient process state.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub mod test_support {
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support {
|
||||
use super::*;
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::store::init_local_disks_with_instance_ctx;
|
||||
|
||||
pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
pub(crate) async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
let mut dirs = Vec::with_capacity(4);
|
||||
let mut endpoints = Vec::with_capacity(4);
|
||||
for disk_idx in 0..4 {
|
||||
@@ -4375,26 +4385,19 @@ mod tests {
|
||||
|
||||
const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
|
||||
|
||||
type RecordedOdmConfig = Option<(Vec<u8>, OffsetDateTime, Uuid)>;
|
||||
type RecordedOdmHookCall = (String, RecordedOdmConfig);
|
||||
|
||||
/// Every `(bucket, config)` the recording hook has seen. Tests filter by
|
||||
/// their own bucket name; the hook is process-wide and set once.
|
||||
static ODM_HOOK_CALLS: std::sync::Mutex<Vec<RecordedOdmHookCall>> = std::sync::Mutex::new(Vec::new());
|
||||
static ODM_HOOK_CALLS: std::sync::Mutex<Vec<(String, Option<OnDemandMigrationConfig>)>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
fn install_recording_odm_hook() {
|
||||
BUCKET_CONFIG_PUBLISH_HOOK.get_or_init(|| {
|
||||
Box::new(|bucket, config_file, config| {
|
||||
assert_eq!(config_file, super::super::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG);
|
||||
ODM_HOOK_CALLS.lock().unwrap().push((
|
||||
bucket.to_string(),
|
||||
config.map(|(bytes, stamp, incarnation)| (bytes.to_vec(), stamp, incarnation)),
|
||||
));
|
||||
ON_DEMAND_MIGRATION_CONFIG_HOOK.get_or_init(|| {
|
||||
Box::new(|bucket, config| {
|
||||
ODM_HOOK_CALLS.lock().unwrap().push((bucket.to_string(), config.cloned()));
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn odm_hook_calls(bucket: &str) -> Vec<RecordedOdmConfig> {
|
||||
fn odm_hook_calls(bucket: &str) -> Vec<Option<OnDemandMigrationConfig>> {
|
||||
ODM_HOOK_CALLS
|
||||
.lock()
|
||||
.unwrap()
|
||||
@@ -4404,6 +4407,54 @@ mod tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
|
||||
/// stored payload it cannot parse as a typed error, never as a default
|
||||
/// and never as `ConfigNotFound`.
|
||||
#[tokio::test]
|
||||
async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
|
||||
use crate::bucket::on_demand_migration::OnDemandMigrationConfigError;
|
||||
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "odm-accessor";
|
||||
|
||||
sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
|
||||
assert_eq!(sys.get_on_demand_migration_config(bucket).await.unwrap(), None);
|
||||
|
||||
let mut corrupt = BucketMetadata::new(bucket);
|
||||
corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
|
||||
sys.set(bucket.to_string(), Arc::new(corrupt)).await;
|
||||
let err = sys
|
||||
.get_on_demand_migration_config(bucket)
|
||||
.await
|
||||
.expect_err("corrupt config must not read as a default");
|
||||
assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
|
||||
let typed = match &err {
|
||||
Error::Io(io) => io
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
|
||||
_ => None,
|
||||
};
|
||||
assert!(
|
||||
matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
|
||||
"typed parse error must survive the Result boundary, got: {err:?}"
|
||||
);
|
||||
|
||||
let mut valid = BucketMetadata::new(bucket);
|
||||
valid
|
||||
.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.unwrap();
|
||||
let stamped = valid.on_demand_migration_config_updated_at;
|
||||
sys.set(bucket.to_string(), Arc::new(valid)).await;
|
||||
let (config, updated_at) = sys
|
||||
.get_on_demand_migration_config(bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("stored config is returned");
|
||||
assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
|
||||
assert_eq!(updated_at, stamped);
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2148: the publish hook fires on every path that
|
||||
/// installs bucket metadata into the cache (set, initial load, peer
|
||||
/// reload, refresh loop, lazy load) and withdraws on removal, mirroring
|
||||
@@ -4417,22 +4468,15 @@ mod tests {
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("physical bucket should exist");
|
||||
}
|
||||
|
||||
let incarnation = Uuid::new_v4();
|
||||
let expected = OnDemandMigrationConfig::from_json(ODM_JSON).unwrap();
|
||||
let expect_publish = |before: usize, label: &str| {
|
||||
let calls = odm_hook_calls(bucket);
|
||||
assert_eq!(calls.len(), before + 1, "{label} must publish exactly once");
|
||||
assert_eq!(
|
||||
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
|
||||
Some(ODM_JSON),
|
||||
"{label} must publish the stored bytes"
|
||||
);
|
||||
assert_eq!(calls.last().unwrap().as_ref().map(|(_, _, id)| *id), Some(incarnation));
|
||||
assert_eq!(calls.last().unwrap().as_ref(), Some(&expected), "{label} must publish the stored config");
|
||||
};
|
||||
|
||||
// set (via persist_new_and_set, which installs through `set`).
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.bucket_incarnation_id = incarnation;
|
||||
bm.update_config(crate::bucket::metadata::BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.unwrap();
|
||||
let writer = BucketMetadataSys::new(ecstore.clone());
|
||||
@@ -4474,18 +4518,14 @@ mod tests {
|
||||
assert_eq!(calls.len(), before + 1, "remove must withdraw exactly once");
|
||||
assert_eq!(calls.last().unwrap(), &None);
|
||||
|
||||
// Opaque bytes reach the application even if they are not valid JSON.
|
||||
// A corrupt payload is withdrawn, never published as a config.
|
||||
let mut corrupt = BucketMetadata::new(bucket);
|
||||
corrupt.on_demand_migration_config_json = b"not-json".to_vec();
|
||||
let before = odm_hook_calls(bucket).len();
|
||||
lazy.set(bucket.to_string(), Arc::new(corrupt)).await;
|
||||
let calls = odm_hook_calls(bucket);
|
||||
assert_eq!(calls.len(), before + 1);
|
||||
assert_eq!(
|
||||
calls.last().unwrap().as_ref().map(|(bytes, _, _)| bytes.as_slice()),
|
||||
Some(b"not-json".as_slice()),
|
||||
"the application validates opaque config bytes"
|
||||
);
|
||||
assert_eq!(calls.last().unwrap(), &None, "unreadable config must publish absence");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -26,6 +26,7 @@ mod metadata_test;
|
||||
pub mod migration;
|
||||
mod msgp_decode;
|
||||
pub mod object_lock;
|
||||
pub mod on_demand_migration;
|
||||
pub mod policy_sys;
|
||||
pub mod quota;
|
||||
pub mod remote_s3_client;
|
||||
|
||||
+84
-377
@@ -25,8 +25,8 @@
|
||||
//! [`BACKFILL_SAVE_INTERVAL`], and at every page end, with an `If-Match`
|
||||
//! compare-and-set so a concurrent cancel or takeover is never overwritten.
|
||||
//! - The `continuation_token` only advances once every pull queued from the
|
||||
//! page before it has succeeded. After a failure it stays at that page,
|
||||
//! so crash recovery cannot skip failed pulls (existing keys are skipped).
|
||||
//! page before it has reported back, so a crash re-lists at most one page
|
||||
//! (already-present keys are then skipped, never re-pulled).
|
||||
//! - The owner holds a lease of [`BACKFILL_LEASE`] renewed by every save. The
|
||||
//! recovery loop ([`run_backfill_recovery_loop`]) scans the buckets this
|
||||
//! node has an ODM state for every [`BACKFILL_RECOVERY_INTERVAL`] and takes
|
||||
@@ -45,12 +45,16 @@
|
||||
|
||||
use super::pull::{EnqueueOutcome, PullReason, QueuedPullOutcome};
|
||||
use super::source_client::{SourceError, SourcePage};
|
||||
use super::storage_api::{
|
||||
BUCKET_META_PREFIX, ECStore, HTTPPreconditions, NamespaceLocking as _, ObjectOperations as _, ObjectOptions,
|
||||
RUSTFS_META_BUCKET, StorageError, WriteCompletion, get_lock_acquire_timeout, get_on_demand_migration_config_in,
|
||||
local_node_name, read_config_with_metadata, save_config_with_opts,
|
||||
};
|
||||
use super::sys::{BucketOdmState, OnDemandMigrationSys};
|
||||
use crate::bucket::metadata_sys::bucket_metadata_sys_of;
|
||||
use crate::config::com::{read_config_with_metadata, save_config_with_opts};
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use crate::error::Error as StorageError;
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::runtime::sources::local_node_name;
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::{namespace::NamespaceLocking as _, object::HTTPPreconditions, object::ObjectOperations as _};
|
||||
use crate::store::ECStore;
|
||||
use async_trait::async_trait;
|
||||
use futures::StreamExt;
|
||||
use futures::stream::FuturesUnordered;
|
||||
@@ -363,15 +367,14 @@ pub struct LocalBackfillObject {
|
||||
pub source_etag: Option<String>,
|
||||
}
|
||||
|
||||
/// Shared report of a new or coalesced pull; absent only when not admitted.
|
||||
pub type PullReport = Option<super::pull::QueuedPullReport>;
|
||||
/// Receiver of one queued pull's report; `None` when the pull was coalesced
|
||||
/// into one already running.
|
||||
pub type PullReport = Option<oneshot::Receiver<QueuedPullOutcome>>;
|
||||
|
||||
/// Everything the job needs from its bucket, so the loop can run against a
|
||||
/// mock in unit tests. Production: [`BucketBackfillContext`].
|
||||
#[async_trait]
|
||||
pub trait BackfillContext: Send + Sync {
|
||||
/// The bucket incarnation captured by this context.
|
||||
fn incarnation_id(&self) -> Uuid;
|
||||
/// One source page in the local key namespace.
|
||||
async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result<SourcePage, SourceError>;
|
||||
/// Whether the breaker admits source traffic right now.
|
||||
@@ -413,10 +416,6 @@ impl BucketBackfillContext {
|
||||
|
||||
#[async_trait]
|
||||
impl BackfillContext for BucketBackfillContext {
|
||||
fn incarnation_id(&self) -> Uuid {
|
||||
self.state.incarnation_id()
|
||||
}
|
||||
|
||||
async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result<SourcePage, SourceError> {
|
||||
let client = self.state.client().map_err(|err| SourceError::Unsupported(err.to_string()))?;
|
||||
let started = Instant::now();
|
||||
@@ -476,10 +475,12 @@ impl BackfillContext for BucketBackfillContext {
|
||||
}
|
||||
|
||||
async fn config_updated_at(&self) -> Result<Option<OffsetDateTime>, StorageError> {
|
||||
Ok(
|
||||
super::config::decode_stored_config(get_on_demand_migration_config_in(&self.api, self.state.bucket()).await?)?
|
||||
.map(|(_, updated_at)| updated_at),
|
||||
)
|
||||
let sys = bucket_metadata_sys_of(&self.api.ctx)?;
|
||||
let guard = sys.read().await;
|
||||
Ok(guard
|
||||
.get_on_demand_migration_config(self.state.bucket())
|
||||
.await?
|
||||
.map(|(_, updated_at)| updated_at))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -667,36 +668,8 @@ pub async fn read_checkpoint(api: &Arc<ECStore>, bucket: &str) -> Result<Option<
|
||||
async fn write_checkpoint(
|
||||
api: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
incarnation_id: Uuid,
|
||||
checkpoint: &BackfillCheckpoint,
|
||||
expected_etag: Option<&str>,
|
||||
) -> Result<String, BackfillError> {
|
||||
let api = Arc::clone(api);
|
||||
let bucket = bucket.to_string();
|
||||
let checkpoint = checkpoint.clone();
|
||||
let expected_etag = expected_etag.map(str::to_string);
|
||||
// The storage commit owns detached work. Keep its user-bucket fence alive
|
||||
// even when a caller aborts its waiter before the erasure tail has drained.
|
||||
tokio::spawn(async move {
|
||||
let fence = api.acquire_bucket_incarnation_fence(&bucket, incarnation_id).await?;
|
||||
let mut opts = ObjectOptions::default();
|
||||
fence.attach_to_object_options(&mut opts);
|
||||
let result = write_checkpoint_while_fenced(&api, &bucket, &checkpoint, expected_etag.as_deref(), opts).await;
|
||||
drop(fence);
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(|err| StorageError::other(format!("backfill checkpoint task failed: {err}")))?
|
||||
}
|
||||
|
||||
/// The caller holds the destination bucket's lifecycle fence through the CAS
|
||||
/// write and its read-back, including the drained erasure write tail.
|
||||
async fn write_checkpoint_while_fenced(
|
||||
api: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
checkpoint: &BackfillCheckpoint,
|
||||
expected_etag: Option<&str>,
|
||||
mut opts: ObjectOptions,
|
||||
) -> Result<String, BackfillError> {
|
||||
let data = checkpoint.to_json()?;
|
||||
let preconditions = match expected_etag {
|
||||
@@ -709,9 +682,12 @@ async fn write_checkpoint_while_fenced(
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
opts.max_parity = true;
|
||||
opts.write_completion = WriteCompletion::TailDrained;
|
||||
opts.http_preconditions = Some(preconditions);
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
http_preconditions: Some(preconditions),
|
||||
..Default::default()
|
||||
};
|
||||
match save_config_with_opts(Arc::clone(api), &checkpoint_path(bucket), data, &opts).await {
|
||||
Ok(()) => {}
|
||||
Err(StorageError::PreconditionFailed) => return Err(BackfillError::Conflict(bucket.to_string())),
|
||||
@@ -882,14 +858,7 @@ impl BackfillRunner {
|
||||
});
|
||||
}
|
||||
let checkpoint = BackfillCheckpoint::new(&request, config_updated_at, &self.node, now);
|
||||
let etag = write_checkpoint(
|
||||
&self.api,
|
||||
bucket,
|
||||
context.incarnation_id(),
|
||||
&checkpoint,
|
||||
stored.as_ref().map(|s| s.etag.as_str()),
|
||||
)
|
||||
.await?;
|
||||
let etag = write_checkpoint(&self.api, bucket, &checkpoint, stored.as_ref().map(|s| s.etag.as_str())).await?;
|
||||
info!(
|
||||
event = EVENT_ODM_BACKFILL_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -924,42 +893,30 @@ impl BackfillRunner {
|
||||
}
|
||||
return Ok(handle.snapshot.lock().clone());
|
||||
}
|
||||
let incarnation_id = self.api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let lock = self.lease_lock(bucket, get_lock_acquire_timeout()).await?;
|
||||
let api = Arc::clone(&self.api);
|
||||
let bucket = bucket.to_string();
|
||||
tokio::spawn(async move {
|
||||
let _lock = lock;
|
||||
let fence = api.acquire_bucket_incarnation_fence(&bucket, incarnation_id).await?;
|
||||
let mut opts = ObjectOptions::default();
|
||||
fence.attach_to_object_options(&mut opts);
|
||||
let Some(stored) = read_checkpoint(&api, &bucket).await? else {
|
||||
return Err(BackfillError::NotFound(bucket.to_string()));
|
||||
};
|
||||
if !stored.checkpoint.state.is_active() {
|
||||
return Ok(stored.checkpoint);
|
||||
}
|
||||
let mut checkpoint = stored.checkpoint;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
checkpoint.state = BackfillState::Cancelled;
|
||||
checkpoint.updated_at = now;
|
||||
write_checkpoint_while_fenced(&api, &bucket, &checkpoint, Some(&stored.etag), opts).await?;
|
||||
info!(
|
||||
event = EVENT_ODM_BACKFILL_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION,
|
||||
state = checkpoint.state.as_str(),
|
||||
result = "cancelled",
|
||||
bucket = %bucket,
|
||||
job_id = %checkpoint.job_id,
|
||||
owner = %checkpoint.owner.as_ref().map(|o| o.node.as_str()).unwrap_or_default(),
|
||||
"On-demand migration backfill job cancelled remotely"
|
||||
);
|
||||
drop(fence);
|
||||
Ok(checkpoint)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| StorageError::other(format!("backfill cancellation task failed: {err}")))?
|
||||
let _lock = self.lease_lock(bucket, get_lock_acquire_timeout()).await?;
|
||||
let Some(stored) = read_checkpoint(&self.api, bucket).await? else {
|
||||
return Err(BackfillError::NotFound(bucket.to_string()));
|
||||
};
|
||||
if !stored.checkpoint.state.is_active() {
|
||||
return Ok(stored.checkpoint);
|
||||
}
|
||||
let mut checkpoint = stored.checkpoint;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
checkpoint.state = BackfillState::Cancelled;
|
||||
checkpoint.updated_at = now;
|
||||
write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?;
|
||||
info!(
|
||||
event = EVENT_ODM_BACKFILL_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION,
|
||||
state = checkpoint.state.as_str(),
|
||||
result = "cancelled",
|
||||
bucket = %bucket,
|
||||
job_id = %checkpoint.job_id,
|
||||
owner = %checkpoint.owner.as_ref().map(|o| o.node.as_str()).unwrap_or_default(),
|
||||
"On-demand migration backfill job cancelled remotely"
|
||||
);
|
||||
Ok(checkpoint)
|
||||
}
|
||||
|
||||
/// Latest checkpoint: the in-memory progress of a local job, else the
|
||||
@@ -1052,7 +1009,7 @@ impl BackfillRunner {
|
||||
checkpoint.state = BackfillState::Cancelled;
|
||||
checkpoint.updated_at = now;
|
||||
checkpoint.record_failure("config_changed", None, now);
|
||||
write_checkpoint(&self.api, bucket, context.incarnation_id(), &checkpoint, Some(&stored.etag)).await?;
|
||||
write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?;
|
||||
info!(
|
||||
event = EVENT_ODM_BACKFILL_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -1071,7 +1028,7 @@ impl BackfillRunner {
|
||||
node: self.node.clone(),
|
||||
lease_until: now + BACKFILL_LEASE,
|
||||
});
|
||||
let etag = write_checkpoint(&self.api, bucket, context.incarnation_id(), &checkpoint, Some(&stored.etag)).await?;
|
||||
let etag = write_checkpoint(&self.api, bucket, &checkpoint, Some(&stored.etag)).await?;
|
||||
warn!(
|
||||
event = EVENT_ODM_BACKFILL_LEASE_TAKEOVER,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -1234,11 +1191,9 @@ impl Job {
|
||||
}
|
||||
|
||||
async fn main_loop(&mut self) -> Result<(), Stop> {
|
||||
let mut cursor = self.checkpoint.continuation_token.clone();
|
||||
let failed_at_resume = self.checkpoint.failed;
|
||||
loop {
|
||||
self.check_cancel()?;
|
||||
let page = self.list_page(cursor.as_deref()).await?;
|
||||
let page = self.list_page().await?;
|
||||
for object in &page.objects {
|
||||
self.check_cancel()?;
|
||||
self.checkpoint.listed += 1;
|
||||
@@ -1250,13 +1205,10 @@ impl Job {
|
||||
self.drain_ready();
|
||||
self.tick(false).await?;
|
||||
}
|
||||
// A persisted cursor certifies successful work, not just listing
|
||||
// progress. Keep it at the first failed page for crash recovery.
|
||||
// Only advance the cursor once every pull of this page reported
|
||||
// back, so a takeover re-lists at most this page.
|
||||
self.drain_all().await?;
|
||||
cursor = page.next_continuation_token;
|
||||
if self.checkpoint.failed == failed_at_resume {
|
||||
self.checkpoint.continuation_token = cursor.clone();
|
||||
}
|
||||
self.checkpoint.continuation_token = page.next_continuation_token.clone();
|
||||
self.tick(true).await?;
|
||||
if !page.is_truncated {
|
||||
return Ok(());
|
||||
@@ -1271,7 +1223,7 @@ impl Job {
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_page(&mut self, cursor: Option<&str>) -> Result<SourcePage, Stop> {
|
||||
async fn list_page(&mut self) -> Result<SourcePage, Stop> {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
while !self.context.source_available() {
|
||||
@@ -1279,7 +1231,7 @@ impl Job {
|
||||
self.tick(false).await?;
|
||||
}
|
||||
let prefix = self.checkpoint.prefix.clone();
|
||||
let token = cursor.map(str::to_string);
|
||||
let token = self.checkpoint.continuation_token.clone();
|
||||
match self
|
||||
.context
|
||||
.list_page(prefix.as_deref(), token.as_deref(), BACKFILL_LIST_PAGE_SIZE)
|
||||
@@ -1353,10 +1305,9 @@ impl Job {
|
||||
}
|
||||
loop {
|
||||
match self.context.enqueue(key) {
|
||||
(EnqueueOutcome::Enqueued | EnqueueOutcome::Coalesced, report) => {
|
||||
(EnqueueOutcome::Enqueued, report) => {
|
||||
self.checkpoint.enqueued += 1;
|
||||
let rx = report.ok_or(Stop::Unavailable)?;
|
||||
{
|
||||
if let Some(rx) = report {
|
||||
let key = key.to_string();
|
||||
self.outstanding.push(Box::pin(async move { (key, rx.await) }));
|
||||
}
|
||||
@@ -1371,6 +1322,11 @@ impl Job {
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
(EnqueueOutcome::Coalesced, _) => {
|
||||
// Someone else pulls it; its result is not ours to count.
|
||||
self.checkpoint.enqueued += 1;
|
||||
return Ok(());
|
||||
}
|
||||
(EnqueueOutcome::QueueFull, _) => {
|
||||
// Wait, never drop: one completion frees a slot.
|
||||
if self.outstanding.is_empty() {
|
||||
@@ -1510,8 +1466,7 @@ impl Job {
|
||||
lease_until: now + BACKFILL_LEASE,
|
||||
});
|
||||
}
|
||||
let etag =
|
||||
write_checkpoint(&self.api, &self.bucket, self.context.incarnation_id(), &self.checkpoint, Some(&self.etag)).await?;
|
||||
let etag = write_checkpoint(&self.api, &self.bucket, &self.checkpoint, Some(&self.etag)).await?;
|
||||
self.etag = etag;
|
||||
self.keys_since_save = 0;
|
||||
self.last_save = Instant::now();
|
||||
@@ -1523,7 +1478,7 @@ impl Job {
|
||||
/// Spawns [`run_backfill_recovery_loop`] on the store's shutdown token;
|
||||
/// `false` (nothing spawned) when the store has no background token.
|
||||
pub fn spawn_backfill_recovery_loop(runner: Arc<BackfillRunner>) -> bool {
|
||||
let Some(cancel) = runner.api.background_cancel_token() else {
|
||||
let Some(cancel) = runner.api.ctx.background_cancel_token() else {
|
||||
return false;
|
||||
};
|
||||
tokio::spawn(run_backfill_recovery_loop(runner, cancel));
|
||||
@@ -1551,13 +1506,10 @@ pub async fn run_backfill_recovery_loop(runner: Arc<BackfillRunner>, cancel: Can
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::storage_api::test_support::{
|
||||
BUCKET_LIFECYCLE_LOCK_OBJECT, BucketOperations as _, PutObjectCommitBarrier, PutObjectCommitPause,
|
||||
isolated_store_over_temp_disks,
|
||||
};
|
||||
use super::*;
|
||||
use crate::on_demand_migration::source_client::SourceObject;
|
||||
use crate::on_demand_migration::sys::PullError;
|
||||
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
|
||||
use crate::bucket::on_demand_migration::source_client::SourceObject;
|
||||
use crate::bucket::on_demand_migration::sys::PullError;
|
||||
use std::collections::{BTreeSet, HashSet};
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
@@ -1680,7 +1632,6 @@ mod tests {
|
||||
|
||||
/// Scripted source + local store + queue with a controllable report path.
|
||||
struct MockContext {
|
||||
incarnation_id: Mutex<Option<Uuid>>,
|
||||
objects: Vec<SourceObject>,
|
||||
page_size: usize,
|
||||
local: Mutex<HashMap<String, LocalBackfillObject>>,
|
||||
@@ -1689,7 +1640,6 @@ mod tests {
|
||||
queue_capacity: usize,
|
||||
pending: Mutex<Vec<(String, oneshot::Sender<QueuedPullOutcome>)>>,
|
||||
fail_keys: HashSet<String>,
|
||||
coalesced: bool,
|
||||
auto_complete: AtomicBool,
|
||||
cancel: CancellationToken,
|
||||
config_updated_at: Mutex<Option<OffsetDateTime>>,
|
||||
@@ -1709,7 +1659,6 @@ mod tests {
|
||||
})
|
||||
.collect();
|
||||
Arc::new(Self {
|
||||
incarnation_id: Mutex::new(None),
|
||||
objects,
|
||||
page_size,
|
||||
local: Mutex::new(HashMap::new()),
|
||||
@@ -1718,7 +1667,6 @@ mod tests {
|
||||
queue_capacity: usize::MAX,
|
||||
pending: Mutex::new(Vec::new()),
|
||||
fail_keys: HashSet::new(),
|
||||
coalesced: false,
|
||||
auto_complete: AtomicBool::new(true),
|
||||
cancel: CancellationToken::new(),
|
||||
config_updated_at: Mutex::new(Some(ts(1_700_000_000))),
|
||||
@@ -1744,10 +1692,6 @@ mod tests {
|
||||
|
||||
#[async_trait]
|
||||
impl BackfillContext for MockContext {
|
||||
fn incarnation_id(&self) -> Uuid {
|
||||
self.incarnation_id.lock().expect("test bucket initialized")
|
||||
}
|
||||
|
||||
async fn list_page(&self, prefix: Option<&str>, token: Option<&str>, max_keys: i32) -> Result<SourcePage, SourceError> {
|
||||
if let Some(err) = self.list_error.lock().take() {
|
||||
return Err(err);
|
||||
@@ -1802,12 +1746,7 @@ mod tests {
|
||||
} else {
|
||||
self.pending.lock().push((key.to_string(), tx));
|
||||
}
|
||||
let outcome = if self.coalesced {
|
||||
EnqueueOutcome::Coalesced
|
||||
} else {
|
||||
EnqueueOutcome::Enqueued
|
||||
};
|
||||
(outcome, Some(futures::FutureExt::shared(rx)))
|
||||
(EnqueueOutcome::Enqueued, Some(rx))
|
||||
}
|
||||
|
||||
fn cancel_token(&self) -> CancellationToken {
|
||||
@@ -1840,17 +1779,12 @@ mod tests {
|
||||
context: Arc<MockContext>,
|
||||
) -> (Vec<tempfile::TempDir>, Arc<ECStore>, Arc<BackfillRunner>) {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
super::super::storage_api::test_support::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("create test bucket");
|
||||
*context.incarnation_id.lock() = Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("test bucket identity"),
|
||||
);
|
||||
// The isolated store has no bucket metadata system; the checkpoint
|
||||
// only needs the bucket's directory under the metadata volume.
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(RUSTFS_META_BUCKET).join(BUCKET_META_PREFIX).join(bucket))
|
||||
.expect("test bucket metadata directory");
|
||||
}
|
||||
let runner = runner_on(node, bucket, context, Arc::clone(&store));
|
||||
(dirs, store, runner)
|
||||
}
|
||||
@@ -1860,129 +1794,6 @@ mod tests {
|
||||
BackfillRunner::new(store, node, Arc::new(contexts))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_checkpoint_waiter_keeps_bucket_fenced_until_commit_finishes() {
|
||||
for (suffix, pause) in [
|
||||
("before", PutObjectCommitPause::BeforeQuotaRename),
|
||||
("after", PutObjectCommitPause::AfterRenameQuorum),
|
||||
] {
|
||||
let bucket = format!("backfill-cancel-tail-{suffix}");
|
||||
let context = MockContext::new(0, 1);
|
||||
let (_dirs, store, _runner) = runner_with("node-a", &bucket, Arc::clone(&context)).await;
|
||||
let original_incarnation = context.incarnation_id();
|
||||
let checkpoint = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", ts(1_700_000_001));
|
||||
let barrier = PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, &checkpoint_path(&bucket), pause);
|
||||
let writer_api = Arc::clone(&store);
|
||||
let writer_bucket = bucket.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
write_checkpoint(&writer_api, &writer_bucket, original_incarnation, &checkpoint, None).await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("caller aborted").is_cancelled());
|
||||
|
||||
let lifecycle_lock = store
|
||||
.new_ns_lock(&bucket, BUCKET_LIFECYCLE_LOCK_OBJECT)
|
||||
.await
|
||||
.expect("lifecycle lock");
|
||||
{
|
||||
let mut probe = Box::pin(lifecycle_lock.get_write_lock(Duration::from_secs(1)));
|
||||
assert!(
|
||||
futures::poll!(probe.as_mut()).is_pending(),
|
||||
"lifecycle writer must first try to acquire the lock"
|
||||
);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), probe.as_mut())
|
||||
.await
|
||||
.is_err(),
|
||||
"the checkpoint owner must retain the user bucket lifecycle read lock after caller cancellation"
|
||||
);
|
||||
}
|
||||
|
||||
let delete_api = Arc::clone(&store);
|
||||
let delete_bucket = bucket.clone();
|
||||
let mut deletion = tokio::spawn(async move { delete_api.delete_bucket(&delete_bucket, &Default::default()).await });
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut deletion).await.is_err(),
|
||||
"DeleteBucket must wait for the checkpoint owner after its caller aborts"
|
||||
);
|
||||
barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(10), deletion)
|
||||
.await
|
||||
.expect("commit must drain and release its lifecycle guard")
|
||||
.expect("delete task")
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &Default::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
assert_ne!(
|
||||
original_incarnation,
|
||||
store.bucket_incarnation_id_from_disk(&bucket).await.expect("new identity")
|
||||
);
|
||||
assert!(
|
||||
read_checkpoint(&store, &bucket)
|
||||
.await
|
||||
.expect("read recreated bucket")
|
||||
.is_none(),
|
||||
"no old checkpoint may outlive bucket deletion"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_checkpoint_writer_cannot_resurrect_or_overwrite_a_recreated_bucket() {
|
||||
let bucket = "backfill-incarnation";
|
||||
let context = MockContext::new(0, 1);
|
||||
let (_dirs, store, _runner) = runner_with("node-a", bucket, Arc::clone(&context)).await;
|
||||
let old_incarnation = context.incarnation_id();
|
||||
let old = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", ts(1_700_000_001));
|
||||
let old_etag = write_checkpoint(&store, bucket, old_incarnation, &old, None)
|
||||
.await
|
||||
.expect("old checkpoint");
|
||||
|
||||
store
|
||||
.delete_bucket(bucket, &Default::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store.make_bucket(bucket, &Default::default()).await.expect("recreate bucket");
|
||||
let current_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("new identity");
|
||||
assert_ne!(old_incarnation, current_incarnation);
|
||||
assert!(
|
||||
read_checkpoint(&store, bucket)
|
||||
.await
|
||||
.expect("read after recreation")
|
||||
.is_none()
|
||||
);
|
||||
for expected_etag in [None, Some(old_etag.as_str())] {
|
||||
let error = write_checkpoint(&store, bucket, old_incarnation, &old, expected_etag)
|
||||
.await
|
||||
.expect_err("stale writer rejected");
|
||||
assert!(matches!(error, BackfillError::Storage(StorageError::BucketNotFound(_))));
|
||||
}
|
||||
assert!(
|
||||
read_checkpoint(&store, bucket)
|
||||
.await
|
||||
.expect("stale writer left no checkpoint")
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let current = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-b", ts(1_700_000_002));
|
||||
let current_etag = write_checkpoint(&store, bucket, current_incarnation, ¤t, None)
|
||||
.await
|
||||
.expect("current checkpoint");
|
||||
let error = write_checkpoint(&store, bucket, old_incarnation, &old, Some(¤t_etag))
|
||||
.await
|
||||
.expect_err("old identity cannot overwrite a matching ETag");
|
||||
assert!(matches!(error, BackfillError::Storage(StorageError::BucketNotFound(_))));
|
||||
let stored = read_checkpoint(&store, bucket)
|
||||
.await
|
||||
.expect("read current checkpoint")
|
||||
.expect("current checkpoint remains");
|
||||
assert_eq!(stored.etag, current_etag);
|
||||
assert_eq!(stored.checkpoint, current);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_backfill_lists_pages_and_counts_every_key() {
|
||||
let bucket = "backfill-full";
|
||||
@@ -2101,7 +1912,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn failed_pulls_are_counted_hashed_and_finish_with_failures() {
|
||||
let bucket = "backfill-failed";
|
||||
let mut context = MockContext::new(5, 2);
|
||||
let mut context = MockContext::new(5, 1000);
|
||||
Arc::get_mut(&mut context)
|
||||
.expect("unshared")
|
||||
.fail_keys
|
||||
@@ -2116,52 +1927,12 @@ mod tests {
|
||||
.checkpoint;
|
||||
assert_eq!(cp.state, BackfillState::CompletedWithFailures);
|
||||
assert_eq!((cp.pulled, cp.failed), (4, 1));
|
||||
assert_eq!(cp.continuation_token.as_deref(), Some("2"), "retain the first failed page for recovery");
|
||||
assert_eq!(cp.failed_keys, vec![key_hash("k/00002")]);
|
||||
let last = cp.last_error.expect("last error");
|
||||
assert_eq!(last.class, "local_write");
|
||||
assert_eq!(last.key_hash.as_deref(), Some(key_hash("k/00002").as_str()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coalesced_pulls_block_the_checkpoint_and_report_failures() {
|
||||
let bucket = "backfill-coalesced";
|
||||
let mut context = MockContext::new(1, 1);
|
||||
{
|
||||
let ctx = Arc::get_mut(&mut context).expect("unshared");
|
||||
ctx.coalesced = true;
|
||||
ctx.auto_complete = AtomicBool::new(false);
|
||||
ctx.fail_keys.insert("k/00000".to_string());
|
||||
}
|
||||
let (_dirs, store, runner) = runner_with("node-a", bucket, Arc::clone(&context)).await;
|
||||
runner.start(bucket, BackfillRequest::default()).await.expect("start");
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while context.pending.lock().is_empty() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("job enqueued");
|
||||
assert!(runner.is_running_locally(bucket), "coalescing is not completion");
|
||||
let cp = read_checkpoint(&store, bucket)
|
||||
.await
|
||||
.expect("read")
|
||||
.expect("checkpoint")
|
||||
.checkpoint;
|
||||
assert!(cp.state.is_active());
|
||||
assert!(cp.continuation_token.is_none());
|
||||
context.complete_pending();
|
||||
runner.wait_until_idle(bucket).await;
|
||||
let cp = read_checkpoint(&store, bucket)
|
||||
.await
|
||||
.expect("read")
|
||||
.expect("checkpoint")
|
||||
.checkpoint;
|
||||
assert_eq!(cp.state, BackfillState::CompletedWithFailures);
|
||||
assert_eq!((cp.enqueued, cp.pulled, cp.failed), (1, 0, 1));
|
||||
assert_eq!(cp.failed_keys, vec![key_hash("k/00000")]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn listing_failure_marks_the_job_failed_with_the_error_class() {
|
||||
let bucket = "backfill-list-error";
|
||||
@@ -2322,7 +2093,7 @@ mod tests {
|
||||
node: "node-a".to_string(),
|
||||
lease_until: now - Duration::from_secs(120),
|
||||
});
|
||||
let etag = write_checkpoint(&store, bucket, context.incarnation_id(), &crashed, None)
|
||||
let etag = write_checkpoint(&store, bucket, &crashed, None)
|
||||
.await
|
||||
.expect("seed checkpoint");
|
||||
|
||||
@@ -2333,7 +2104,7 @@ mod tests {
|
||||
lease_until: now + Duration::from_secs(60),
|
||||
});
|
||||
live.updated_at = now;
|
||||
let etag = write_checkpoint(&store, bucket, context.incarnation_id(), &live, Some(&etag))
|
||||
let etag = write_checkpoint(&store, bucket, &live, Some(&etag))
|
||||
.await
|
||||
.expect("live lease");
|
||||
assert_eq!(runner.recover_once().await.taken_over, 0, "unexpired lease must not be taken over");
|
||||
@@ -2350,7 +2121,7 @@ mod tests {
|
||||
lease_until: now - Duration::from_secs(1),
|
||||
});
|
||||
expired.updated_at = now + Duration::from_millis(1);
|
||||
write_checkpoint(&store, bucket, context.incarnation_id(), &expired, Some(&etag))
|
||||
write_checkpoint(&store, bucket, &expired, Some(&etag))
|
||||
.await
|
||||
.expect("expire lease");
|
||||
let stats = runner.recover_once().await;
|
||||
@@ -2374,68 +2145,6 @@ mod tests {
|
||||
assert_eq!(runner.recover_once().await.taken_over, 0, "a finished job is not recovered");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_advances_past_historical_failures_but_pins_new_failures() {
|
||||
let bucket = "backfill-takeover-failed";
|
||||
let mut context = MockContext::new(8, 2);
|
||||
{
|
||||
let ctx = Arc::get_mut(&mut context).expect("unshared");
|
||||
ctx.auto_complete = AtomicBool::new(false);
|
||||
ctx.fail_keys.insert("k/00004".to_string());
|
||||
}
|
||||
let (_dirs, store, runner) = runner_with("node-b", bucket, Arc::clone(&context)).await;
|
||||
let crashed_at = OffsetDateTime::now_utc() - Duration::from_secs(300);
|
||||
let mut crashed = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", crashed_at);
|
||||
crashed.continuation_token = Some("2".to_string());
|
||||
crashed.failed = 1;
|
||||
crashed.record_failure("local_write", Some("k/00002"), crashed_at);
|
||||
write_checkpoint(&store, bucket, context.incarnation_id(), &crashed, None)
|
||||
.await
|
||||
.expect("seed failed page with an expired lease");
|
||||
|
||||
assert_eq!(runner.recover_once().await.taken_over, 1);
|
||||
for (page_start, durable_token, failures) in [(2, "2", 1), (4, "4", 1), (6, "4", 2)] {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
if context.pending.lock().len() == 2 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("resumed page enqueued before its reports complete");
|
||||
assert_eq!(
|
||||
context.pending.lock().iter().map(|(key, _)| key.clone()).collect::<Vec<_>>(),
|
||||
vec![format!("k/{page_start:05}"), format!("k/{:05}", page_start + 1)]
|
||||
);
|
||||
let cp = read_checkpoint(&store, bucket)
|
||||
.await
|
||||
.expect("read persisted page boundary")
|
||||
.expect("checkpoint")
|
||||
.checkpoint;
|
||||
assert_eq!(cp.job_id, crashed.job_id);
|
||||
assert_eq!(cp.owner.as_ref().map(|owner| owner.node.as_str()), Some("node-b"));
|
||||
assert_eq!(cp.continuation_token.as_deref(), Some(durable_token));
|
||||
assert_eq!(cp.failed, failures);
|
||||
context.complete_pending();
|
||||
}
|
||||
runner.wait_until_idle(bucket).await;
|
||||
let cp = read_checkpoint(&store, bucket)
|
||||
.await
|
||||
.expect("read completed checkpoint")
|
||||
.expect("checkpoint")
|
||||
.checkpoint;
|
||||
assert_eq!(cp.state, BackfillState::CompletedWithFailures);
|
||||
assert_eq!((cp.pulled, cp.failed), (5, 2));
|
||||
assert_eq!(cp.continuation_token.as_deref(), Some("4"));
|
||||
assert_eq!(cp.failed_keys, vec![key_hash("k/00002"), key_hash("k/00004")]);
|
||||
assert_eq!(
|
||||
context.list_requests.lock().as_slice(),
|
||||
&[Some("2".to_string()), Some("4".to_string()), Some("6".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_cancels_a_job_whose_config_changed_and_reclaims_own_node_jobs() {
|
||||
let bucket = "backfill-recovery-config";
|
||||
@@ -2445,9 +2154,7 @@ mod tests {
|
||||
|
||||
// Same node name, unexpired lease: only a restart can produce this.
|
||||
let own = BackfillCheckpoint::new(&BackfillRequest::default(), ts(1_700_000_000), "node-a", now);
|
||||
let etag = write_checkpoint(&store, bucket, context.incarnation_id(), &own, None)
|
||||
.await
|
||||
.expect("seed");
|
||||
let etag = write_checkpoint(&store, bucket, &own, None).await.expect("seed");
|
||||
assert_eq!(runner.recover_once().await.taken_over, 1, "own-node running job is reclaimed at once");
|
||||
runner.wait_until_idle(bucket).await;
|
||||
let cp = read_checkpoint(&store, bucket)
|
||||
@@ -2465,7 +2172,7 @@ mod tests {
|
||||
node: "node-z".to_string(),
|
||||
lease_until: now - Duration::from_secs(1),
|
||||
});
|
||||
write_checkpoint(&store, bucket, context.incarnation_id(), &stale, Some(&stored.etag))
|
||||
write_checkpoint(&store, bucket, &stale, Some(&stored.etag))
|
||||
.await
|
||||
.expect("seed stale");
|
||||
let stats = runner.recover_once().await;
|
||||
+17
-472
@@ -14,44 +14,22 @@
|
||||
|
||||
//! Bucket-level On-Demand Migration configuration: wire model (JSON stored
|
||||
//! under `on-demand-migration.json`), pure validation, credential redaction,
|
||||
//! and persisted-config decoding (rustfs/backlog#2148).
|
||||
//! and the publish hook the runtime registers into (rustfs/backlog#2148).
|
||||
//!
|
||||
//! The persisted blob is not encrypted; it shares the trust boundary of
|
||||
//! `bucket-targets.json` and `tier-config.bin`.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt;
|
||||
use std::sync::OnceLock;
|
||||
use url::Url;
|
||||
|
||||
/// Decode bytes only at the service boundary, preserving typed corruption errors.
|
||||
pub(super) fn decode_stored_config(
|
||||
stored: Option<(Vec<u8>, time::OffsetDateTime)>,
|
||||
) -> Result<Option<(OnDemandMigrationConfig, time::OffsetDateTime)>, super::storage_api::StorageError> {
|
||||
stored
|
||||
.map(|(bytes, updated_at)| {
|
||||
OnDemandMigrationConfig::from_json(&bytes)
|
||||
.map(|config| (config, updated_at))
|
||||
.map_err(super::storage_api::StorageError::other)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(crate) async fn get_config(
|
||||
bucket: &str,
|
||||
) -> Result<Option<(OnDemandMigrationConfig, time::OffsetDateTime)>, super::storage_api::StorageError> {
|
||||
decode_stored_config(super::storage_api::get_on_demand_migration_config(bucket).await?)
|
||||
}
|
||||
|
||||
/// The only wire version this build reads and writes.
|
||||
pub const ON_DEMAND_MIGRATION_CONFIG_VERSION: u32 = 1;
|
||||
|
||||
const REDACTED: &str = "REDACTED";
|
||||
const AUTO_REGION: &str = "auto";
|
||||
const AUTO_REGION_FALLBACK: &str = "us-east-1";
|
||||
/// Public Azure Blob host suffix; the account name is the first label.
|
||||
pub const AZURE_BLOB_SUFFIX: &str = "blob.core.windows.net";
|
||||
/// Public Google Cloud Storage endpoint for the native provider.
|
||||
pub const GCS_DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com";
|
||||
|
||||
const KIB: u64 = 1024;
|
||||
const MIB: u64 = 1024 * KIB;
|
||||
@@ -97,25 +75,14 @@ pub struct SourceConfig {
|
||||
pub bucket: String,
|
||||
#[serde(default)]
|
||||
pub path_style: PathStyle,
|
||||
/// `None` means anonymous access to a public source bucket. Only the
|
||||
/// SigV4 providers read it; `azure` and `gcs_native` carry their own
|
||||
/// credentials in `azure` / `gcs`.
|
||||
/// `None` means anonymous access to a public source bucket.
|
||||
#[serde(default)]
|
||||
pub credentials: Option<SourceCredentials>,
|
||||
#[serde(default)]
|
||||
pub tls: TlsConfig,
|
||||
/// Required for [`Provider::Azure`] and rejected for every other
|
||||
/// provider.
|
||||
#[serde(default)]
|
||||
pub azure: Option<AzureSourceConfig>,
|
||||
/// Required for [`Provider::GcsNative`] and rejected for every other
|
||||
/// provider. [`Provider::Gcs`] keeps using `credentials` because it
|
||||
/// speaks the S3 interoperability API.
|
||||
#[serde(default)]
|
||||
pub gcs: Option<GcsSourceConfig>,
|
||||
}
|
||||
|
||||
/// Source vendor family.
|
||||
/// Source vendor family. `azure` is deliberately absent from this version.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Provider {
|
||||
@@ -127,12 +94,6 @@ pub enum Provider {
|
||||
R2,
|
||||
/// GCS XML interoperability API with HMAC keys.
|
||||
Gcs,
|
||||
/// Native Azure Blob service; parameters in `source.azure`.
|
||||
Azure,
|
||||
/// Native GCS JSON API with a service-account key; parameters in
|
||||
/// `source.gcs`.
|
||||
#[serde(rename = "gcs_native")]
|
||||
GcsNative,
|
||||
}
|
||||
|
||||
impl Provider {
|
||||
@@ -144,22 +105,13 @@ impl Provider {
|
||||
Provider::Rustfs => "rustfs",
|
||||
Provider::R2 => "r2",
|
||||
Provider::Gcs => "gcs",
|
||||
Provider::Azure => "azure",
|
||||
Provider::GcsNative => "gcs_native",
|
||||
}
|
||||
}
|
||||
|
||||
/// Providers that do not speak S3 and therefore ignore `region`,
|
||||
/// `path_style` and `credentials`.
|
||||
pub fn is_native(&self) -> bool {
|
||||
matches!(self, Provider::Azure | Provider::GcsNative)
|
||||
}
|
||||
|
||||
/// Providers whose SDKs accept `region = "auto"`; RustFS maps it to
|
||||
/// `us-east-1` for signing. The native providers never sign with a
|
||||
/// region, so they accept it as well.
|
||||
/// `us-east-1` for signing.
|
||||
fn accepts_auto_region(&self) -> bool {
|
||||
matches!(self, Provider::R2 | Provider::Minio | Provider::Rustfs) || self.is_native()
|
||||
matches!(self, Provider::R2 | Provider::Minio | Provider::Rustfs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,73 +164,6 @@ impl fmt::Debug for SourceCredentials {
|
||||
}
|
||||
}
|
||||
|
||||
/// Native Azure Blob source parameters. The container is `source.bucket`,
|
||||
/// so a config never carries two names for the same container. Exactly one
|
||||
/// of `account_key` and `sas_token` must be set: the account key signs with
|
||||
/// Shared Key, the SAS token is appended to every request URL.
|
||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AzureSourceConfig {
|
||||
/// Storage account name; also derives the default `blob.core.windows.net`
|
||||
/// endpoint when `source.endpoint` is absent.
|
||||
pub account: String,
|
||||
/// Base64 shared key of the storage account.
|
||||
#[serde(default)]
|
||||
pub account_key: Option<String>,
|
||||
/// SAS query string without the leading `?`.
|
||||
#[serde(default)]
|
||||
pub sas_token: Option<String>,
|
||||
}
|
||||
|
||||
impl AzureSourceConfig {
|
||||
/// A copy safe to return to admin clients or log: both secrets are
|
||||
/// replaced by `REDACTED`, and whether each is set stays visible.
|
||||
pub fn redacted(&self) -> Self {
|
||||
Self {
|
||||
account: self.account.clone(),
|
||||
account_key: self.account_key.as_ref().map(|_| REDACTED.to_string()),
|
||||
sas_token: self.sas_token.as_ref().map(|_| REDACTED.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for AzureSourceConfig {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("AzureSourceConfig")
|
||||
.field("account", &self.account)
|
||||
.field("account_key", &self.account_key.as_ref().map(|_| REDACTED))
|
||||
.field("sas_token", &self.sas_token.as_ref().map(|_| REDACTED))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Native Google Cloud Storage source parameters. The bucket is
|
||||
/// `source.bucket`; only the service-account key lives here.
|
||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GcsSourceConfig {
|
||||
/// Service-account key JSON, verbatim as downloaded from Google Cloud.
|
||||
pub service_account_json: String,
|
||||
}
|
||||
|
||||
impl GcsSourceConfig {
|
||||
/// A copy safe to return to admin clients or log: the whole key JSON is
|
||||
/// a secret (it embeds the private key), so it is replaced wholesale.
|
||||
pub fn redacted(&self) -> Self {
|
||||
Self {
|
||||
service_account_json: REDACTED.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for GcsSourceConfig {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("GcsSourceConfig")
|
||||
.field("service_account_json", &REDACTED)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TlsConfig {
|
||||
@@ -469,14 +354,6 @@ pub enum OnDemandMigrationConfigError {
|
||||
InvalidBucket(&'static str),
|
||||
#[error("source credentials field {0} must not be empty")]
|
||||
EmptyCredential(&'static str),
|
||||
#[error("source.{0} is required for provider {1}")]
|
||||
MissingProviderBlock(&'static str, Provider),
|
||||
#[error("source.{0} is not valid for provider {1}")]
|
||||
UnexpectedProviderBlock(&'static str, Provider),
|
||||
/// Carries only the reason: the block holds account keys, SAS tokens and
|
||||
/// service-account JSON, so no value of it is ever echoed.
|
||||
#[error("source.{0} is invalid: {1}")]
|
||||
InvalidProviderBlock(&'static str, &'static str),
|
||||
#[error("source tls.ca_cert_pem is not a PEM certificate")]
|
||||
InvalidCaCert,
|
||||
#[error("filter.{0} must be null or a non-empty string")]
|
||||
@@ -511,8 +388,6 @@ impl OnDemandMigrationConfig {
|
||||
pub fn redacted(&self) -> Self {
|
||||
let mut copy = self.clone();
|
||||
copy.source.credentials = self.source.credentials.as_ref().map(SourceCredentials::redacted);
|
||||
copy.source.azure = self.source.azure.as_ref().map(AzureSourceConfig::redacted);
|
||||
copy.source.gcs = self.source.gcs.as_ref().map(GcsSourceConfig::redacted);
|
||||
copy
|
||||
}
|
||||
|
||||
@@ -558,12 +433,6 @@ impl SourceConfig {
|
||||
match (&self.endpoint, self.provider) {
|
||||
(Some(endpoint), _) => endpoint.clone(),
|
||||
(None, Provider::Aws) => format!("https://s3.{}.amazonaws.com", self.region),
|
||||
(None, Provider::Azure) => self
|
||||
.azure
|
||||
.as_ref()
|
||||
.map(|azure| format!("https://{}.{AZURE_BLOB_SUFFIX}", azure.account))
|
||||
.unwrap_or_default(),
|
||||
(None, Provider::GcsNative) => GCS_DEFAULT_ENDPOINT.to_string(),
|
||||
(None, _) => String::new(),
|
||||
}
|
||||
}
|
||||
@@ -579,8 +448,6 @@ impl SourceConfig {
|
||||
}
|
||||
|
||||
fn validate(&self) -> Result<(), OnDemandMigrationConfigError> {
|
||||
self.validate_provider_block()?;
|
||||
|
||||
if self.region.is_empty() {
|
||||
return Err(OnDemandMigrationConfigError::EmptyRegion);
|
||||
}
|
||||
@@ -599,9 +466,6 @@ impl SourceConfig {
|
||||
));
|
||||
}
|
||||
}
|
||||
// Both native providers derive a fixed endpoint; Azure's is built
|
||||
// from the account name, already checked by `validate_provider_block`.
|
||||
None if self.provider.is_native() => {}
|
||||
None => return Err(OnDemandMigrationConfigError::MissingEndpoint(self.provider)),
|
||||
}
|
||||
|
||||
@@ -632,84 +496,6 @@ impl SourceConfig {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The provider-specific block must be present for exactly its own
|
||||
/// provider: a stray `azure` block on an `s3` source would otherwise be
|
||||
/// accepted, stored, and silently ignored by the client builder.
|
||||
fn validate_provider_block(&self) -> Result<(), OnDemandMigrationConfigError> {
|
||||
let missing = OnDemandMigrationConfigError::MissingProviderBlock;
|
||||
let unexpected = OnDemandMigrationConfigError::UnexpectedProviderBlock;
|
||||
let invalid = OnDemandMigrationConfigError::InvalidProviderBlock;
|
||||
|
||||
if self.provider != Provider::Azure && self.azure.is_some() {
|
||||
return Err(unexpected("azure", self.provider));
|
||||
}
|
||||
if self.provider != Provider::GcsNative && self.gcs.is_some() {
|
||||
return Err(unexpected("gcs", self.provider));
|
||||
}
|
||||
|
||||
match self.provider {
|
||||
Provider::Azure => {
|
||||
let azure = self.azure.as_ref().ok_or(missing("azure", self.provider))?;
|
||||
if azure.account.is_empty() {
|
||||
return Err(invalid("azure", "account must not be empty"));
|
||||
}
|
||||
// The account feeds a hostname when the endpoint is derived:
|
||||
// keep it to label characters so it cannot rewrite the host.
|
||||
if !azure.account.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') {
|
||||
return Err(invalid("azure", "account contains characters outside [A-Za-z0-9-]"));
|
||||
}
|
||||
match (azure.account_key.as_deref(), azure.sas_token.as_deref()) {
|
||||
(Some(_), Some(_)) => return Err(invalid("azure", "account_key and sas_token are mutually exclusive")),
|
||||
(None, None) => return Err(invalid("azure", "one of account_key and sas_token is required")),
|
||||
(Some(key), None) => {
|
||||
if key.is_empty() {
|
||||
return Err(invalid("azure", "account_key must not be empty"));
|
||||
}
|
||||
// Decoded here so a mistyped key fails at the admin
|
||||
// boundary instead of on the first source request.
|
||||
if base64_simd::STANDARD.decode_to_vec(key.as_bytes()).is_err() {
|
||||
return Err(invalid("azure", "account_key is not base64"));
|
||||
}
|
||||
}
|
||||
(None, Some(sas)) => {
|
||||
if sas.is_empty() {
|
||||
return Err(invalid("azure", "sas_token must not be empty"));
|
||||
}
|
||||
if sas.starts_with('?') {
|
||||
return Err(invalid("azure", "sas_token must not start with '?'"));
|
||||
}
|
||||
if sas.chars().any(char::is_whitespace) {
|
||||
return Err(invalid("azure", "sas_token must not contain whitespace"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Provider::GcsNative => {
|
||||
let gcs = self.gcs.as_ref().ok_or(missing("gcs", self.provider))?;
|
||||
let key: serde_json::Value = serde_json::from_str(&gcs.service_account_json)
|
||||
.map_err(|_| invalid("gcs", "service_account_json is not valid JSON"))?;
|
||||
let Some(object) = key.as_object() else {
|
||||
return Err(invalid("gcs", "service_account_json is not a JSON object"));
|
||||
};
|
||||
if object.get("type").and_then(serde_json::Value::as_str) != Some("service_account") {
|
||||
return Err(invalid("gcs", "service_account_json is not a service_account key"));
|
||||
}
|
||||
for field in ["client_email", "private_key"] {
|
||||
if object
|
||||
.get(field)
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.is_none_or(str::is_empty)
|
||||
{
|
||||
return Err(invalid("gcs", "service_account_json is missing client_email or private_key"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Provider::S3 | Provider::Aws | Provider::Minio | Provider::Rustfs | Provider::R2 | Provider::Gcs => {}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_endpoint(endpoint: &str) -> Result<(), OnDemandMigrationConfigError> {
|
||||
@@ -804,6 +590,16 @@ impl EndpointKey {
|
||||
}
|
||||
}
|
||||
|
||||
/// Signature of the runtime publish hook: called with the bucket name and
|
||||
/// its parsed config (`None` when absent, cleared, or unreadable) every time
|
||||
/// the bucket's metadata is installed into or removed from the cache.
|
||||
pub type ConfigPublishHook = Box<dyn Fn(&str, Option<&OnDemandMigrationConfig>) + Send + Sync>;
|
||||
|
||||
/// Registration point for the runtime (`OnDemandMigrationSys`). Until it is
|
||||
/// set, metadata publishes are no-ops for ODM, so this crate carries no
|
||||
/// runtime dependency and the config layer stays inert.
|
||||
pub static ON_DEMAND_MIGRATION_CONFIG_HOOK: OnceLock<ConfigPublishHook> = OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -903,15 +699,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
"provider enum",
|
||||
r#"{"source":{"provider":"swift","endpoint":"https://h","region":"r","bucket":"b"}}"#,
|
||||
),
|
||||
(
|
||||
"azure block",
|
||||
r#"{"source":{"provider":"azure","region":"auto","bucket":"b","azure":{"account":"acct","account_key":"a2V5","extra":1}}}"#,
|
||||
),
|
||||
(
|
||||
"gcs block",
|
||||
r#"{"source":{"provider":"gcs_native","region":"auto","bucket":"b","gcs":{"service_account_json":"{}","extra":1}}}"#,
|
||||
r#"{"source":{"provider":"azure","endpoint":"https://h","region":"r","bucket":"b"}}"#,
|
||||
),
|
||||
] {
|
||||
let err = OnDemandMigrationConfig::from_json(json.as_bytes()).expect_err(label);
|
||||
@@ -1032,201 +820,9 @@ mod tests {
|
||||
"{provider}"
|
||||
);
|
||||
}
|
||||
// The native providers never sign with a region, so "auto" is the
|
||||
// honest value to write for them.
|
||||
for cfg in [azure_cfg(), gcs_native_cfg()] {
|
||||
assert_eq!(cfg.source.region, "auto");
|
||||
cfg.validate(empty_ctx())
|
||||
.unwrap_or_else(|err| panic!("{}: {err}", cfg.source.provider));
|
||||
}
|
||||
assert_eq!(sample().source.effective_region(), "us-west-1");
|
||||
}
|
||||
|
||||
const SERVICE_ACCOUNT_JSON: &str = r#"{"type":"service_account","project_id":"p","client_email":"a@b.iam.gserviceaccount.com","private_key":"-----BEGIN PRIVATE KEY-----\nsecret\n-----END PRIVATE KEY-----"}"#;
|
||||
|
||||
fn azure_cfg() -> OnDemandMigrationConfig {
|
||||
let mut cfg = sample();
|
||||
cfg.source.provider = Provider::Azure;
|
||||
cfg.source.endpoint = None;
|
||||
cfg.source.region = "auto".to_string();
|
||||
cfg.source.credentials = None;
|
||||
cfg.source.azure = Some(AzureSourceConfig {
|
||||
account: "legacyaccount".to_string(),
|
||||
account_key: Some("c2VjcmV0LWtleQ==".to_string()),
|
||||
sas_token: None,
|
||||
});
|
||||
cfg
|
||||
}
|
||||
|
||||
fn gcs_native_cfg() -> OnDemandMigrationConfig {
|
||||
let mut cfg = sample();
|
||||
cfg.source.provider = Provider::GcsNative;
|
||||
cfg.source.endpoint = None;
|
||||
cfg.source.region = "auto".to_string();
|
||||
cfg.source.credentials = None;
|
||||
cfg.source.gcs = Some(GcsSourceConfig {
|
||||
service_account_json: SERVICE_ACCOUNT_JSON.to_string(),
|
||||
});
|
||||
cfg
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_providers_derive_their_endpoint_and_round_trip_on_the_wire() {
|
||||
let azure = azure_cfg();
|
||||
assert_eq!(azure.source.effective_endpoint(), "https://legacyaccount.blob.core.windows.net");
|
||||
let gcs = gcs_native_cfg();
|
||||
assert_eq!(gcs.source.effective_endpoint(), "https://storage.googleapis.com");
|
||||
|
||||
for cfg in [azure_cfg(), gcs_native_cfg()] {
|
||||
let json = cfg.to_json().expect("config must serialize");
|
||||
assert_eq!(OnDemandMigrationConfig::from_json(&json).expect("config must parse"), cfg);
|
||||
}
|
||||
// The wire labels are part of the admin contract.
|
||||
assert!(
|
||||
String::from_utf8(azure_cfg().to_json().expect("json"))
|
||||
.expect("utf8")
|
||||
.contains(r#""provider":"azure""#)
|
||||
);
|
||||
assert!(
|
||||
String::from_utf8(gcs_native_cfg().to_json().expect("json"))
|
||||
.expect("utf8")
|
||||
.contains(r#""provider":"gcs_native""#)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_explicit_endpoint_overrides_the_derived_native_one() {
|
||||
// Azurite and fake-gcs-server are addressed this way.
|
||||
let mut cfg = azure_cfg();
|
||||
cfg.source.endpoint = Some("http://azurite.example.com:10000".to_string());
|
||||
cfg.validate(empty_ctx()).expect("an explicit native endpoint is allowed");
|
||||
assert_eq!(cfg.source.effective_endpoint(), "http://azurite.example.com:10000");
|
||||
|
||||
cfg.source.endpoint = Some("http://azurite.example.com:10000/devstoreaccount1".to_string());
|
||||
assert!(
|
||||
matches!(cfg.validate(empty_ctx()), Err(OnDemandMigrationConfigError::InvalidEndpoint(_))),
|
||||
"a native endpoint is still an origin"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_provider_block_belongs_to_exactly_its_own_provider() {
|
||||
let mut cfg = sample();
|
||||
cfg.source.azure = azure_cfg().source.azure;
|
||||
assert_eq!(
|
||||
cfg.validate(empty_ctx()),
|
||||
Err(OnDemandMigrationConfigError::UnexpectedProviderBlock("azure", Provider::S3))
|
||||
);
|
||||
|
||||
let mut cfg = sample();
|
||||
cfg.source.gcs = gcs_native_cfg().source.gcs;
|
||||
assert_eq!(
|
||||
cfg.validate(empty_ctx()),
|
||||
Err(OnDemandMigrationConfigError::UnexpectedProviderBlock("gcs", Provider::S3))
|
||||
);
|
||||
|
||||
let mut cfg = azure_cfg();
|
||||
cfg.source.azure = None;
|
||||
assert_eq!(
|
||||
cfg.validate(empty_ctx()),
|
||||
Err(OnDemandMigrationConfigError::MissingProviderBlock("azure", Provider::Azure))
|
||||
);
|
||||
|
||||
let mut cfg = gcs_native_cfg();
|
||||
cfg.source.gcs = None;
|
||||
assert_eq!(
|
||||
cfg.validate(empty_ctx()),
|
||||
Err(OnDemandMigrationConfigError::MissingProviderBlock("gcs", Provider::GcsNative))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn azure_block_rules() {
|
||||
let with = |account: &str, key: Option<&str>, sas: Option<&str>| {
|
||||
let mut cfg = azure_cfg();
|
||||
cfg.source.azure = Some(AzureSourceConfig {
|
||||
account: account.to_string(),
|
||||
account_key: key.map(str::to_string),
|
||||
sas_token: sas.map(str::to_string),
|
||||
});
|
||||
cfg.validate(empty_ctx())
|
||||
};
|
||||
|
||||
with("legacyaccount", None, Some("sv=2021-08-06&sig=abc%3D")).expect("a SAS token is a complete credential");
|
||||
with("legacyaccount", Some("c2VjcmV0LWtleQ=="), None).expect("an account key is a complete credential");
|
||||
|
||||
for (label, result) in [
|
||||
("empty account", with("", Some("c2VjcmV0LWtleQ=="), None)),
|
||||
// The account becomes the first label of the derived hostname.
|
||||
("account with a dot", with("legacy.account", Some("c2VjcmV0LWtleQ=="), None)),
|
||||
("account with a slash", with("legacy/account", Some("c2VjcmV0LWtleQ=="), None)),
|
||||
("no credential", with("legacyaccount", None, None)),
|
||||
("both credentials", with("legacyaccount", Some("c2VjcmV0LWtleQ=="), Some("sv=1"))),
|
||||
("empty key", with("legacyaccount", Some(""), None)),
|
||||
("key that is not base64", with("legacyaccount", Some("not base64!"), None)),
|
||||
("empty sas", with("legacyaccount", None, Some(""))),
|
||||
("sas with a leading question mark", with("legacyaccount", None, Some("?sv=1"))),
|
||||
("sas with whitespace", with("legacyaccount", None, Some("sv=1 &sig=a"))),
|
||||
] {
|
||||
assert!(
|
||||
matches!(result, Err(OnDemandMigrationConfigError::InvalidProviderBlock("azure", _))),
|
||||
"{label}: {result:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gcs_native_block_requires_a_usable_service_account_key() {
|
||||
let with = |json: &str| {
|
||||
let mut cfg = gcs_native_cfg();
|
||||
cfg.source.gcs = Some(GcsSourceConfig {
|
||||
service_account_json: json.to_string(),
|
||||
});
|
||||
cfg.validate(empty_ctx())
|
||||
};
|
||||
|
||||
with(SERVICE_ACCOUNT_JSON).expect("a service-account key is accepted");
|
||||
for (label, json) in [
|
||||
("empty", ""),
|
||||
("not json", "not json"),
|
||||
("not an object", "[]"),
|
||||
("wrong type", r#"{"type":"authorized_user","client_email":"a@b","private_key":"k"}"#),
|
||||
("no private key", r#"{"type":"service_account","client_email":"a@b"}"#),
|
||||
("empty client email", r#"{"type":"service_account","client_email":"","private_key":"k"}"#),
|
||||
] {
|
||||
let result = with(json);
|
||||
assert!(
|
||||
matches!(result, Err(OnDemandMigrationConfigError::InvalidProviderBlock("gcs", _))),
|
||||
"{label}: {result:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_secrets_never_survive_redaction_or_debug() {
|
||||
let mut azure = azure_cfg();
|
||||
azure.source.azure.as_mut().expect("block").sas_token = Some("sv=2021-08-06&sig=top-secret".to_string());
|
||||
azure.source.azure.as_mut().expect("block").account_key = None;
|
||||
let gcs = gcs_native_cfg();
|
||||
|
||||
for rendered in [
|
||||
format!("{:?}", azure.redacted()),
|
||||
format!("{azure:?}"),
|
||||
String::from_utf8(azure.redacted().to_json().expect("json")).expect("utf8"),
|
||||
] {
|
||||
assert!(!rendered.contains("top-secret"), "{rendered}");
|
||||
assert!(rendered.contains("legacyaccount"), "the account name is not a secret: {rendered}");
|
||||
}
|
||||
for rendered in [
|
||||
format!("{:?}", gcs.redacted()),
|
||||
format!("{gcs:?}"),
|
||||
String::from_utf8(gcs.redacted().to_json().expect("json")).expect("utf8"),
|
||||
] {
|
||||
assert!(!rendered.contains("PRIVATE KEY-----"), "{rendered}");
|
||||
assert!(!rendered.contains("gserviceaccount"), "{rendered}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_rules() {
|
||||
let mut cfg = sample();
|
||||
@@ -1509,55 +1105,4 @@ mod tests {
|
||||
assert!(!rendered.contains("topsecret"), "{rendered}");
|
||||
assert!(!rendered.contains("SK"), "{rendered}");
|
||||
}
|
||||
/// rustfs/backlog#2148: the accessor reports absence as `Ok(None)` and a
|
||||
/// stored payload it cannot parse as a typed error, never as a default
|
||||
/// and never as `ConfigNotFound`.
|
||||
#[tokio::test]
|
||||
async fn get_on_demand_migration_config_distinguishes_absent_from_corrupt() {
|
||||
use super::super::storage_api::StorageError as Error;
|
||||
use super::super::storage_api::test_support::{
|
||||
BUCKET_ON_DEMAND_MIGRATION_CONFIG, BucketMetadata, BucketMetadataSys, isolated_store_over_temp_disks,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
const ODM_JSON: &[u8] = br#"{"source":{"provider":"minio","endpoint":"https://legacy.example.com:9000","region":"auto","bucket":"legacy-bucket","credentials":{"access_key":"AK","secret_key":"SK"}}}"#;
|
||||
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
let sys = BucketMetadataSys::new(ecstore);
|
||||
let bucket = "odm-accessor";
|
||||
|
||||
sys.set(bucket.to_string(), Arc::new(BucketMetadata::new(bucket))).await;
|
||||
assert_eq!(
|
||||
decode_stored_config(sys.get_on_demand_migration_config(bucket).await.unwrap()).unwrap(),
|
||||
None
|
||||
);
|
||||
|
||||
let mut corrupt = BucketMetadata::new(bucket);
|
||||
corrupt.on_demand_migration_config_json = br#"{"source":{"provider":"s3"},"bogus":1}"#.to_vec();
|
||||
sys.set(bucket.to_string(), Arc::new(corrupt)).await;
|
||||
let err = decode_stored_config(sys.get_on_demand_migration_config(bucket).await.unwrap())
|
||||
.expect_err("corrupt config must not read as a default");
|
||||
assert_ne!(err, Error::ConfigNotFound, "corruption must not be reported as absence");
|
||||
let typed = match &err {
|
||||
Error::Io(io) => io
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<OnDemandMigrationConfigError>()),
|
||||
_ => None,
|
||||
};
|
||||
assert!(
|
||||
matches!(typed, Some(OnDemandMigrationConfigError::Malformed(_))),
|
||||
"typed parse error must survive the Result boundary, got: {err:?}"
|
||||
);
|
||||
|
||||
let mut valid = BucketMetadata::new(bucket);
|
||||
valid
|
||||
.update_config(BUCKET_ON_DEMAND_MIGRATION_CONFIG, ODM_JSON.to_vec())
|
||||
.unwrap();
|
||||
let stamped = valid.on_demand_migration_config_updated_at;
|
||||
sys.set(bucket.to_string(), Arc::new(valid)).await;
|
||||
let (config, updated_at) = decode_stored_config(sys.get_on_demand_migration_config(bucket).await.unwrap())
|
||||
.unwrap()
|
||||
.expect("stored config is returned");
|
||||
assert_eq!(config, OnDemandMigrationConfig::from_json(ODM_JSON).unwrap());
|
||||
assert_eq!(updated_at, stamped);
|
||||
}
|
||||
}
|
||||
+27
-358
@@ -25,21 +25,13 @@ use parking_lot::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// The continuation-token version used by ordinary progressing pages.
|
||||
/// The only continuation-token envelope version this build reads and writes.
|
||||
pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1;
|
||||
const LIST_THROUGH_PROGRESS_TOKEN_VERSION: u32 = 2;
|
||||
|
||||
/// The sixteenth consecutive merged page without a key or new EOF fails.
|
||||
/// This also bounds legitimate sparse listings; it is not a cycle detector.
|
||||
pub const MAX_LIST_NO_PROGRESS_PAGES: u8 = 16;
|
||||
|
||||
/// Envelope marker. A bucket that is *not* merging hands out the local
|
||||
/// listing's own marker, so the decoder needs a positive signal before it
|
||||
/// treats an opaque token as a merged one.
|
||||
const LIST_THROUGH_TOKEN_TAG: &str = "odm-list";
|
||||
// Object keys cannot contain NUL (bucket::utils::is_valid_object_prefix),
|
||||
// so this framing cannot collide with a local key used as an opaque marker.
|
||||
const LIST_THROUGH_TOKEN_PREFIX: &str = "\0odm-list:";
|
||||
|
||||
/// Pages fetched per side per request: the first page, plus at most one refill
|
||||
/// when the first one was mostly consumed by the previous page. Two pages of
|
||||
@@ -94,7 +86,8 @@ pub struct MergePick {
|
||||
}
|
||||
|
||||
/// The continuation-token envelope. Opaque to clients: it is serialized as
|
||||
/// framed JSON and then base64-encoded by the same helper as a local marker.
|
||||
/// JSON and then base64-encoded by the same helper that encodes a plain local
|
||||
/// marker, so the wire shape is `base64(json)`.
|
||||
///
|
||||
/// A `null` cursor with `done = false` means "list that side from the start";
|
||||
/// `done = true` means the side is finished and must not be listed again.
|
||||
@@ -118,10 +111,6 @@ pub struct ListThroughToken {
|
||||
/// common prefix compares as itself, never as its members.
|
||||
#[serde(default)]
|
||||
pub last_key: Option<String>,
|
||||
/// Consecutive empty truncated merged pages, present only in v2 tokens.
|
||||
/// Ordinary v1 tokens retain their original serialized shape.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub no_progress: Option<u8>,
|
||||
}
|
||||
|
||||
impl ListThroughToken {
|
||||
@@ -134,14 +123,13 @@ impl ListThroughToken {
|
||||
source: source.token,
|
||||
source_done: source.done,
|
||||
last_key,
|
||||
no_progress: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> String {
|
||||
// The envelope is built here from owned strings, so serialization
|
||||
// cannot fail; the fallback keeps the signature infallible.
|
||||
format!("{LIST_THROUGH_TOKEN_PREFIX}{}", serde_json::to_string(self).unwrap_or_default())
|
||||
serde_json::to_string(self).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -165,35 +153,24 @@ pub enum ListThroughTokenError {
|
||||
|
||||
/// Classifies an already base64-decoded continuation token.
|
||||
///
|
||||
/// Only a framed JSON object is read as a merged token;
|
||||
/// Only a JSON object carrying the envelope marker is read as a merged token;
|
||||
/// anything else is a local marker, so a bucket that turns `list_through` off
|
||||
/// keeps paginating with the tokens it handed out. A token that *is* an
|
||||
/// envelope but was tampered with (unknown version, unknown field, truncated
|
||||
/// JSON) is an error, never a silent fallback.
|
||||
pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, ListThroughTokenError> {
|
||||
let Some(payload) = decoded.strip_prefix(LIST_THROUGH_TOKEN_PREFIX) else {
|
||||
if !decoded.starts_with('{') {
|
||||
return Ok(ListThroughCursor::Local(decoded.to_string()));
|
||||
}
|
||||
let Ok(value) = serde_json::from_str::<serde_json::Value>(decoded) else {
|
||||
// Not JSON at all: an object key may legitimately start with '{'.
|
||||
return Ok(ListThroughCursor::Local(decoded.to_string()));
|
||||
};
|
||||
let value = serde_json::from_str::<serde_json::Value>(payload).map_err(|_| ListThroughTokenError::Malformed)?;
|
||||
if value.get("t").and_then(serde_json::Value::as_str) != Some(LIST_THROUGH_TOKEN_TAG) {
|
||||
return Err(ListThroughTokenError::Malformed);
|
||||
return Ok(ListThroughCursor::Local(decoded.to_string()));
|
||||
}
|
||||
match value.get("v").and_then(serde_json::Value::as_u64) {
|
||||
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {
|
||||
// v1 readers reject this field even when it is null or zero.
|
||||
if value.get("no_progress").is_some() {
|
||||
return Err(ListThroughTokenError::Malformed);
|
||||
}
|
||||
}
|
||||
Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => {
|
||||
if !value
|
||||
.get("no_progress")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
.is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count))
|
||||
{
|
||||
return Err(ListThroughTokenError::Malformed);
|
||||
}
|
||||
}
|
||||
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {}
|
||||
Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
|
||||
None => return Err(ListThroughTokenError::Malformed),
|
||||
}
|
||||
@@ -311,8 +288,6 @@ pub enum ListPageError {
|
||||
Empty,
|
||||
#[error("truncated listing repeats a continuation token")]
|
||||
Repeated,
|
||||
#[error("listing exhausted its consecutive no-progress page budget")]
|
||||
NoProgress(MergeSide),
|
||||
}
|
||||
|
||||
pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> {
|
||||
@@ -377,7 +352,6 @@ pub struct MergeOutcome {
|
||||
#[derive(Debug)]
|
||||
pub struct ListThroughMerger {
|
||||
max_keys: usize,
|
||||
no_progress: Option<u8>,
|
||||
last_key: Option<String>,
|
||||
local: SideState,
|
||||
source: SideState,
|
||||
@@ -397,7 +371,6 @@ impl ListThroughMerger {
|
||||
};
|
||||
Self {
|
||||
max_keys,
|
||||
no_progress: token.and_then(|token| token.no_progress),
|
||||
last_key,
|
||||
local,
|
||||
source,
|
||||
@@ -463,18 +436,13 @@ impl ListThroughMerger {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `issue_progress_tokens` allows a v1 chain to start carrying a budget.
|
||||
/// An existing v2 budget is always enforced, including on reader-only nodes.
|
||||
/// Borrowing lets a source failure re-merge the fetched local buffers.
|
||||
pub fn finish(&self, issue_progress_tokens: bool) -> Result<MergeOutcome, ListPageError> {
|
||||
pub fn finish(self) -> MergeOutcome {
|
||||
let Self {
|
||||
max_keys,
|
||||
no_progress,
|
||||
last_key,
|
||||
local,
|
||||
source,
|
||||
} = self;
|
||||
let max_keys = *max_keys;
|
||||
|
||||
// A side with more pages behind it can only be trusted up to the last
|
||||
// key it handed over: past that horizon the other side's entries could
|
||||
@@ -540,44 +508,12 @@ impl ListThroughMerger {
|
||||
let source_left = !source.disabled && (!source_cursor.done || consumed_source < source.entries.len());
|
||||
let is_truncated = local_left || source_left;
|
||||
|
||||
let reached_eof = (!local.start.done && local_cursor.done) || (!source.start.done && source_cursor.done);
|
||||
let next_no_progress = if !is_truncated || !picks.is_empty() || reached_eof {
|
||||
None
|
||||
} else if max_keys == 0 {
|
||||
// A zero-sized request cannot consume entries. Preserve an existing
|
||||
// budget without spending it or starting a new one.
|
||||
*no_progress
|
||||
} else if issue_progress_tokens || no_progress.is_some() {
|
||||
let count = no_progress.unwrap_or(0).saturating_add(1);
|
||||
if count >= MAX_LIST_NO_PROGRESS_PAGES {
|
||||
// An empty truncated side closes the merge horizon. Local
|
||||
// failure takes precedence; disabling the source cannot fix it.
|
||||
let side = if local.more && local.entries.is_empty() {
|
||||
MergeSide::Local
|
||||
} else if !source.disabled && source.more && source.entries.is_empty() {
|
||||
MergeSide::Source
|
||||
} else {
|
||||
MergeSide::Local
|
||||
};
|
||||
return Err(ListPageError::NoProgress(side));
|
||||
}
|
||||
Some(count)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let last_key = consumed_key.or_else(|| last_key.clone());
|
||||
Ok(MergeOutcome {
|
||||
let last_key = consumed_key.or(last_key);
|
||||
MergeOutcome {
|
||||
picks,
|
||||
is_truncated,
|
||||
next_token: is_truncated.then(|| {
|
||||
let mut token = ListThroughToken::new(local_cursor, source_cursor, last_key);
|
||||
if let Some(count) = next_no_progress {
|
||||
token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION;
|
||||
token.no_progress = Some(count);
|
||||
}
|
||||
token
|
||||
}),
|
||||
})
|
||||
next_token: is_truncated.then(|| ListThroughToken::new(local_cursor, source_cursor, last_key)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -705,7 +641,7 @@ mod tests {
|
||||
.push_page(fetch.side, kept, truncated, next)
|
||||
.expect("reference provider pages must advance");
|
||||
}
|
||||
let outcome = merger.finish(false).expect("valid merge outcome");
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(outcome.is_truncated, outcome.next_token.is_some());
|
||||
if outcome.is_truncated {
|
||||
assert_ne!(outcome.next_token, token, "every truncated merged page must make progress");
|
||||
@@ -788,7 +724,7 @@ mod tests {
|
||||
.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None)
|
||||
.expect("local EOF is valid");
|
||||
assert_eq!(merger.next_fetch(), None);
|
||||
let outcome = merger.finish(false).expect("valid merge outcome");
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(outcome.picks.len(), 1);
|
||||
assert!(!outcome.is_truncated);
|
||||
assert!(outcome.next_token.is_none());
|
||||
@@ -804,7 +740,6 @@ mod tests {
|
||||
source: Some("source-1".to_string()),
|
||||
source_done: false,
|
||||
last_key: Some("a".to_string()),
|
||||
no_progress: None,
|
||||
};
|
||||
let mut merger = ListThroughMerger::new(1, Some(&resume));
|
||||
merger.disable_source();
|
||||
@@ -816,7 +751,7 @@ mod tests {
|
||||
Some("local-2".to_string()),
|
||||
)
|
||||
.expect("local cursor advances");
|
||||
let outcome = merger.finish(false).expect("valid merge outcome");
|
||||
let outcome = merger.finish();
|
||||
assert!(outcome.is_truncated);
|
||||
let token = outcome.next_token.expect("truncated page carries a token");
|
||||
assert_eq!(token.source.as_deref(), Some("source-1"), "the source cursor must not move");
|
||||
@@ -895,7 +830,7 @@ mod tests {
|
||||
.expect("opaque cursor advances regardless of sort order");
|
||||
}
|
||||
assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget");
|
||||
let outcome = merger.finish(false).expect("valid merge outcome");
|
||||
let outcome = merger.finish();
|
||||
assert!(outcome.picks.is_empty());
|
||||
assert!(outcome.is_truncated);
|
||||
let token = outcome.next_token.expect("empty progressing page has a cursor");
|
||||
@@ -905,7 +840,7 @@ mod tests {
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None)
|
||||
.expect("source EOF");
|
||||
let outcome = merger.finish(false).expect("valid merge outcome");
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(
|
||||
outcome.picks,
|
||||
vec![MergePick {
|
||||
@@ -952,7 +887,7 @@ mod tests {
|
||||
Err(ListPageError::Repeated)
|
||||
);
|
||||
merger.disable_source();
|
||||
let outcome = merger.finish(false).expect("valid merge outcome");
|
||||
let outcome = merger.finish();
|
||||
assert_eq!(
|
||||
outcome.picks,
|
||||
vec![MergePick {
|
||||
@@ -1044,287 +979,21 @@ mod tests {
|
||||
let encoded = token.encode();
|
||||
assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token))));
|
||||
|
||||
let bumped = encoded.replace("\"v\":1", "\"v\":3");
|
||||
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(3)));
|
||||
let bumped = encoded.replace("\"v\":1", "\"v\":2");
|
||||
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(2)));
|
||||
|
||||
let extra = encoded.replace("{", "{\"x\":1,");
|
||||
assert_eq!(decode_continuation_token(&extra), Err(ListThroughTokenError::Malformed));
|
||||
|
||||
let truncated = &encoded[..encoded.len() - 3];
|
||||
assert_eq!(decode_continuation_token(truncated), Err(ListThroughTokenError::Malformed));
|
||||
assert_eq!(decode_continuation_token(truncated), Ok(ListThroughCursor::Local(truncated.to_string())));
|
||||
|
||||
let no_version = "\0odm-list:{\"t\":\"odm-list\"}";
|
||||
let no_version = "{\"t\":\"odm-list\"}";
|
||||
assert_eq!(decode_continuation_token(no_version), Err(ListThroughTokenError::Malformed));
|
||||
}
|
||||
|
||||
fn progress_token(count: Option<u8>, local_done: bool, source_done: bool) -> ListThroughToken {
|
||||
let mut token = ListThroughToken::new(
|
||||
SideCursor {
|
||||
token: None,
|
||||
done: local_done,
|
||||
},
|
||||
SideCursor {
|
||||
token: Some("A".into()),
|
||||
done: source_done,
|
||||
},
|
||||
Some("last-key".into()),
|
||||
);
|
||||
if let Some(count) = count {
|
||||
token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION;
|
||||
token.no_progress = Some(count);
|
||||
}
|
||||
token
|
||||
}
|
||||
|
||||
fn push_empty_pages(merger: &mut ListThroughMerger, side: MergeSide) {
|
||||
for _ in 0..MAX_LIST_FETCHES_PER_SIDE {
|
||||
let fetch = merger.next_fetch().expect("empty truncated side must be fetched");
|
||||
assert_eq!(fetch.side, side);
|
||||
let next = format!("{}:next", fetch.token.unwrap_or_default());
|
||||
merger
|
||||
.push_page(side, vec![], true, Some(next))
|
||||
.expect("opaque cursor advances");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() {
|
||||
fn framed(payload: &str) -> String {
|
||||
format!("{LIST_THROUGH_TOKEN_PREFIX}{payload}")
|
||||
}
|
||||
|
||||
let token = progress_token(None, true, false);
|
||||
assert_eq!(
|
||||
token.encode(),
|
||||
concat!(
|
||||
"\0odm-list:",
|
||||
r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
|
||||
)
|
||||
);
|
||||
for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
|
||||
let token = progress_token(Some(count), true, false);
|
||||
assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
|
||||
}
|
||||
for version in [1, 2] {
|
||||
for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
|
||||
let encoded = framed(&format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#));
|
||||
assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
|
||||
}
|
||||
}
|
||||
for payload in [
|
||||
r#"{"t":"odm-list","v":1,"no_progress":1}"#,
|
||||
r#"{"t":"odm-list","v":2}"#,
|
||||
r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#,
|
||||
] {
|
||||
let encoded = framed(payload);
|
||||
assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reader_only_nodes_do_not_start_a_budget_but_mixed_readers_preserve_one() {
|
||||
let mut token = progress_token(None, true, false);
|
||||
for _ in 0..MAX_LIST_NO_PROGRESS_PAGES {
|
||||
let mut merger = ListThroughMerger::new(2, Some(&token));
|
||||
push_empty_pages(&mut merger, MergeSide::Source);
|
||||
token = merger
|
||||
.finish(false)
|
||||
.expect("reader-only v1 behavior")
|
||||
.next_token
|
||||
.expect("truncated cursor");
|
||||
assert_eq!(token.v, 1);
|
||||
assert_eq!(token.no_progress, None);
|
||||
}
|
||||
for count in 1..=MAX_LIST_NO_PROGRESS_PAGES {
|
||||
let mut merger = ListThroughMerger::new(2, Some(&token));
|
||||
push_empty_pages(&mut merger, MergeSide::Source);
|
||||
assert!(merger.next_fetch().is_none(), "the per-request two-fetch limit stays intact");
|
||||
let outcome = merger.finish(count % 2 == 1);
|
||||
if count == MAX_LIST_NO_PROGRESS_PAGES {
|
||||
assert_eq!(outcome, Err(ListPageError::NoProgress(MergeSide::Source)));
|
||||
break;
|
||||
}
|
||||
token = outcome.expect("budget not exhausted").next_token.expect("truncated cursor");
|
||||
assert_eq!(token.no_progress, Some(count));
|
||||
let ListThroughCursor::Merged(decoded) = decode_continuation_token(&token.encode()).expect("round-trip v2") else {
|
||||
panic!("merged cursor expected");
|
||||
};
|
||||
token = *decoded;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn objects_and_common_prefixes_reset_a_budget_at_the_boundary() {
|
||||
for entry in [ListEntryKey::object("result"), ListEntryKey::prefix("result/")] {
|
||||
for issue_tokens in [false, true] {
|
||||
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![], true, Some("B".into()))
|
||||
.expect("empty advancing page");
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![entry.clone()], true, Some("C".into()))
|
||||
.expect("real progress");
|
||||
let outcome = merger
|
||||
.finish(issue_tokens)
|
||||
.expect("real progress does not exhaust the budget");
|
||||
assert_eq!(
|
||||
outcome.picks,
|
||||
vec![MergePick {
|
||||
side: MergeSide::Source,
|
||||
index: 0
|
||||
}]
|
||||
);
|
||||
let next = outcome.next_token.expect("source remains truncated");
|
||||
assert_eq!(next.last_key.as_deref(), Some(entry.name.as_str()));
|
||||
assert_eq!(next.v, 1);
|
||||
assert_eq!(next.no_progress, None);
|
||||
assert!(!next.encode().contains("no_progress"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_new_eof_transition_resets_the_empty_page_budget() {
|
||||
for finished_side in [MergeSide::Local, MergeSide::Source] {
|
||||
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
if finished_side == MergeSide::Local {
|
||||
merger
|
||||
.push_page(MergeSide::Local, vec![], false, None)
|
||||
.expect("new local EOF");
|
||||
push_empty_pages(&mut merger, MergeSide::Source);
|
||||
} else {
|
||||
push_empty_pages(&mut merger, MergeSide::Local);
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![], false, None)
|
||||
.expect("new source EOF");
|
||||
}
|
||||
let next = merger
|
||||
.finish(false)
|
||||
.expect("new EOF is progress")
|
||||
.next_token
|
||||
.expect("other side truncated");
|
||||
assert_eq!(next.no_progress, None);
|
||||
assert_eq!(next.v, 1);
|
||||
assert_eq!(next.local_done, finished_side == MergeSide::Local);
|
||||
assert_eq!(next.source_done, finished_side == MergeSide::Source);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&next));
|
||||
let remaining = if finished_side == MergeSide::Local {
|
||||
MergeSide::Source
|
||||
} else {
|
||||
MergeSide::Local
|
||||
};
|
||||
push_empty_pages(&mut merger, remaining);
|
||||
let next = merger
|
||||
.finish(true)
|
||||
.expect("a new budget starts")
|
||||
.next_token
|
||||
.expect("truncated");
|
||||
assert_eq!(next.no_progress, Some(1), "an already-done side cannot reset every page");
|
||||
}
|
||||
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
merger.push_page(MergeSide::Source, vec![], false, None).expect("final EOF");
|
||||
let outcome = merger.finish(false).expect("EOF succeeds at the budget boundary");
|
||||
assert!(!outcome.is_truncated);
|
||||
assert!(outcome.next_token.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filtered_duplicates_cannot_reset_the_no_progress_budget() {
|
||||
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
for next in ["B", "C"] {
|
||||
let entries = [ListEntryKey::object("last-key"), ListEntryKey::object("earlier")]
|
||||
.into_iter()
|
||||
.filter(|entry| merger.accepts(&entry.name))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(entries.is_empty(), "both provider entries were already consumed");
|
||||
merger
|
||||
.push_page(MergeSide::Source, entries, true, Some(next.into()))
|
||||
.expect("advancing cursor");
|
||||
}
|
||||
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_progress_is_attributed_to_local_when_source_cannot_unblock_it() {
|
||||
for source_mode in ["disabled", "done", "empty", "data"] {
|
||||
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, source_mode == "done");
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
if source_mode == "disabled" {
|
||||
merger.disable_source();
|
||||
}
|
||||
push_empty_pages(&mut merger, MergeSide::Local);
|
||||
match source_mode {
|
||||
"empty" => push_empty_pages(&mut merger, MergeSide::Source),
|
||||
"data" => merger
|
||||
.push_page(MergeSide::Source, vec![ListEntryKey::object("source")], false, None)
|
||||
.expect("source data"),
|
||||
_ => {}
|
||||
}
|
||||
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Local)), "{source_mode}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_budget_failure_remerges_local_objects_and_prefixes_without_refetching() {
|
||||
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false);
|
||||
let mut merger = ListThroughMerger::new(2, Some(&resume));
|
||||
merger
|
||||
.push_page(MergeSide::Local, vec![ListEntryKey::object("local")], true, Some("L1".into()))
|
||||
.expect("local object");
|
||||
merger
|
||||
.push_page(MergeSide::Local, vec![ListEntryKey::prefix("prefix/")], true, Some("L2".into()))
|
||||
.expect("local prefix");
|
||||
push_empty_pages(&mut merger, MergeSide::Source);
|
||||
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source)));
|
||||
merger.disable_source();
|
||||
assert!(merger.next_fetch().is_none(), "fallback does not perform another fetch");
|
||||
let outcome = merger.finish(false).expect("local data makes progress");
|
||||
assert_eq!(
|
||||
outcome.picks,
|
||||
vec![
|
||||
MergePick {
|
||||
side: MergeSide::Local,
|
||||
index: 0
|
||||
},
|
||||
MergePick {
|
||||
side: MergeSide::Local,
|
||||
index: 1
|
||||
}
|
||||
]
|
||||
);
|
||||
let token = outcome.next_token.expect("remaining local page");
|
||||
assert_eq!(token.local.as_deref(), Some("L2"));
|
||||
assert_eq!(token.source.as_deref(), Some("A"));
|
||||
assert_eq!(token.last_key.as_deref(), Some("prefix/"));
|
||||
assert_eq!(token.no_progress, None);
|
||||
assert_eq!(token.v, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zero_sized_merge_preserves_an_existing_budget() {
|
||||
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
|
||||
let mut merger = ListThroughMerger::new(0, Some(&resume));
|
||||
merger
|
||||
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], true, Some("B".into()))
|
||||
.expect("source page");
|
||||
let outcome = merger.finish(false).expect("a zero-sized request cannot consume entries");
|
||||
assert!(outcome.picks.is_empty());
|
||||
assert_eq!(outcome.next_token.expect("unconsumed source").no_progress, resume.no_progress);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plain_local_marker_stays_local() {
|
||||
for marker in [
|
||||
r#"{"t":"odm-list","v":1}"#,
|
||||
r#"{"t":"odm-list","v":2,"local_done":true}"#,
|
||||
r#"{"t":"odm-list"}"#,
|
||||
] {
|
||||
assert_eq!(decode_continuation_token(marker), Ok(ListThroughCursor::Local(marker.to_string())));
|
||||
}
|
||||
assert_eq!(
|
||||
decode_continuation_token("photos/2024/01.jpg"),
|
||||
Ok(ListThroughCursor::Local("photos/2024/01.jpg".to_string()))
|
||||
+7
-30
@@ -19,45 +19,30 @@
|
||||
//! client, and the per-node runtime (`sys`) that turns configs into live
|
||||
//! clients guarded by a breaker, a negative cache, singleflight and a pull
|
||||
//! concurrency limit (rustfs/backlog#2147).
|
||||
//!
|
||||
//! A source is reached through one `SourceBackend`: the S3 dialect for every
|
||||
//! S3-compatible provider, and a native backend for the providers that have no
|
||||
//! S3 API (`azure`, `gcs_native`).
|
||||
|
||||
pub mod azure;
|
||||
#[cfg(test)]
|
||||
mod backend_contract;
|
||||
pub mod backfill;
|
||||
pub mod breaker;
|
||||
pub mod config;
|
||||
#[cfg(feature = "gcs")]
|
||||
pub mod gcs;
|
||||
pub mod list_through;
|
||||
mod metrics;
|
||||
mod native_http;
|
||||
pub mod negative_cache;
|
||||
pub mod pull;
|
||||
pub mod source_client;
|
||||
pub mod stats;
|
||||
mod storage_api;
|
||||
pub mod sys;
|
||||
#[cfg(test)]
|
||||
mod test_http_fixture;
|
||||
|
||||
pub use breaker::{
|
||||
BREAKER_FAILURE_THRESHOLD, BREAKER_FAILURE_WINDOW, BREAKER_HALF_OPEN_MAX_PROBES, BREAKER_OPEN_DURATION, Breaker,
|
||||
BreakerState, BreakerTransition, BreakerVerdict,
|
||||
};
|
||||
pub use config::{
|
||||
AzureSourceConfig, FilterConfig, GcsSourceConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_VERSION, OnDemandMigrationConfig,
|
||||
OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig, SourceCredentials,
|
||||
SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
|
||||
ConfigPublishHook, FilterConfig, HeadPolicy, ON_DEMAND_MIGRATION_CONFIG_HOOK, ON_DEMAND_MIGRATION_CONFIG_VERSION,
|
||||
OnDemandMigrationConfig, OnDemandMigrationConfigError, PathStyle, PolicyConfig, Provider, RangeGetPolicy, SourceConfig,
|
||||
SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
|
||||
};
|
||||
pub use list_through::{
|
||||
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
|
||||
ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome, MergePick,
|
||||
MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
|
||||
decode_continuation_token, source_list_plan,
|
||||
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
|
||||
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
|
||||
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
|
||||
};
|
||||
pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
|
||||
pub use pull::{
|
||||
@@ -65,19 +50,11 @@ pub use pull::{
|
||||
PullQueue, PullReason, PullSource, QueuedPullOutcome, SourceBody, SourceIdleGuard, WriteBackBody, WriteBackError,
|
||||
WriteBackOutcome, WriteBackPart, WriteBackRequest, commit_inline, commit_inline_with, idle_guarded_body,
|
||||
};
|
||||
pub use source_client::{
|
||||
SourceClient, SourceError, SourceGet, SourceHead, SourceListRequest, SourceObject, SourcePage, SourceSse, is_multipart_etag,
|
||||
};
|
||||
pub use stats::{
|
||||
GaugeGuard, LastSourceError, LatencyBucketSnapshot, OdmOp, OdmOutcome, OdmStats, OdmStatsSnapshot, PullFailureReason,
|
||||
PullPath, SOURCE_LATENCY_BUCKET_BOUNDS_MS, SourceLatencySnapshot,
|
||||
};
|
||||
pub use sys::{
|
||||
ApplyOutcome, BucketOdmState, GLOBAL_ON_DEMAND_MIGRATION_SYS, OdmBucketSnapshot, OdmLookup, OdmStateError,
|
||||
OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_backend_spec,
|
||||
source_client_spec,
|
||||
OnDemandMigrationSys, PullError, PullFollower, PullLeader, PullOutcome, PullResult, PullSlot, source_client_spec,
|
||||
};
|
||||
|
||||
pub(crate) fn register_metrics() {
|
||||
metrics::register();
|
||||
}
|
||||
+25
-59
@@ -46,10 +46,10 @@ use super::stats::{PullFailureReason, PullPath};
|
||||
use super::sys::{BucketOdmState, OnDemandMigrationSys, PullError, PullOutcome, PullSlot};
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{FutureExt, Stream, StreamExt, future::Shared};
|
||||
use futures::{Stream, StreamExt};
|
||||
use parking_lot::Mutex;
|
||||
use rand::RngExt;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
@@ -133,8 +133,6 @@ pub enum QueuedPullOutcome {
|
||||
Failed(PullError),
|
||||
}
|
||||
|
||||
pub type QueuedPullReport = Shared<oneshot::Receiver<QueuedPullOutcome>>;
|
||||
|
||||
/// Result of [`PullQueue::enqueue`].
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub enum EnqueueOutcome {
|
||||
@@ -243,8 +241,6 @@ impl PullSource for SourceClient {
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WriteBackRequest {
|
||||
pub bucket: String,
|
||||
/// Identity captured with the source configuration, retained through cleanup.
|
||||
pub bucket_incarnation_id: uuid::Uuid,
|
||||
pub key: String,
|
||||
/// Source HEAD/GET of the whole object.
|
||||
pub head: SourceHead,
|
||||
@@ -255,7 +251,6 @@ pub struct WriteBackRequest {
|
||||
pub preserve_etag: bool,
|
||||
/// `policy.emit_events`.
|
||||
pub emit_events: bool,
|
||||
pub respect_delete_marker: bool,
|
||||
/// Source tags to copy (`policy.copy_tags`), `None` to skip.
|
||||
pub tags: Option<HashMap<String, String>>,
|
||||
}
|
||||
@@ -265,14 +260,12 @@ impl WriteBackRequest {
|
||||
let config = state.config();
|
||||
Self {
|
||||
bucket: state.bucket().to_string(),
|
||||
bucket_incarnation_id: state.incarnation_id(),
|
||||
key: key.to_string(),
|
||||
head,
|
||||
source_label: format!("{}:{}", config.source.provider.as_str(), config.source.bucket),
|
||||
pulled_at: OffsetDateTime::now_utc(),
|
||||
preserve_etag: config.policy.preserve_etag,
|
||||
emit_events: config.policy.emit_events,
|
||||
respect_delete_marker: config.policy.respect_local_delete_marker,
|
||||
tags,
|
||||
}
|
||||
}
|
||||
@@ -360,7 +353,7 @@ pub trait OdmWriteBack: Send + Sync {
|
||||
parts: Vec<WriteBackPart>,
|
||||
) -> Result<WriteBackOutcome, WriteBackError>;
|
||||
|
||||
async fn abort_multipart_upload(&self, request: &WriteBackRequest, upload_id: &str) -> Result<(), WriteBackError>;
|
||||
async fn abort_multipart_upload(&self, bucket: &str, key: &str, upload_id: &str) -> Result<(), WriteBackError>;
|
||||
}
|
||||
|
||||
/// Why the pump stopped feeding the write-back before EOF.
|
||||
@@ -665,7 +658,9 @@ async fn write_multipart(
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
if completed.is_err()
|
||||
&& let Err(abort_err) = write_back.abort_multipart_upload(request, &upload_id).await
|
||||
&& let Err(abort_err) = write_back
|
||||
.abort_multipart_upload(&request.bucket, &request.key, &upload_id)
|
||||
.await
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_ODM_PULL_FAILED,
|
||||
@@ -835,7 +830,7 @@ pub struct PullQueue {
|
||||
bucket: String,
|
||||
tx: mpsc::Sender<PullJob>,
|
||||
/// Keys queued or running; the job removes its key when it ends.
|
||||
pending: Mutex<HashMap<String, QueuedPullReport>>,
|
||||
pending: Mutex<HashSet<String>>,
|
||||
capacity: usize,
|
||||
cancel: CancellationToken,
|
||||
stats: Arc<super::stats::OdmStats>,
|
||||
@@ -874,7 +869,7 @@ impl PullQueue {
|
||||
let queue = Arc::new(Self {
|
||||
bucket: state.bucket().to_string(),
|
||||
tx,
|
||||
pending: Mutex::new(HashMap::new()),
|
||||
pending: Mutex::new(HashSet::new()),
|
||||
capacity,
|
||||
cancel: state.cancel_token(),
|
||||
stats: Arc::clone(state.stats()),
|
||||
@@ -908,24 +903,29 @@ impl PullQueue {
|
||||
self.enqueue_with_report(key, reason).0
|
||||
}
|
||||
|
||||
/// [`Self::enqueue`] with a shared report, including for coalesced pulls.
|
||||
pub fn enqueue_with_report(&self, key: &str, reason: PullReason) -> (EnqueueOutcome, Option<QueuedPullReport>) {
|
||||
/// [`Self::enqueue`] that also hands back the job's report channel when
|
||||
/// a new job was queued (`Coalesced` pulls report to their first
|
||||
/// requester only).
|
||||
pub fn enqueue_with_report(
|
||||
&self,
|
||||
key: &str,
|
||||
reason: PullReason,
|
||||
) -> (EnqueueOutcome, Option<oneshot::Receiver<QueuedPullOutcome>>) {
|
||||
if self.cancel.is_cancelled() {
|
||||
return (EnqueueOutcome::Unavailable, None);
|
||||
}
|
||||
let mut pending = self.pending.lock();
|
||||
if let Some(report) = pending.get(key) {
|
||||
return (EnqueueOutcome::Coalesced, Some(report.clone()));
|
||||
if pending.contains(key) {
|
||||
return (EnqueueOutcome::Coalesced, None);
|
||||
}
|
||||
let (report_tx, report_rx) = oneshot::channel();
|
||||
let report_rx = report_rx.shared();
|
||||
match self.tx.try_send(PullJob {
|
||||
key: key.to_string(),
|
||||
reason,
|
||||
report: Some(report_tx),
|
||||
}) {
|
||||
Ok(()) => {
|
||||
pending.insert(key.to_string(), report_rx.clone());
|
||||
pending.insert(key.to_string());
|
||||
(EnqueueOutcome::Enqueued, Some(report_rx))
|
||||
}
|
||||
Err(TrySendError::Full(_)) => {
|
||||
@@ -1072,7 +1072,7 @@ impl BucketOdmState {
|
||||
self: &Arc<Self>,
|
||||
key: &str,
|
||||
reason: PullReason,
|
||||
) -> (EnqueueOutcome, Option<QueuedPullReport>) {
|
||||
) -> (EnqueueOutcome, Option<oneshot::Receiver<QueuedPullOutcome>>) {
|
||||
match self.pull_queue() {
|
||||
Some(queue) => queue.enqueue_with_report(key, reason),
|
||||
None => (EnqueueOutcome::Unavailable, None),
|
||||
@@ -1094,7 +1094,7 @@ impl OnDemandMigrationSys {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::on_demand_migration::config::{
|
||||
use crate::bucket::on_demand_migration::config::{
|
||||
FilterConfig, OnDemandMigrationConfig, PathStyle as ConfigPathStyle, PolicyConfig, Provider, SourceConfig,
|
||||
SourceCredentials, TlsConfig,
|
||||
};
|
||||
@@ -1119,8 +1119,6 @@ mod tests {
|
||||
session_token: None,
|
||||
}),
|
||||
tls: TlsConfig::default(),
|
||||
azure: None,
|
||||
gcs: None,
|
||||
},
|
||||
filter: FilterConfig::default(),
|
||||
policy: PolicyConfig::default(),
|
||||
@@ -1341,7 +1339,7 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
async fn abort_multipart_upload(&self, _request: &WriteBackRequest, upload_id: &str) -> Result<(), WriteBackError> {
|
||||
async fn abort_multipart_upload(&self, _bucket: &str, _key: &str, upload_id: &str) -> Result<(), WriteBackError> {
|
||||
self.aborted.lock().push(upload_id.to_string());
|
||||
Ok(())
|
||||
}
|
||||
@@ -1401,21 +1399,13 @@ mod tests {
|
||||
assert_eq!(queue.capacity(), 1024);
|
||||
|
||||
let mut outcomes = HashMap::new();
|
||||
let mut shared_report = None;
|
||||
for _ in 0..100 {
|
||||
let (outcome, report) = queue.enqueue_with_report("a", PullReason::RangeGet);
|
||||
*outcomes.entry(outcome).or_insert(0) += 1;
|
||||
shared_report = report;
|
||||
*outcomes.entry(queue.enqueue("a", PullReason::RangeGet)).or_insert(0) += 1;
|
||||
}
|
||||
assert_eq!(outcomes.get(&EnqueueOutcome::Enqueued), Some(&1));
|
||||
assert_eq!(outcomes.get(&EnqueueOutcome::Coalesced), Some(&99));
|
||||
assert_eq!(queue.pending_keys(), 1);
|
||||
|
||||
assert_eq!(
|
||||
shared_report.expect("coalesced report").await,
|
||||
Ok(QueuedPullOutcome::Stored { size: 1000 })
|
||||
);
|
||||
|
||||
wait_until("first pull to finish", || queue.pending_keys() == 0).await;
|
||||
assert_eq!(source.head_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(source.get_calls.load(Ordering::SeqCst), 1);
|
||||
@@ -1448,23 +1438,6 @@ mod tests {
|
||||
assert_eq!(queue.enqueue("a", PullReason::RangeGet), EnqueueOutcome::Unavailable);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coalesced_enqueues_share_failure_reports() {
|
||||
let sys = OnDemandMigrationSys::new();
|
||||
let state = enabled_state(&sys, &config()).await;
|
||||
let source = MockSource::with_object("missing", 1000, BodyKind::Bytes(body_bytes(1000)));
|
||||
let queue = PullQueue::start(Arc::clone(&state), source, Arc::new(MockWriteBack::default()));
|
||||
let (first, first_report) = queue.enqueue_with_report("absent", PullReason::RangeGet);
|
||||
let (second, second_report) = queue.enqueue_with_report("absent", PullReason::Backfill);
|
||||
assert_eq!(first, EnqueueOutcome::Enqueued);
|
||||
assert_eq!(second, EnqueueOutcome::Coalesced);
|
||||
let (first, second) = tokio::join!(first_report.expect("leader report"), second_report.expect("coalesced report"));
|
||||
assert_eq!(first, second);
|
||||
assert!(matches!(first, Ok(QueuedPullOutcome::Failed(_))));
|
||||
sys.remove(BUCKET);
|
||||
queue.wait_until_stopped().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_full_is_reported_and_cancel_drains_without_leaking_tasks() {
|
||||
let sys = OnDemandMigrationSys::new();
|
||||
@@ -1494,23 +1467,16 @@ mod tests {
|
||||
wait_until("dispatcher to wait for a slot", || state.stats().queue_depth() == 1).await;
|
||||
assert_eq!(queue.enqueue("c", PullReason::LargeObject), EnqueueOutcome::Enqueued);
|
||||
assert_eq!(queue.enqueue("d", PullReason::LargeObject), EnqueueOutcome::QueueFull);
|
||||
let (coalesced, canceled_report) = queue.enqueue_with_report("c", PullReason::LargeObject);
|
||||
assert_eq!(coalesced, EnqueueOutcome::Coalesced);
|
||||
assert_eq!(queue.enqueue("c", PullReason::LargeObject), EnqueueOutcome::Coalesced);
|
||||
assert_eq!(queue.pending_keys(), 3);
|
||||
assert_eq!(failures(&state).get("queue_full"), Some(&1));
|
||||
assert!(!queue.is_stopped());
|
||||
|
||||
assert_eq!(sys.remove(BUCKET), crate::on_demand_migration::ApplyOutcome::Removed);
|
||||
assert_eq!(sys.remove(BUCKET), crate::bucket::on_demand_migration::ApplyOutcome::Removed);
|
||||
tokio::time::timeout(Duration::from_secs(5), queue.wait_until_stopped())
|
||||
.await
|
||||
.expect("dispatcher and in-flight job must exit after cancel");
|
||||
assert!(queue.is_stopped());
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(5), canceled_report.expect("coalesced cancellation report"))
|
||||
.await
|
||||
.expect("cancellation closes the report")
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(queue.pending_keys(), 0);
|
||||
assert_eq!(state.inflight_keys(), 0);
|
||||
assert_eq!(state.stats().inflight_pulls(), 0);
|
||||
+33
-305
@@ -25,14 +25,11 @@
|
||||
//! Client-supplied `If-*`, `Authorization`, `Host` and SSE-C headers are never
|
||||
//! forwarded: v1 rejects SSE-C source objects outright.
|
||||
|
||||
use super::azure::AzureSourceBackend;
|
||||
#[cfg(feature = "gcs")]
|
||||
use super::gcs::GcsNativeSourceBackend;
|
||||
use super::list_through::{ListPageError, validate_list_page};
|
||||
use super::storage_api::HTTPRangeSpec;
|
||||
use super::storage_api::remote_s3_client::{
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3EndpointSpec, RemoteS3RetryPolicy, build_remote_s3_config,
|
||||
};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
use aws_sdk_s3::Client as S3Client;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::operation::get_object::GetObjectOutput;
|
||||
@@ -68,10 +65,6 @@ pub enum SourceProvider {
|
||||
/// Generic S3-compatible service.
|
||||
#[default]
|
||||
S3,
|
||||
/// Native Azure Blob service; not an S3 dialect.
|
||||
Azure,
|
||||
/// Native GCS JSON API with a service-account key; not an S3 dialect.
|
||||
GcsNative,
|
||||
}
|
||||
|
||||
impl SourceProvider {
|
||||
@@ -83,8 +76,6 @@ impl SourceProvider {
|
||||
"minio" => Some(Self::Minio),
|
||||
"rustfs" => Some(Self::Rustfs),
|
||||
"s3" => Some(Self::S3),
|
||||
"azure" => Some(Self::Azure),
|
||||
"gcs_native" => Some(Self::GcsNative),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -97,8 +88,6 @@ impl SourceProvider {
|
||||
Self::Minio => "minio",
|
||||
Self::Rustfs => "rustfs",
|
||||
Self::S3 => "s3",
|
||||
Self::Azure => "azure",
|
||||
Self::GcsNative => "gcs_native",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,75 +153,12 @@ pub struct SourceClientSpec {
|
||||
/// Wire requests one logical source call may cost. The pull pipeline and
|
||||
/// the backfill job own the retry budget (`pull.rs` `PULL_MAX_RETRIES`,
|
||||
/// `backfill.rs` `LIST_MAX_RETRIES`) and the breaker counts logical calls,
|
||||
/// so ODM declares [`RemoteS3RetryPolicy::Disabled`]. An ambiguous HEAD
|
||||
/// 404 additionally probes the bucket before declaring a key absent.
|
||||
/// so ODM declares [`RemoteS3RetryPolicy::Disabled`] and keeps one counted
|
||||
/// failure equal to one request against a struggling source.
|
||||
pub retry: RemoteS3RetryPolicy,
|
||||
/// Bytes per second the pull pipeline may consume from this source;
|
||||
/// `None` means unlimited. Enforced by the consumer, not by this client.
|
||||
pub bandwidth_limit: Option<NonZeroU64>,
|
||||
/// Which [`SourceBackend`] to build. The S3 variant reads `region`,
|
||||
/// `path_style` and `credentials`; the native variants ignore all three
|
||||
/// and carry their own credentials.
|
||||
pub backend: SourceBackendSpec,
|
||||
}
|
||||
|
||||
/// Provider-specific half of [`SourceClientSpec`].
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub enum SourceBackendSpec {
|
||||
#[default]
|
||||
S3,
|
||||
Azure(AzureSourceSpec),
|
||||
Gcs(GcsSourceSpec),
|
||||
}
|
||||
|
||||
/// Native Azure Blob parameters. The container is [`SourceClientSpec::bucket`].
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct AzureSourceSpec {
|
||||
pub account: String,
|
||||
pub auth: AzureAuth,
|
||||
}
|
||||
|
||||
impl fmt::Debug for AzureSourceSpec {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("AzureSourceSpec")
|
||||
.field("account", &self.account)
|
||||
.field("auth", &self.auth)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// How Azure requests are authorized.
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub enum AzureAuth {
|
||||
/// Base64 storage-account key, signed per request with Shared Key.
|
||||
SharedKey(String),
|
||||
/// SAS query string without the leading `?`, appended to every URL.
|
||||
Sas(String),
|
||||
}
|
||||
|
||||
impl fmt::Debug for AzureAuth {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
// Both variants are secrets; only the scheme may be rendered.
|
||||
f.write_str(match self {
|
||||
Self::SharedKey(_) => "SharedKey(REDACTED)",
|
||||
Self::Sas(_) => "Sas(REDACTED)",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Native GCS parameters. The bucket is [`SourceClientSpec::bucket`].
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct GcsSourceSpec {
|
||||
/// Service-account key JSON.
|
||||
pub service_account_json: String,
|
||||
}
|
||||
|
||||
impl fmt::Debug for GcsSourceSpec {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("GcsSourceSpec")
|
||||
.field("service_account_json", &"REDACTED")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl SourceClientSpec {
|
||||
@@ -335,9 +261,8 @@ const THROTTLE_CODES: &[&str] = &[
|
||||
"RequestLimitExceeded",
|
||||
"TooManyRequests",
|
||||
"RequestThrottled",
|
||||
"ServerBusy",
|
||||
];
|
||||
const NOT_FOUND_CODES: &[&str] = &["NoSuchKey", "BlobNotFound"];
|
||||
const NOT_FOUND_CODES: &[&str] = &["NoSuchKey", "NotFound", "NoSuchBucket", "NoSuchVersion"];
|
||||
const ACCESS_DENIED_CODES: &[&str] = &[
|
||||
"AccessDenied",
|
||||
"InvalidAccessKeyId",
|
||||
@@ -345,10 +270,9 @@ const ACCESS_DENIED_CODES: &[&str] = &[
|
||||
"AllAccessDisabled",
|
||||
"ExpiredToken",
|
||||
"InvalidToken",
|
||||
"AuthorizationPermissionMismatch",
|
||||
];
|
||||
|
||||
pub(super) fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError {
|
||||
fn classify_status(status: u16, code: Option<&str>, message: String) -> SourceError {
|
||||
if let Some(code) = code {
|
||||
if THROTTLE_CODES.contains(&code) {
|
||||
return SourceError::Throttled;
|
||||
@@ -361,6 +285,7 @@ pub(super) fn classify_status(status: u16, code: Option<&str>, message: String)
|
||||
}
|
||||
}
|
||||
match status {
|
||||
404 => SourceError::NotFound,
|
||||
401 | 403 => SourceError::AccessDenied,
|
||||
429 | 503 => SourceError::Throttled,
|
||||
500..=599 => SourceError::ServerError(status),
|
||||
@@ -420,11 +345,6 @@ pub struct SourceHead {
|
||||
pub storage_class: Option<String>,
|
||||
pub sse: Option<SourceSse>,
|
||||
pub is_multipart_etag: bool,
|
||||
/// The provider's ETag is not derived from the object bytes (Azure
|
||||
/// stamps an opaque concurrency token). Such an ETag is recorded for
|
||||
/// provenance but must never be read as a content digest, so the
|
||||
/// write-back path refuses to use it as the expected MD5.
|
||||
pub etag_is_opaque: bool,
|
||||
}
|
||||
|
||||
/// Per-operation fields shared by HEAD and GET outputs.
|
||||
@@ -446,7 +366,7 @@ struct HeadParts {
|
||||
sse_customer_algorithm: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) fn normalize_etag(etag: Option<String>) -> Option<String> {
|
||||
fn normalize_etag(etag: Option<String>) -> Option<String> {
|
||||
etag.map(|etag| etag.trim().trim_matches('"').to_string())
|
||||
.filter(|etag| !etag.is_empty())
|
||||
}
|
||||
@@ -495,7 +415,6 @@ fn source_head(parts: HeadParts) -> Result<SourceHead, SourceError> {
|
||||
storage_class: parts.storage_class,
|
||||
sse,
|
||||
is_multipart_etag,
|
||||
etag_is_opaque: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -706,55 +625,14 @@ impl fmt::Debug for SourceClient {
|
||||
|
||||
impl SourceClient {
|
||||
pub async fn new(spec: &SourceClientSpec) -> Result<Self, RemoteS3ClientError> {
|
||||
match &spec.backend {
|
||||
SourceBackendSpec::S3 => {
|
||||
let endpoint = spec.endpoint_spec()?;
|
||||
let config = build_remote_s3_config(&endpoint).await?;
|
||||
Ok(Self::from_config_builder(config, endpoint.endpoint_url(), spec))
|
||||
}
|
||||
SourceBackendSpec::Azure(azure) => {
|
||||
let backend = AzureSourceBackend::new(
|
||||
&spec.endpoint,
|
||||
&spec.bucket,
|
||||
azure,
|
||||
spec.timeouts,
|
||||
spec.skip_tls_verify,
|
||||
spec.ca_cert_pem.as_deref(),
|
||||
)?;
|
||||
Ok(Self::from_backend(Box::new(backend), spec))
|
||||
}
|
||||
#[cfg(not(feature = "gcs"))]
|
||||
SourceBackendSpec::Gcs(_) => Err(RemoteS3ClientError::BackendNotCompiled("gcs_native")),
|
||||
#[cfg(feature = "gcs")]
|
||||
SourceBackendSpec::Gcs(gcs) => {
|
||||
let backend = GcsNativeSourceBackend::new(
|
||||
&spec.endpoint,
|
||||
&spec.bucket,
|
||||
gcs,
|
||||
spec.timeouts,
|
||||
spec.skip_tls_verify,
|
||||
spec.ca_cert_pem.as_deref(),
|
||||
)?;
|
||||
Ok(Self::from_backend(Box::new(backend), spec))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wraps a ready backend in the prefix-mapping client. The endpoint is
|
||||
/// kept only for `Debug` and admin status.
|
||||
fn from_backend(backend: Box<dyn SourceBackend>, spec: &SourceClientSpec) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
endpoint: spec.endpoint.clone(),
|
||||
bucket: spec.bucket.clone(),
|
||||
source_prefix: spec.source_prefix.clone().filter(|prefix| !prefix.is_empty()),
|
||||
timeouts: spec.timeouts,
|
||||
bandwidth_limit: spec.bandwidth_limit,
|
||||
}
|
||||
let endpoint = spec.endpoint_spec()?;
|
||||
let config = build_remote_s3_config(&endpoint).await?;
|
||||
Ok(Self::from_config_builder(config, endpoint.endpoint_url(), spec))
|
||||
}
|
||||
|
||||
/// `config` must come from [`SourceClientSpec::endpoint_spec`], which is
|
||||
/// where the policy disabling SDK-level retries is declared.
|
||||
/// where the retry policy that keeps one logical call equal to one wire
|
||||
/// request is declared.
|
||||
fn from_config_builder(config: aws_sdk_s3::config::Builder, endpoint: String, spec: &SourceClientSpec) -> Self {
|
||||
let client = S3Client::from_conf(config.interceptor(SourceProxyMarkerInterceptor::new()).build());
|
||||
Self {
|
||||
@@ -876,16 +754,15 @@ impl SourceClient {
|
||||
#[async_trait::async_trait]
|
||||
impl SourceBackend for S3SourceBackend {
|
||||
async fn head(&self, key: &str) -> Result<SourceHead, SourceError> {
|
||||
match self.client.head_object().bucket(&self.bucket).key(key).send().await {
|
||||
Ok(output) => source_head_from_head_output(output),
|
||||
Err(err) if err.raw_response().is_some_and(|response| response.status().as_u16() == 404) => {
|
||||
// HEAD has no error body: a missing bucket must not poison
|
||||
// the per-key negative cache as though only the key was absent.
|
||||
self.probe().await?;
|
||||
Err(SourceError::NotFound)
|
||||
}
|
||||
Err(err) => Err(classify_sdk_error(err)),
|
||||
}
|
||||
let output = self
|
||||
.client
|
||||
.head_object()
|
||||
.bucket(&self.bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(classify_sdk_error)?;
|
||||
source_head_from_head_output(output)
|
||||
}
|
||||
|
||||
/// Streams the object; `range` is passed through as an HTTP `Range`
|
||||
@@ -932,8 +809,8 @@ impl SourceBackend for S3SourceBackend {
|
||||
.contents
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(s3_source_object)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
.filter_map(s3_source_object)
|
||||
.collect();
|
||||
let common_prefixes = output
|
||||
.common_prefixes
|
||||
.unwrap_or_default()
|
||||
@@ -972,20 +849,14 @@ impl SourceBackend for S3SourceBackend {
|
||||
}
|
||||
}
|
||||
|
||||
fn s3_source_object(object: SdkObject) -> Result<SourceObject, SourceError> {
|
||||
let key = object
|
||||
.key
|
||||
.ok_or_else(|| SourceError::Other("source listing object has no key".to_string()))?;
|
||||
let size = object
|
||||
.size
|
||||
.and_then(|size| u64::try_from(size).ok())
|
||||
.ok_or_else(|| SourceError::Other("source listing object has no valid size".to_string()))?;
|
||||
fn s3_source_object(object: SdkObject) -> Option<SourceObject> {
|
||||
let key = object.key?;
|
||||
let etag = normalize_etag(object.e_tag);
|
||||
let is_multipart_etag = etag.as_deref().is_some_and(is_multipart_etag);
|
||||
Ok(SourceObject {
|
||||
Some(SourceObject {
|
||||
key,
|
||||
etag,
|
||||
size,
|
||||
size: object.size.and_then(|size| u64::try_from(size).ok()).unwrap_or(0),
|
||||
last_modified: system_time(object.last_modified),
|
||||
storage_class: object.storage_class.map(|class| class.as_str().to_string()),
|
||||
is_multipart_etag,
|
||||
@@ -995,7 +866,6 @@ fn s3_source_object(object: SdkObject) -> Result<SourceObject, SourceError> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::on_demand_migration::backend_contract::{BackendCapabilities, OBJECT_MD5, assert_backend_contract};
|
||||
use aws_smithy_runtime_api::client::http::{HttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn};
|
||||
use aws_smithy_runtime_api::client::orchestrator::HttpRequest;
|
||||
use aws_smithy_runtime_api::client::result::ConnectorError;
|
||||
@@ -1113,32 +983,9 @@ mod tests {
|
||||
retry: RemoteS3RetryPolicy::Disabled,
|
||||
timeouts: SourceTimeouts::default(),
|
||||
bandwidth_limit: NonZeroU64::new(1_000_000),
|
||||
backend: SourceBackendSpec::S3,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "gcs"))]
|
||||
#[tokio::test]
|
||||
async fn gcs_backend_not_compiled_keeps_hmac_s3_available() {
|
||||
let mut native = spec(None);
|
||||
native.provider = SourceProvider::GcsNative;
|
||||
native.credentials = None;
|
||||
native.backend = SourceBackendSpec::Gcs(GcsSourceSpec {
|
||||
service_account_json: "{}".to_string(),
|
||||
});
|
||||
assert!(matches!(
|
||||
SourceClient::new(&native).await,
|
||||
Err(RemoteS3ClientError::BackendNotCompiled("gcs_native"))
|
||||
));
|
||||
|
||||
let mut hmac = spec(None);
|
||||
hmac.provider = SourceProvider::Gcs;
|
||||
hmac.endpoint = "https://storage.googleapis.com".to_string();
|
||||
SourceClient::new(&hmac)
|
||||
.await
|
||||
.expect("GCS HMAC uses the always-available S3 backend");
|
||||
}
|
||||
|
||||
async fn scripted_client(spec: &SourceClientSpec, responses: Vec<Scripted>) -> (SourceClient, Recorded) {
|
||||
let requests: Recorded = Arc::new(Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(ScriptedConnector {
|
||||
@@ -1642,10 +1489,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn source_error_classification_covers_every_class() {
|
||||
let cases: Vec<(Scripted, &str, bool)> = vec![
|
||||
(status(404, ""), "other", false),
|
||||
(status(404, "<Error><Code>NoSuchKey</Code></Error>"), "not_found", false),
|
||||
(status(404, "<Error><Code>NoSuchBucket</Code></Error>"), "other", false),
|
||||
(status(404, "<Error><Code>NoSuchVersion</Code></Error>"), "other", false),
|
||||
(status(404, ""), "not_found", false),
|
||||
(status(403, ACCESS_DENIED_BODY), "access_denied", false),
|
||||
(status(401, ""), "access_denied", false),
|
||||
(status(429, ""), "throttled", true),
|
||||
@@ -1668,35 +1512,14 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
let (client, requests) = scripted_client(&spec(None), vec![status(404, ""), status(200, "")]).await;
|
||||
// HEAD carries no error body, so the classification must work from the
|
||||
// status alone as well.
|
||||
let (client, _) = scripted_client(&spec(None), vec![status(404, "")]).await;
|
||||
assert!(matches!(client.head_object("missing").await, Err(SourceError::NotFound)));
|
||||
assert_eq!(recorded(&requests).len(), 2, "ambiguous HEAD 404 must check the bucket");
|
||||
let (client, _) = scripted_client(&spec(None), vec![status(404, ""), status(404, "")]).await;
|
||||
assert!(matches!(client.head_object("missing").await, Err(SourceError::Other(_))));
|
||||
let (client, _) = scripted_client(&spec(None), vec![status(404, ""), status(403, "")]).await;
|
||||
assert!(matches!(client.head_object("missing").await, Err(SourceError::AccessDenied)));
|
||||
let (client, _) = scripted_client(&spec(None), vec![status(403, "")]).await;
|
||||
assert!(matches!(client.head_object("secret").await, Err(SourceError::AccessDenied)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_listing_rejects_missing_and_negative_sizes() {
|
||||
for size in [None, Some(-1)] {
|
||||
let object = SdkObject::builder().key("key").set_size(size).build();
|
||||
assert!(matches!(s3_source_object(object), Err(SourceError::Other(_))));
|
||||
}
|
||||
assert!(matches!(
|
||||
s3_source_object(SdkObject::builder().size(0).build()),
|
||||
Err(SourceError::Other(_))
|
||||
));
|
||||
assert_eq!(
|
||||
s3_source_object(SdkObject::builder().key("empty").size(0).build())
|
||||
.expect("empty object")
|
||||
.size,
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn source_client_debug_redacts_credentials() {
|
||||
let (client, _) = scripted_client(&spec(Some("data/")), Vec::new()).await;
|
||||
@@ -1763,102 +1586,7 @@ mod tests {
|
||||
assert_eq!(resolve_path_style(PathStyle::VirtualHost, Minio, "10.0.0.1"), PathStyle::VirtualHost);
|
||||
assert_eq!(resolve_path_style(PathStyle::Path, Aws, "s3.amazonaws.com"), PathStyle::Path);
|
||||
assert_eq!(SourceProvider::from_label(" AWS "), Some(Aws));
|
||||
assert_eq!(SourceProvider::from_label(" Azure "), Some(Azure));
|
||||
assert_eq!(SourceProvider::from_label("gcs_native"), Some(GcsNative));
|
||||
assert_eq!(SourceProvider::from_label("swift"), None);
|
||||
}
|
||||
|
||||
const CONTRACT_LIST_PAGE_ONE: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>source-bucket</Name>
|
||||
<IsTruncated>true</IsTruncated>
|
||||
<NextContinuationToken>cursor-1</NextContinuationToken>
|
||||
<Contents>
|
||||
<Key>dir/a.txt</Key>
|
||||
<LastModified>2015-10-21T07:28:00.000Z</LastModified>
|
||||
<ETag>"5d41402abc4b2a76b9719d911017c592"</ETag>
|
||||
<Size>5</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>
|
||||
<CommonPrefixes><Prefix>dir/sub/</Prefix></CommonPrefixes>
|
||||
</ListBucketResult>"#;
|
||||
|
||||
const CONTRACT_LIST_PAGE_TWO: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>source-bucket</Name>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents>
|
||||
<Key>dir/b.txt</Key>
|
||||
<LastModified>2015-10-21T07:28:00.000Z</LastModified>
|
||||
<ETag>"7d41402abc4b2a76b9719d911017c592"</ETag>
|
||||
<Size>7</Size>
|
||||
</Contents>
|
||||
</ListBucketResult>"#;
|
||||
|
||||
const CONTRACT_TAGGING: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><TagSet>
|
||||
<Tag><Key>env</Key><Value>prod</Value></Tag>
|
||||
</TagSet></Tagging>"#;
|
||||
|
||||
fn contract_object_headers(content_length: u64) -> Vec<(&'static str, String)> {
|
||||
vec![
|
||||
("etag", format!("\"{OBJECT_MD5}\"")),
|
||||
("content-length", content_length.to_string()),
|
||||
("content-type", "text/plain".to_string()),
|
||||
("last-modified", "Wed, 21 Oct 2015 07:28:00 GMT".to_string()),
|
||||
("x-amz-meta-owner", "alice".to_string()),
|
||||
("x-amz-storage-class", "STANDARD".to_string()),
|
||||
]
|
||||
}
|
||||
|
||||
/// The S3 backend behind the scripted connector, without the prefix-mapping
|
||||
/// client on top: the contract is a property of the backend itself.
|
||||
async fn scripted_s3_backend(responses: Vec<Scripted>) -> S3SourceBackend {
|
||||
let spec = spec(None);
|
||||
let connector = SharedHttpConnector::new(ScriptedConnector {
|
||||
requests: Arc::new(Mutex::new(Vec::new())),
|
||||
responses: Arc::new(Mutex::new(responses.into_iter().collect())),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let endpoint = spec.endpoint_spec().expect("test spec endpoint should parse");
|
||||
let config = build_remote_s3_config(&endpoint)
|
||||
.await
|
||||
.expect("test spec should build")
|
||||
.http_client(http_client)
|
||||
.interceptor(SourceProxyMarkerInterceptor::new());
|
||||
S3SourceBackend {
|
||||
client: S3Client::from_conf(config.build()),
|
||||
bucket: spec.bucket.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_backend_satisfies_the_shared_backend_contract() {
|
||||
let mut ranged = contract_object_headers(3);
|
||||
ranged.push(("content-range", "bytes 1-3/5".to_string()));
|
||||
let backend = scripted_s3_backend(vec![
|
||||
ok(contract_object_headers(5), ""),
|
||||
ok(contract_object_headers(5), "hello"),
|
||||
ok(ranged, "ell"),
|
||||
ok(Vec::new(), CONTRACT_LIST_PAGE_ONE),
|
||||
ok(Vec::new(), CONTRACT_LIST_PAGE_TWO),
|
||||
ok(Vec::new(), CONTRACT_TAGGING),
|
||||
ok(Vec::new(), ""),
|
||||
status(404, ""),
|
||||
ok(Vec::new(), ""),
|
||||
status(403, ACCESS_DENIED_BODY),
|
||||
])
|
||||
.await;
|
||||
|
||||
assert_backend_contract(
|
||||
&backend,
|
||||
BackendCapabilities {
|
||||
etag_is_opaque: false,
|
||||
supports_start_after: true,
|
||||
supports_tagging: true,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(SourceProvider::from_label("azure"), None);
|
||||
}
|
||||
|
||||
fn prefix_client(prefix: Option<String>) -> SourceClient {
|
||||
+37
-299
@@ -19,12 +19,13 @@
|
||||
//! [`SourceClient`], a circuit breaker, a negative cache, a per-key
|
||||
//! singleflight table, a pull concurrency limit and counters. Its lifecycle
|
||||
//! follows the bucket metadata cache through the publish hook registered in
|
||||
//! [`BUCKET_CONFIG_PUBLISH_HOOK`]; the hook fires on every cache install
|
||||
//! [`ON_DEMAND_MIGRATION_CONFIG_HOOK`]; the hook fires on every cache install
|
||||
//! path (initial load, admin update, peer reload, refresh loop, lazy load).
|
||||
//!
|
||||
//! Change detection compares the config by value (`PartialEq`) rather than
|
||||
//! by `updated_at`. The bucket incarnation is part of this comparison:
|
||||
//! recreating a bucket must cancel old work even with identical configuration.
|
||||
//! by `updated_at`: the hook does not carry the timestamp, fetching it would
|
||||
//! re-enter the metadata system from inside its own publish path, and a
|
||||
//! byte-identical config never needs a new client anyway.
|
||||
//!
|
||||
//! Client construction is async (TLS material may be read from disk), so
|
||||
//! the hook does not build inline: `publish` removes state synchronously and
|
||||
@@ -40,19 +41,17 @@
|
||||
|
||||
use super::backfill::{PriorityPullPermits, PullPermit, PullPriority};
|
||||
use super::breaker::{Breaker, BreakerState, BreakerTransition, BreakerVerdict};
|
||||
use super::config::{OnDemandMigrationConfig, PathStyle as ConfigPathStyle, Provider, SourceConfig};
|
||||
use super::config::{
|
||||
ON_DEMAND_MIGRATION_CONFIG_HOOK, OnDemandMigrationConfig, PathStyle as ConfigPathStyle, Provider, SourceConfig,
|
||||
};
|
||||
use super::list_through::{SOURCE_LIST_RATE_PER_SEC, SourceListRateLimiter};
|
||||
use super::negative_cache::NegativeCache;
|
||||
use super::pull::{OdmWriteBack, PullQueue};
|
||||
use super::source_client::{
|
||||
AzureAuth, AzureSourceSpec, GcsSourceSpec, SourceBackendSpec, SourceClient, SourceClientSpec, SourceError, SourceProvider,
|
||||
SourceTimeouts,
|
||||
};
|
||||
use super::source_client::{SourceClient, SourceClientSpec, SourceError, SourceProvider, SourceTimeouts};
|
||||
use super::stats::{GaugeGuard, OdmStats, OdmStatsSnapshot, PullFailureReason};
|
||||
use super::storage_api::remote_s3_client::{
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle as ClientPathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
|
||||
};
|
||||
use super::storage_api::{BUCKET_CONFIG_PUBLISH_HOOK, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
use parking_lot::{Mutex, RwLock};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
@@ -83,8 +82,6 @@ pub static GLOBAL_ON_DEMAND_MIGRATION_SYS: OnceLock<OnDemandMigrationSys> = Once
|
||||
/// `resolve` as [`OdmLookup::Unavailable`] and through status snapshots.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum OdmStateError {
|
||||
#[error("the {0} backend is not included in this build")]
|
||||
BackendNotCompiled(&'static str),
|
||||
/// `source.credentials` is `null`; the shared client builder has no
|
||||
/// anonymous mode yet (rustfs/backlog#2149 follow-up).
|
||||
#[error("anonymous source access is not supported yet; configure source credentials")]
|
||||
@@ -269,7 +266,6 @@ impl Drop for InflightEntryGuard<'_> {
|
||||
/// config change (counters excepted), removed when the config goes away.
|
||||
pub struct BucketOdmState {
|
||||
bucket: String,
|
||||
incarnation_id: uuid::Uuid,
|
||||
config: OnDemandMigrationConfig,
|
||||
applied_at: OffsetDateTime,
|
||||
endpoint_host: String,
|
||||
@@ -307,24 +303,21 @@ impl BucketOdmState {
|
||||
async fn build(
|
||||
bucket: &str,
|
||||
config: &OnDemandMigrationConfig,
|
||||
incarnation_id: uuid::Uuid,
|
||||
stats: Arc<OdmStats>,
|
||||
write_back: Option<Arc<dyn OdmWriteBack>>,
|
||||
) -> Arc<Self> {
|
||||
let spec = source_client_spec(config);
|
||||
let client = if config.source.credentials.is_none() && !config.source.provider.is_native() {
|
||||
let client = if config.source.credentials.is_none() {
|
||||
Err(OdmStateError::AnonymousUnsupported)
|
||||
} else {
|
||||
SourceClient::new(&spec).await.map(Arc::new).map_err(|err| match err {
|
||||
RemoteS3ClientError::MissingCredentials => OdmStateError::AnonymousUnsupported,
|
||||
RemoteS3ClientError::BackendNotCompiled(provider) => OdmStateError::BackendNotCompiled(provider),
|
||||
other => OdmStateError::ClientBuild(other.to_string()),
|
||||
})
|
||||
};
|
||||
let policy = &config.policy;
|
||||
Arc::new(Self {
|
||||
bucket: bucket.to_string(),
|
||||
incarnation_id,
|
||||
endpoint_host: endpoint_host(&config.source),
|
||||
config: config.clone(),
|
||||
applied_at: OffsetDateTime::now_utc(),
|
||||
@@ -342,18 +335,10 @@ impl BucketOdmState {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn filter_incarnation(self: Arc<Self>, incarnation_id: uuid::Uuid) -> Option<Arc<Self>> {
|
||||
(self.incarnation_id == incarnation_id && !self.is_cancelled()).then_some(self)
|
||||
}
|
||||
|
||||
pub fn bucket(&self) -> &str {
|
||||
&self.bucket
|
||||
}
|
||||
|
||||
pub fn incarnation_id(&self) -> uuid::Uuid {
|
||||
self.incarnation_id
|
||||
}
|
||||
|
||||
pub fn config(&self) -> &OnDemandMigrationConfig {
|
||||
&self.config
|
||||
}
|
||||
@@ -634,7 +619,6 @@ pub fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClientSpec
|
||||
// load on a source that is already failing.
|
||||
retry: RemoteS3RetryPolicy::Disabled,
|
||||
bandwidth_limit: policy.bandwidth_limit_bytes_per_sec.and_then(NonZeroU64::new),
|
||||
backend: source_backend_spec(source),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -646,31 +630,6 @@ fn source_provider(provider: Provider) -> SourceProvider {
|
||||
Provider::Rustfs => SourceProvider::Rustfs,
|
||||
Provider::R2 => SourceProvider::R2,
|
||||
Provider::Gcs => SourceProvider::Gcs,
|
||||
Provider::Azure => SourceProvider::Azure,
|
||||
Provider::GcsNative => SourceProvider::GcsNative,
|
||||
}
|
||||
}
|
||||
|
||||
/// Which backend the client builds. A native provider whose block is missing
|
||||
/// falls back to the S3 spec, where the builder reports the missing
|
||||
/// credentials: the config layer already refuses to store that shape, so this
|
||||
/// only covers a config written by an older or hand-edited build.
|
||||
pub fn source_backend_spec(source: &SourceConfig) -> SourceBackendSpec {
|
||||
match (source.provider, source.azure.as_ref(), source.gcs.as_ref()) {
|
||||
(Provider::Azure, Some(azure), _) => SourceBackendSpec::Azure(AzureSourceSpec {
|
||||
account: azure.account.clone(),
|
||||
auth: match (&azure.account_key, &azure.sas_token) {
|
||||
(Some(key), _) => AzureAuth::SharedKey(key.clone()),
|
||||
(None, Some(sas)) => AzureAuth::Sas(sas.clone()),
|
||||
// Refused by `SourceConfig::validate`; an empty shared key
|
||||
// fails closed at the builder rather than signing with none.
|
||||
(None, None) => AzureAuth::SharedKey(String::new()),
|
||||
},
|
||||
}),
|
||||
(Provider::GcsNative, _, Some(gcs)) => SourceBackendSpec::Gcs(GcsSourceSpec {
|
||||
service_account_json: gcs.service_account_json.clone(),
|
||||
}),
|
||||
_ => SourceBackendSpec::S3,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -749,51 +708,21 @@ impl OnDemandMigrationSys {
|
||||
/// Registers `publish` as the bucket-metadata publish hook. Returns
|
||||
/// `false` when a hook was already registered.
|
||||
pub fn register_config_hook(&'static self) -> bool {
|
||||
BUCKET_CONFIG_PUBLISH_HOOK
|
||||
.set(Box::new(move |bucket, config_file, stored| {
|
||||
if config_file == BUCKET_ON_DEMAND_MIGRATION_CONFIG {
|
||||
self.publish_stored(bucket, stored.map(|(bytes, _, incarnation)| (bytes, incarnation)));
|
||||
}
|
||||
}))
|
||||
ON_DEMAND_MIGRATION_CONFIG_HOOK
|
||||
.set(Box::new(move |bucket, config| self.publish(bucket, config)))
|
||||
.is_ok()
|
||||
}
|
||||
|
||||
/// Corrupt persisted bytes withdraw state synchronously, just like deletion.
|
||||
fn publish_stored(&'static self, bucket: &str, stored: Option<(&[u8], uuid::Uuid)>) {
|
||||
let incarnation_id = stored.map(|(_, id)| id).unwrap_or_default();
|
||||
match stored.map(|(bytes, _)| OnDemandMigrationConfig::from_json(bytes)).transpose() {
|
||||
Ok(config) => self.publish_for_incarnation(bucket, incarnation_id, config.as_ref()),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_ODM_BUCKET_STATE_APPLIED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ON_DEMAND_MIGRATION,
|
||||
result = "invalid",
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
"Failed to parse on-demand migration config"
|
||||
);
|
||||
self.publish_for_incarnation(bucket, incarnation_id, None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hook entry point: removals apply immediately, installs are spawned
|
||||
/// (client construction is async). Requires a Tokio runtime for the
|
||||
/// install path; without one the config is logged and skipped.
|
||||
pub fn publish_for_incarnation(
|
||||
&'static self,
|
||||
bucket: &str,
|
||||
incarnation_id: uuid::Uuid,
|
||||
config: Option<&OnDemandMigrationConfig>,
|
||||
) {
|
||||
let config = self.desired(config).filter(|_| !incarnation_id.is_nil());
|
||||
let generation = self.reserve_generation(bucket, config.is_some());
|
||||
let Some(config) = config else {
|
||||
pub fn publish(&'static self, bucket: &str, config: Option<&OnDemandMigrationConfig>) {
|
||||
let generation = self.next_generation();
|
||||
let Some(config) = self.desired(config) else {
|
||||
self.remove_with_generation(bucket, generation);
|
||||
return;
|
||||
};
|
||||
if self.is_unchanged(bucket, incarnation_id, config, generation) {
|
||||
if self.is_unchanged(bucket, config, generation) {
|
||||
return;
|
||||
}
|
||||
let Ok(handle) = tokio::runtime::Handle::try_current() else {
|
||||
@@ -812,49 +741,31 @@ impl OnDemandMigrationSys {
|
||||
let bucket = bucket.to_string();
|
||||
let config = config.clone();
|
||||
handle.spawn(async move {
|
||||
self.apply_with_generation(&bucket, incarnation_id, Some(&config), generation)
|
||||
.await;
|
||||
self.apply_with_generation(&bucket, Some(&config), generation).await;
|
||||
});
|
||||
}
|
||||
|
||||
/// Installs, rebuilds, or removes the bucket state for `config`.
|
||||
/// Idempotent: the same config on an installed bucket is a no-op.
|
||||
#[cfg(test)]
|
||||
pub async fn apply(&self, bucket: &str, config: Option<&OnDemandMigrationConfig>) -> ApplyOutcome {
|
||||
self.apply_for_incarnation(bucket, uuid::Uuid::from_u128(1), config).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn publish(&'static self, bucket: &str, config: Option<&OnDemandMigrationConfig>) {
|
||||
self.publish_for_incarnation(bucket, uuid::Uuid::from_u128(1), config);
|
||||
}
|
||||
|
||||
pub async fn apply_for_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
incarnation_id: uuid::Uuid,
|
||||
config: Option<&OnDemandMigrationConfig>,
|
||||
) -> ApplyOutcome {
|
||||
let config = self.desired(config).filter(|_| !incarnation_id.is_nil());
|
||||
let generation = self.reserve_generation(bucket, config.is_some());
|
||||
self.apply_with_generation(bucket, incarnation_id, config, generation).await
|
||||
let generation = self.next_generation();
|
||||
self.apply_with_generation(bucket, config, generation).await
|
||||
}
|
||||
|
||||
async fn apply_with_generation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
incarnation_id: uuid::Uuid,
|
||||
config: Option<&OnDemandMigrationConfig>,
|
||||
generation: u64,
|
||||
) -> ApplyOutcome {
|
||||
let Some(config) = self.desired(config) else {
|
||||
return self.remove_with_generation(bucket, generation);
|
||||
};
|
||||
if self.is_unchanged(bucket, incarnation_id, config, generation) {
|
||||
if self.is_unchanged(bucket, config, generation) {
|
||||
return ApplyOutcome::Unchanged;
|
||||
}
|
||||
let stats = self.state(bucket).map(|state| Arc::clone(&state.stats)).unwrap_or_default();
|
||||
let state = BucketOdmState::build(bucket, config, incarnation_id, stats, self.write_back()).await;
|
||||
let state = BucketOdmState::build(bucket, config, stats, self.write_back()).await;
|
||||
|
||||
let (outcome, previous) = {
|
||||
let mut buckets = self.buckets.write();
|
||||
@@ -898,13 +809,12 @@ impl OnDemandMigrationSys {
|
||||
|
||||
/// Removes a bucket's state (idempotent), cancelling its token.
|
||||
pub fn remove(&self, bucket: &str) -> ApplyOutcome {
|
||||
let generation = self.reserve_generation(bucket, false);
|
||||
let generation = self.next_generation();
|
||||
self.remove_with_generation(bucket, generation)
|
||||
}
|
||||
|
||||
/// One-shot lookup: module switch, bucket state, prefix filter,
|
||||
/// client availability, negative cache, breaker, in that order.
|
||||
#[cfg(test)]
|
||||
pub fn resolve(&self, bucket: &str, key: &str) -> Option<OdmLookup> {
|
||||
if !self.is_module_enabled() {
|
||||
return None;
|
||||
@@ -912,13 +822,6 @@ impl OnDemandMigrationSys {
|
||||
self.state(bucket)?.resolve_key(key)
|
||||
}
|
||||
|
||||
pub fn resolve_for_incarnation(&self, bucket: &str, key: &str, incarnation_id: uuid::Uuid) -> Option<OdmLookup> {
|
||||
if !self.is_module_enabled() {
|
||||
return None;
|
||||
}
|
||||
self.state(bucket)?.filter_incarnation(incarnation_id)?.resolve_key(key)
|
||||
}
|
||||
|
||||
pub fn state(&self, bucket: &str) -> Option<Arc<BucketOdmState>> {
|
||||
self.buckets.read().get(bucket).and_then(|slot| slot.state.clone())
|
||||
}
|
||||
@@ -947,17 +850,8 @@ impl OnDemandMigrationSys {
|
||||
snapshots
|
||||
}
|
||||
|
||||
fn reserve_generation(&self, bucket: &str, installing: bool) -> u64 {
|
||||
// Reserve a desired install before its async client build, under the
|
||||
// same lock that orders removals. Unconfigured buckets need no slot.
|
||||
let mut buckets = self.buckets.write();
|
||||
let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if installing {
|
||||
buckets.entry(bucket.to_string()).or_default().generation = generation;
|
||||
} else if let Some(slot) = buckets.get_mut(bucket) {
|
||||
slot.generation = generation;
|
||||
}
|
||||
generation
|
||||
fn next_generation(&self) -> u64 {
|
||||
self.generation.fetch_add(1, Ordering::Relaxed) + 1
|
||||
}
|
||||
|
||||
fn desired<'c>(&self, config: Option<&'c OnDemandMigrationConfig>) -> Option<&'c OnDemandMigrationConfig> {
|
||||
@@ -966,26 +860,15 @@ impl OnDemandMigrationSys {
|
||||
|
||||
/// Claims `generation` for the bucket when the installed state already
|
||||
/// matches `config` and has a usable client.
|
||||
fn is_unchanged(&self, bucket: &str, incarnation_id: uuid::Uuid, config: &OnDemandMigrationConfig, generation: u64) -> bool {
|
||||
fn is_unchanged(&self, bucket: &str, config: &OnDemandMigrationConfig, generation: u64) -> bool {
|
||||
let mut buckets = self.buckets.write();
|
||||
let Some(slot) = buckets.get_mut(bucket) else {
|
||||
return false;
|
||||
};
|
||||
if slot.generation > generation {
|
||||
return false;
|
||||
}
|
||||
if slot
|
||||
.state
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.incarnation_id != incarnation_id)
|
||||
&& let Some(previous) = slot.state.take()
|
||||
{
|
||||
previous.cancel.cancel();
|
||||
}
|
||||
let unchanged = slot
|
||||
.state
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.client.is_ok() && state.incarnation_id == incarnation_id && state.config == *config);
|
||||
.is_some_and(|state| state.client.is_ok() && state.config == *config);
|
||||
if unchanged && slot.generation < generation {
|
||||
slot.generation = generation;
|
||||
}
|
||||
@@ -1025,8 +908,8 @@ impl OnDemandMigrationSys {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::on_demand_migration::breaker::BREAKER_FAILURE_THRESHOLD;
|
||||
use crate::on_demand_migration::config::{FilterConfig, PolicyConfig, SourceCredentials, SourceTimeout, TlsConfig};
|
||||
use crate::bucket::on_demand_migration::breaker::BREAKER_FAILURE_THRESHOLD;
|
||||
use crate::bucket::on_demand_migration::config::{FilterConfig, PolicyConfig, SourceCredentials, SourceTimeout, TlsConfig};
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
@@ -1046,8 +929,6 @@ mod tests {
|
||||
session_token: None,
|
||||
}),
|
||||
tls: TlsConfig::default(),
|
||||
azure: None,
|
||||
gcs: None,
|
||||
},
|
||||
filter: FilterConfig {
|
||||
prefix: prefix.map(str::to_string),
|
||||
@@ -1164,45 +1045,6 @@ mod tests {
|
||||
assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Rebuilt);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn native_azure_uses_provider_credentials_without_s3_credentials() {
|
||||
let sys = enabled_sys();
|
||||
let mut cfg = config(None);
|
||||
cfg.source.provider = Provider::Azure;
|
||||
cfg.source.endpoint = None;
|
||||
cfg.source.credentials = None;
|
||||
cfg.source.azure = Some(super::super::config::AzureSourceConfig {
|
||||
account: "legacyaccount".to_string(),
|
||||
account_key: Some("c2VjcmV0LWtleQ==".to_string()),
|
||||
sas_token: None,
|
||||
});
|
||||
assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed);
|
||||
let state = ready_state(sys.resolve("b", "k"));
|
||||
assert!(state.client().is_ok(), "native credentials must not be classified as anonymous S3");
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "gcs"))]
|
||||
#[tokio::test]
|
||||
async fn gcs_backend_not_compiled_is_unavailable_not_anonymous() {
|
||||
let sys = enabled_sys();
|
||||
let mut cfg = config(None);
|
||||
cfg.source.provider = Provider::GcsNative;
|
||||
cfg.source.credentials = None;
|
||||
cfg.source.gcs = Some(super::super::config::GcsSourceConfig {
|
||||
service_account_json: "{}".to_string(),
|
||||
});
|
||||
let encoded = cfg.to_json().expect("GCS config is serializable without the backend");
|
||||
let restored: OnDemandMigrationConfig = serde_json::from_slice(&encoded).expect("GCS config stays readable");
|
||||
assert_eq!(restored, cfg);
|
||||
assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed);
|
||||
match sys.resolve("b", "k") {
|
||||
Some(OdmLookup::Unavailable { error, .. }) => {
|
||||
assert_eq!(error, OdmStateError::BackendNotCompiled("gcs_native"));
|
||||
}
|
||||
other => panic!("expected unavailable backend, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn singleflight_admits_one_leader_per_key() {
|
||||
let sys = enabled_sys();
|
||||
@@ -1418,128 +1260,24 @@ mod tests {
|
||||
assert!(state.is_cancelled());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn identical_config_on_recreated_bucket_cancels_old_state() {
|
||||
let sys = enabled_sys();
|
||||
let cfg = config(None);
|
||||
let old_id = uuid::Uuid::new_v4();
|
||||
let new_id = uuid::Uuid::new_v4();
|
||||
sys.apply_for_incarnation("recreated", old_id, Some(&cfg)).await;
|
||||
let old = sys.state("recreated").expect("old state installed");
|
||||
assert!(sys.resolve_for_incarnation("recreated", "key", new_id).is_none());
|
||||
sys.apply_for_incarnation("recreated", new_id, Some(&cfg)).await;
|
||||
let replacement = sys.state("recreated").expect("replacement state installed");
|
||||
assert!(old.is_cancelled());
|
||||
assert!(!Arc::ptr_eq(&old, &replacement));
|
||||
assert_eq!(replacement.incarnation_id(), new_id);
|
||||
assert!(sys.resolve_for_incarnation("recreated", "key", old_id).is_none());
|
||||
assert!(sys.resolve_for_incarnation("recreated", "key", new_id).is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn changed_delete_marker_policy_withdraws_the_captured_lookup() {
|
||||
let sys = enabled_sys();
|
||||
let incarnation = uuid::Uuid::new_v4();
|
||||
let mut cfg = config(None);
|
||||
cfg.policy.respect_local_delete_marker = false;
|
||||
sys.apply_for_incarnation("policy-snapshot", incarnation, Some(&cfg)).await;
|
||||
let captured = sys.state("policy-snapshot").expect("policy A installed");
|
||||
assert!(!captured.config().policy.respect_local_delete_marker);
|
||||
|
||||
cfg.policy.respect_local_delete_marker = true;
|
||||
sys.apply_for_incarnation("policy-snapshot", incarnation, Some(&cfg)).await;
|
||||
let replacement = sys.state("policy-snapshot").expect("policy B installed");
|
||||
assert!(replacement.config().policy.respect_local_delete_marker);
|
||||
assert!(captured.is_cancelled());
|
||||
assert!(
|
||||
captured
|
||||
.filter_incarnation(incarnation)
|
||||
.and_then(|state| state.resolve_key("key"))
|
||||
.is_none(),
|
||||
"a request that evaluated policy A cannot continue through policy B"
|
||||
);
|
||||
assert!(
|
||||
replacement
|
||||
.clone()
|
||||
.filter_incarnation(incarnation)
|
||||
.and_then(|state| state.resolve_key("key"))
|
||||
.is_some()
|
||||
);
|
||||
assert_eq!(
|
||||
replacement
|
||||
.stats()
|
||||
.snapshot(replacement.breaker().state())
|
||||
.source_latency
|
||||
.count,
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_incarnation_cannot_install_or_retain_source_state() {
|
||||
let sys: &'static OnDemandMigrationSys = Box::leak(Box::new(enabled_sys()));
|
||||
let cfg = config(None);
|
||||
assert_eq!(
|
||||
sys.apply_for_incarnation("missing", uuid::Uuid::nil(), Some(&cfg)).await,
|
||||
ApplyOutcome::NotDesired
|
||||
);
|
||||
sys.publish_for_incarnation("missing", uuid::Uuid::nil(), Some(&cfg));
|
||||
assert!(sys.state("missing").is_none());
|
||||
|
||||
sys.apply_for_incarnation("missing", uuid::Uuid::new_v4(), Some(&cfg)).await;
|
||||
let state = sys.state("missing").expect("valid identity installed");
|
||||
sys.publish_for_incarnation("missing", uuid::Uuid::nil(), Some(&cfg));
|
||||
assert!(sys.state("missing").is_none());
|
||||
assert!(state.is_cancelled());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_stored_config_withdraws_runtime_state() {
|
||||
let sys: &'static OnDemandMigrationSys = Box::leak(Box::new(enabled_sys()));
|
||||
let cfg = config(None);
|
||||
assert_eq!(sys.apply("corrupt", Some(&cfg)).await, ApplyOutcome::Installed);
|
||||
let state = sys.state("corrupt").expect("state installed");
|
||||
sys.publish_stored("corrupt", Some((b"not-json", uuid::Uuid::from_u128(1))));
|
||||
assert!(sys.state("corrupt").is_none(), "corruption cannot keep an older source active");
|
||||
assert!(state.is_cancelled(), "corruption cancels in-flight work");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn absent_config_updates_do_not_allocate_bucket_slots() {
|
||||
let sys = enabled_sys();
|
||||
for index in 0..1000 {
|
||||
let bucket = format!("unconfigured-{index}");
|
||||
assert_eq!(sys.apply(&bucket, None).await, ApplyOutcome::NotDesired);
|
||||
assert_eq!(sys.remove(&bucket), ApplyOutcome::NotDesired);
|
||||
}
|
||||
assert!(sys.buckets.read().is_empty(), "unconfigured buckets must not accumulate tombstones");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_install_cannot_overwrite_a_later_removal() {
|
||||
let sys = enabled_sys();
|
||||
let cfg = config(None);
|
||||
let older = sys.reserve_generation("b", true);
|
||||
let newer = sys.reserve_generation("b", false);
|
||||
let older = sys.next_generation();
|
||||
let newer = sys.next_generation();
|
||||
assert_eq!(sys.remove_with_generation("b", newer), ApplyOutcome::NotDesired);
|
||||
assert_eq!(
|
||||
sys.apply_with_generation("b", uuid::Uuid::from_u128(1), Some(&cfg), older)
|
||||
.await,
|
||||
ApplyOutcome::Superseded
|
||||
);
|
||||
assert!(sys.state("b").is_none(), "removal must supersede an in-flight first install");
|
||||
// The removal above did not create a slot; simulate an install that
|
||||
// started before it and finishes after.
|
||||
sys.apply_with_generation("b", Some(&cfg), older).await;
|
||||
assert!(sys.state("b").is_some(), "no slot yet, so the older install lands");
|
||||
|
||||
assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed);
|
||||
let installed = sys.state("b").unwrap();
|
||||
let older = sys.reserve_generation("b", true);
|
||||
let newer = sys.reserve_generation("b", false);
|
||||
let older = sys.next_generation();
|
||||
let newer = sys.next_generation();
|
||||
assert_eq!(sys.remove_with_generation("b", newer), ApplyOutcome::Removed);
|
||||
assert!(installed.is_cancelled());
|
||||
assert_eq!(
|
||||
sys.apply_with_generation("b", uuid::Uuid::from_u128(1), Some(&cfg), older)
|
||||
.await,
|
||||
ApplyOutcome::Superseded
|
||||
);
|
||||
assert_eq!(sys.apply_with_generation("b", Some(&cfg), older).await, ApplyOutcome::Superseded);
|
||||
assert!(sys.state("b").is_none(), "the stale install is discarded");
|
||||
}
|
||||
|
||||
@@ -180,8 +180,6 @@ impl RemoteS3EndpointSpec {
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RemoteS3ClientError {
|
||||
#[error("the {0} backend is not included in this build")]
|
||||
BackendNotCompiled(&'static str),
|
||||
#[error("remote endpoint requires credentials")]
|
||||
MissingCredentials,
|
||||
#[error("{0}")]
|
||||
@@ -283,7 +281,9 @@ impl Intercept for UserAgentSuffixInterceptor {
|
||||
|
||||
/// Builds the SDK config for `spec` without finalizing it, so callers can add
|
||||
/// interceptors or (in tests) swap the HTTP client before `build()`.
|
||||
pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
|
||||
pub(crate) async fn build_remote_s3_config(
|
||||
spec: &RemoteS3EndpointSpec,
|
||||
) -> Result<aws_sdk_s3::config::Builder, RemoteS3ClientError> {
|
||||
let Some(credentials) = &spec.credentials else {
|
||||
return Err(RemoteS3ClientError::MissingCredentials);
|
||||
};
|
||||
@@ -523,7 +523,7 @@ fn validate_ca_pem_bundle(ca_cert_pem: &[u8]) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
|
||||
pub(crate) fn validate_target_ca_pem(ca_cert_pem: &str) -> Result<(), RemoteS3ClientError> {
|
||||
validate_ca_pem_bundle(ca_cert_pem.as_bytes()).map_err(RemoteS3ClientError::InvalidCaPem)
|
||||
}
|
||||
|
||||
|
||||
@@ -3013,7 +3013,7 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<Decommissio
|
||||
.ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?;
|
||||
let id_kind = parts
|
||||
.next()
|
||||
.filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id" | "control_id"))
|
||||
.filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id"))
|
||||
.ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?;
|
||||
let source_path = parts
|
||||
.next()
|
||||
@@ -3023,9 +3023,8 @@ fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<Decommissio
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token")));
|
||||
}
|
||||
match id_kind {
|
||||
"operation_id" | "control_id" if !is_sha256_checksum(id) => {
|
||||
let id_label = id_kind.trim_end_matches("_id");
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid {id_label} id")));
|
||||
"operation_id" if !is_sha256_checksum(id) => {
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid operation id")));
|
||||
}
|
||||
"transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => {
|
||||
return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID")));
|
||||
@@ -19301,8 +19300,8 @@ mod pools_tests {
|
||||
load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done,
|
||||
merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh,
|
||||
missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
|
||||
parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates,
|
||||
read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
|
||||
pool_meta_has_active_decommission, publish_pool_meta_updates, read_pool_meta_replica,
|
||||
reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
|
||||
record_decommission_unresolved_entry, recover_decommission_capacity_reservations,
|
||||
renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers,
|
||||
reserve_decommission_start_target_capacity, resolve_decommission_bucket_state,
|
||||
@@ -20170,26 +20169,6 @@ mod pools_tests {
|
||||
assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_recovery_control_receipt_path_round_trips() {
|
||||
let run_token = "b".repeat(64);
|
||||
let control_id = "a".repeat(64);
|
||||
let source_path = format!(
|
||||
"ilm/recovery-controls/transition_transaction/{}/{}/{}.json",
|
||||
&control_id[..2],
|
||||
&control_id[2..4],
|
||||
control_id
|
||||
);
|
||||
let path = decommission_durable_ilm_receipt_path(&run_token, &source_path, "control_id", &control_id);
|
||||
|
||||
let locator = parse_decommission_durable_ilm_receipt_path(&path).expect("recovery control receipt path should parse");
|
||||
|
||||
assert_eq!(locator.run_token, run_token);
|
||||
assert_eq!(locator.source_path, source_path);
|
||||
assert_eq!(locator.id_kind, "control_id");
|
||||
assert_eq!(locator.id, control_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_receipt_merge_preserves_terminal_proof() {
|
||||
let operation_id = "a".repeat(64);
|
||||
|
||||
@@ -99,17 +99,11 @@ pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK: &str = "reader_open_m
|
||||
pub(crate) const GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS: &str = "reader_open_mmap_copy_success";
|
||||
pub(crate) const GET_STAGE_READER_OPEN_STREAM: &str = "reader_open_stream";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_ACCESS_CHECK: &str = "reader_mmap_access_check";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_TASK: &str = "reader_mmap_blocking_task";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_BLOCKING_WAIT: &str = "reader_mmap_blocking_wait";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_COPY_BUFFER: &str = "reader_mmap_copy_buffer";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_DIRECT_READ_COPY: &str = "reader_mmap_direct_read_copy";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_FILE_OPEN: &str = "reader_mmap_file_open";
|
||||
#[cfg(unix)]
|
||||
pub(crate) const GET_STAGE_READER_MMAP_MAP: &str = "reader_mmap_map";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_METADATA_LOOKUP: &str = "reader_mmap_metadata_lookup";
|
||||
pub(crate) const GET_STAGE_READER_MMAP_METADATA_VALIDATE: &str = "reader_mmap_metadata_validate";
|
||||
|
||||
@@ -324,30 +324,6 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
pub(in crate::disk) async fn undo_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"delete_version",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
// Preserve the old DiskAPI future's boxing boundary.
|
||||
Box::pin(
|
||||
self.disk
|
||||
.undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner),
|
||||
)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
@@ -357,34 +333,6 @@ impl LocalDiskWrapper {
|
||||
dst_path: &str,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::RenameDataObservation {
|
||||
self.rename_data_observed_with_guards(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
super::RenameDataGuards {
|
||||
external_guard,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed_with_guards(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
guards: super::RenameDataGuards,
|
||||
) -> super::RenameDataObservation {
|
||||
let super::RenameDataGuards {
|
||||
external_guard,
|
||||
namespace_owner,
|
||||
..
|
||||
} = guards;
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -409,15 +357,13 @@ impl LocalDiskWrapper {
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await;
|
||||
let observed =
|
||||
Box::pin(
|
||||
operation
|
||||
.disk
|
||||
.rename_data_observed(&src_volume, &src_path, &fi, &dst_volume, &dst_path),
|
||||
)
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
},
|
||||
@@ -1355,7 +1301,6 @@ impl LocalDiskWrapper {
|
||||
self.disk.get_object_path(volume, path)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn get_object_path_for_io(&self, volume: &str, path: &str) -> crate::disk::error::Result<std::path::PathBuf> {
|
||||
self.disk.get_object_path_for_io(volume, path)
|
||||
}
|
||||
|
||||
@@ -218,12 +218,10 @@ pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<
|
||||
fs::rename(from, to).await
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
|
||||
fs::read(path.as_ref()).await
|
||||
|
||||
+1336
-1000
File diff suppressed because it is too large
Load Diff
@@ -17,14 +17,13 @@
|
||||
|
||||
#[cfg(all(test, windows))]
|
||||
use super::run_destination_commit_directory_preparation;
|
||||
#[cfg(any(not(windows), test))]
|
||||
use super::should_fail_local_inline_rollback_hardlink;
|
||||
use super::{
|
||||
EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
|
||||
LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, inline_metadata_rollback_dir, observe_old_current_size,
|
||||
remove_dir_all_if_exists, remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature,
|
||||
run_inline_preparation_before_backup, should_fail_after_metadata_commit, should_fail_before_old_metadata_backup,
|
||||
should_fail_commit_rename, should_remove_staged_meta_before_commit, skip_access_checks,
|
||||
should_fail_commit_rename, should_fail_local_inline_rollback_hardlink, should_remove_staged_meta_before_commit,
|
||||
skip_access_checks,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
|
||||
@@ -34,7 +33,7 @@ use crate::disk::{
|
||||
error::{DiskError, Result},
|
||||
error_conv::{to_access_error, to_file_error},
|
||||
os,
|
||||
os::check_path_length,
|
||||
os::{check_path_length, rename_all},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
@@ -74,8 +73,6 @@ fn rollback_inline_metadata_commit_std(
|
||||
rollback_data_dir: Option<Uuid>,
|
||||
local_rollback_path: Option<&Path>,
|
||||
) -> std::io::Result<()> {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Rollback, dst_file_path);
|
||||
if let Some(backup_path) = local_rollback_path {
|
||||
// The commit immediately before this rollback renamed the staged
|
||||
// xl.meta from the same directory as `backup_path` onto
|
||||
@@ -89,7 +86,6 @@ fn rollback_inline_metadata_commit_std(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
pub(super) fn create_local_inline_rollback_backup(
|
||||
dst_file_path: &Path,
|
||||
staging_file_path: &Path,
|
||||
@@ -235,12 +231,6 @@ async fn restore_published_data_source(
|
||||
#[derive(Debug)]
|
||||
pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[tracing::instrument(name = "rename_data", target = "rustfs_ecstore::disk::local", level = "trace", skip_all)]
|
||||
pub(super) async fn rename_data_inner(
|
||||
@@ -250,7 +240,7 @@ impl LocalDisk {
|
||||
fi: FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
state: &mut RenameDataState,
|
||||
preflight_rejection: &mut Option<LocalRenamePreflightRejection>,
|
||||
) -> Result<RenameDataResp> {
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
let mut fi = fi;
|
||||
@@ -279,13 +269,7 @@ impl LocalDisk {
|
||||
Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
|
||||
None => None,
|
||||
};
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
state.namespace_owner.take(),
|
||||
)
|
||||
.await;
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease(&self.root, dst_volume, &destination_object_path).await;
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -318,7 +302,7 @@ impl LocalDisk {
|
||||
error = %e,
|
||||
"Disk local access check failed"
|
||||
);
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
*preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
@@ -336,7 +320,7 @@ impl LocalDisk {
|
||||
error = %e,
|
||||
"Disk local access check failed"
|
||||
);
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
*preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||
}
|
||||
|
||||
@@ -544,9 +528,7 @@ impl LocalDisk {
|
||||
// rename below.
|
||||
if fi_healing
|
||||
&& let Some((_, dst_data_path)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = self
|
||||
.move_to_trash_with_namespace_owner(dst_data_path, true, false, Some(mutation_lease.clone()))
|
||||
.await
|
||||
&& let Err(err) = self.move_to_trash(dst_data_path, true, false).await
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs_ecstore::disk::local",
|
||||
@@ -773,7 +755,7 @@ impl LocalDisk {
|
||||
&& let Some(parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(mutation_lease.clone())).await {
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit(parent).await {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
|
||||
fsync_started,
|
||||
@@ -811,7 +793,7 @@ impl LocalDisk {
|
||||
break;
|
||||
}
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dir_with_owner(dir, Some(mutation_lease.clone())).await {
|
||||
if let Err(err) = os::fsync_dir(dir).await {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
|
||||
fsync_started,
|
||||
@@ -1042,15 +1024,7 @@ impl LocalDisk {
|
||||
// rename_all acquires the backup path's namespace lease. Do not
|
||||
// hold a disk admission while acquiring another namespace lock.
|
||||
drop(file_sync_admission.take());
|
||||
if let Err(err) = os::rename_all_with_owner(
|
||||
staged_backup,
|
||||
&backup_path,
|
||||
&dst_volume_dir,
|
||||
&self.publication_root,
|
||||
Some(mutation_lease.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1246,18 +1220,14 @@ impl LocalDisk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::super::RenameDataObservation {
|
||||
let mut state = RenameDataState {
|
||||
namespace_owner,
|
||||
..Default::default()
|
||||
};
|
||||
let mut preflight_rejection = None;
|
||||
let result = self
|
||||
.rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut state)
|
||||
.rename_data_inner(src_volume, src_path, fi.clone(), dst_volume, dst_path, &mut preflight_rejection)
|
||||
.await;
|
||||
super::super::RenameDataObservation {
|
||||
result,
|
||||
preflight_rejection: state.preflight_rejection,
|
||||
preflight_rejection,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,14 +75,6 @@ use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Independent admission and physical ownership for one disk rename.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RenameDataGuards {
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
pub(crate) struct RenameDataObservation {
|
||||
pub(crate) result: Result<RenameDataResp>,
|
||||
@@ -198,17 +190,11 @@ pub struct MmapCopyStageMetrics {
|
||||
pub(crate) path_resolve_stage: &'static str,
|
||||
pub(crate) metadata_lookup_stage: &'static str,
|
||||
pub(crate) metadata_validate_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) blocking_wait_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) blocking_task_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) file_open_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) mmap_map_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) mmap_copy_stage: &'static str,
|
||||
#[cfg(unix)]
|
||||
pub(crate) direct_read_copy_stage: &'static str,
|
||||
}
|
||||
|
||||
@@ -732,25 +718,6 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep local undo publication owned independently of the wrapper deadline.
|
||||
/// Remote undo retains its existing RPC contract; this is not a remote drain proof.
|
||||
pub(crate) async fn undo_write_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.undo_write_with_namespace_owner(volume, path, fi, opts, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => disk.delete_version(volume, path, fi, false, opts).await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
@@ -770,12 +737,12 @@ impl Disk {
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
guards: RenameDataGuards,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> RenameDataObservation {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
local_disk
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.rename_data_observed(src_volume, src_path, fi, dst_volume, dst_path, None)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
@@ -786,7 +753,7 @@ impl Disk {
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
@@ -955,7 +922,6 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn get_object_path_for_io_if_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
|
||||
+49
-522
@@ -91,7 +91,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
static RECORDED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static LIMITED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static GROUPED: Mutex<Vec<(PathBuf, usize)>> = Mutex::new(Vec::new());
|
||||
#[cfg(unix)]
|
||||
static BEFORE_LIMITED: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
static BEFORE_GROUP_BATCH: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
@@ -151,7 +150,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn record_limited(dir: &Path) {
|
||||
record_path(&LIMITED, dir, "limited fsync dir recorder");
|
||||
let hook = remove_hook(&BEFORE_LIMITED, dir, "limited fsync hook poisoned");
|
||||
@@ -164,7 +162,6 @@ pub(crate) mod fsync_dir_recorder {
|
||||
contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) {
|
||||
BEFORE_LIMITED
|
||||
.lock()
|
||||
@@ -240,56 +237,11 @@ pub(crate) mod fsync_dir_recorder {
|
||||
.insert(dir.to_path_buf(), kind);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn take_grouped_failure(dir: &Path) -> Option<io::ErrorKind> {
|
||||
remove_path_keyed(&GROUPED_FAILURES, dir, "grouped fsync failure hook poisoned")
|
||||
}
|
||||
}
|
||||
|
||||
/// Pause a real namespace mutation inside its physical executor.
|
||||
#[cfg(all(test, not(windows)))]
|
||||
pub(crate) mod prepared_publication_test_hooks {
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub(crate) enum Stage {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
Rollback,
|
||||
DirFsync,
|
||||
}
|
||||
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
type Key = (Stage, PathBuf);
|
||||
static BEFORE_PUBLICATION: LazyLock<Mutex<HashMap<Key, Hook>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
pub(crate) struct Guard(Key);
|
||||
|
||||
impl Drop for Guard {
|
||||
fn drop(&mut self) {
|
||||
BEFORE_PUBLICATION.lock().remove(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
||||
install_at(Stage::PreparedRename, path, hook)
|
||||
}
|
||||
|
||||
pub(crate) fn install_at(stage: Stage, path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
||||
let key = (stage, path.to_path_buf());
|
||||
assert!(BEFORE_PUBLICATION.lock().insert(key.clone(), Box::new(hook)).is_none());
|
||||
Guard(key)
|
||||
}
|
||||
|
||||
pub(crate) fn run(stage: Stage, path: &Path) {
|
||||
let hook = BEFORE_PUBLICATION.lock().remove(&(stage, path.to_path_buf()));
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, windows))]
|
||||
pub(crate) mod windows_rename_test_hooks {
|
||||
use super::*;
|
||||
@@ -624,7 +576,6 @@ impl OpenedDstDirFsyncGroup {
|
||||
}
|
||||
|
||||
struct DstDirFsyncWaiter {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
result_tx: oneshot::Sender<SharedDstDirFsyncResult>,
|
||||
}
|
||||
|
||||
@@ -683,7 +634,6 @@ impl DstDirFsyncGroupCommit {
|
||||
fn enqueue_opened(
|
||||
&self,
|
||||
opened: OpenedDstDirFsyncGroup,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<(oneshot::Receiver<SharedDstDirFsyncResult>, Option<Arc<DstDirFsyncGroup>>)> {
|
||||
let (result_tx, result_rx) = oneshot::channel();
|
||||
let mut registry = self.inner.lock();
|
||||
@@ -714,10 +664,7 @@ impl DstDirFsyncGroupCommit {
|
||||
group
|
||||
};
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.pending.push_back(DstDirFsyncWaiter {
|
||||
result_tx,
|
||||
namespace_owner,
|
||||
});
|
||||
group_state.pending.push_back(DstDirFsyncWaiter { result_tx });
|
||||
let start_worker = !group_state.worker_running;
|
||||
if start_worker {
|
||||
group_state.worker_running = true;
|
||||
@@ -739,13 +686,7 @@ impl DstDirFsyncGroupCommit {
|
||||
fn remove_idle_group(&self, group: &Arc<DstDirFsyncGroup>) {
|
||||
let mut registry = self.inner.lock();
|
||||
let group_state = group.inner.lock();
|
||||
if !group_state.worker_running
|
||||
&& group_state.pending.is_empty()
|
||||
&& registry
|
||||
.groups
|
||||
.get(&group.key)
|
||||
.is_some_and(|registered| Arc::ptr_eq(registered, group))
|
||||
{
|
||||
if !group_state.worker_running && group_state.pending.is_empty() {
|
||||
registry.groups.remove(&group.key);
|
||||
}
|
||||
}
|
||||
@@ -768,20 +709,16 @@ impl DstDirFsyncGroupCommit {
|
||||
&self,
|
||||
dir: &Path,
|
||||
) -> io::Result<(oneshot::Receiver<SharedDstDirFsyncResult>, Option<Arc<DstDirFsyncGroup>>)> {
|
||||
self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?, None)
|
||||
self.enqueue_opened(OpenedDstDirFsyncGroup::open(dir)?)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
|
||||
#[cfg(test)]
|
||||
let dir = group.dir.clone();
|
||||
let dir_file = group.dir_file.clone();
|
||||
fsync_spawn_blocking(move || {
|
||||
// The batch worker may be cancelled while this syscall is still running.
|
||||
let _namespace_owners = namespace_owners;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &dir);
|
||||
#[cfg(test)]
|
||||
{
|
||||
if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
|
||||
@@ -796,118 +733,66 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Ve
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Vec<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
let _namespace_owners = namespace_owners;
|
||||
async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
|
||||
fsync_dir(&group.dir).await
|
||||
}
|
||||
|
||||
struct DstDirFsyncWorkerGuard {
|
||||
group: Arc<DstDirFsyncGroup>,
|
||||
in_flight: usize,
|
||||
armed: bool,
|
||||
}
|
||||
|
||||
impl Drop for DstDirFsyncWorkerGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.armed {
|
||||
async fn run_dst_dir_fsync_group_worker(group: Arc<DstDirFsyncGroup>) {
|
||||
loop {
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::run_before_group_batch(&group.dir);
|
||||
tokio::task::yield_now().await;
|
||||
let batch: Vec<DstDirFsyncWaiter> = {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.pending.drain(..).collect()
|
||||
};
|
||||
if batch.is_empty() {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.worker_running = false;
|
||||
drop(group_state);
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
return;
|
||||
}
|
||||
// Cancellation must release queued owners, but the physical batch keeps
|
||||
// its own owners until its blocking syscall returns.
|
||||
let pending = {
|
||||
let mut registry = DST_DIR_FSYNC_GROUP_COMMIT.inner.lock();
|
||||
let mut group_state = self.group.inner.lock();
|
||||
let pending = std::mem::take(&mut group_state.pending);
|
||||
group_state.worker_running = false;
|
||||
if registry
|
||||
.groups
|
||||
.get(&self.group.key)
|
||||
.is_some_and(|group| Arc::ptr_eq(group, &self.group))
|
||||
{
|
||||
registry.total_waiters = registry.total_waiters.saturating_sub(pending.len() + self.in_flight);
|
||||
registry.groups.remove(&self.group.key);
|
||||
}
|
||||
pending
|
||||
};
|
||||
// Lease and channel destructors must run outside the registry locks.
|
||||
drop(pending);
|
||||
}
|
||||
}
|
||||
|
||||
fn run_dst_dir_fsync_group_worker(group: Arc<DstDirFsyncGroup>) -> impl std::future::Future<Output = ()> {
|
||||
// Capture before spawning: shutdown may drop the future without polling it.
|
||||
let worker_guard = DstDirFsyncWorkerGuard {
|
||||
group: group.clone(),
|
||||
in_flight: 0,
|
||||
armed: true,
|
||||
};
|
||||
async move {
|
||||
let mut worker_guard = worker_guard;
|
||||
loop {
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::run_before_group_batch(&group.dir);
|
||||
tokio::task::yield_now().await;
|
||||
let mut batch: Vec<DstDirFsyncWaiter> = {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.pending.drain(..).collect()
|
||||
};
|
||||
if batch.is_empty() {
|
||||
let mut group_state = group.inner.lock();
|
||||
worker_guard.armed = false;
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::record_grouped(&group.dir, batch.len());
|
||||
let result = fsync_open_dst_dir_group(&group)
|
||||
.await
|
||||
.map_err(SharedDstDirFsyncError::from_error);
|
||||
let batch_len = batch.len();
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len);
|
||||
|
||||
let should_stop = {
|
||||
let mut group_state = group.inner.lock();
|
||||
if group_state.pending.is_empty() {
|
||||
group_state.worker_running = false;
|
||||
drop(group_state);
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
return;
|
||||
}
|
||||
worker_guard.in_flight = batch.len();
|
||||
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::record_grouped(&group.dir, batch.len());
|
||||
let namespace_owners = batch.iter_mut().filter_map(|waiter| waiter.namespace_owner.take()).collect();
|
||||
let result = fsync_open_dst_dir_group(&group, namespace_owners)
|
||||
.await
|
||||
.map_err(SharedDstDirFsyncError::from_error);
|
||||
let batch_len = batch.len();
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.complete_batch(batch_len);
|
||||
worker_guard.in_flight = 0;
|
||||
|
||||
let should_stop = {
|
||||
let mut group_state = group.inner.lock();
|
||||
if group_state.pending.is_empty() {
|
||||
worker_guard.armed = false;
|
||||
group_state.worker_running = false;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_stop {
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
}
|
||||
for waiter in batch {
|
||||
let _ = waiter.result_tx.send(result.clone());
|
||||
}
|
||||
if should_stop {
|
||||
return;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_stop {
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
}
|
||||
for waiter in batch {
|
||||
let _ = waiter.result_tx.send(result.clone());
|
||||
}
|
||||
if should_stop {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn fsync_dst_dir_group_commit_with_enabled(
|
||||
dir: impl AsRef<Path>,
|
||||
enabled: bool,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
async fn fsync_dst_dir_group_commit_with_enabled(dir: impl AsRef<Path>, enabled: bool) -> io::Result<()> {
|
||||
if !enabled {
|
||||
return fsync_dir_with_owner(dir.as_ref(), namespace_owner).await;
|
||||
return fsync_dir(dir).await;
|
||||
}
|
||||
|
||||
let dir = dir.as_ref().to_path_buf();
|
||||
let opened = tokio::task::spawn_blocking(move || OpenedDstDirFsyncGroup::open(&dir))
|
||||
.await
|
||||
.map_err(|err| io::Error::other(format!("blocking dst dir group open failed: {err}")))??;
|
||||
let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened, namespace_owner)?;
|
||||
let (result_rx, worker) = DST_DIR_FSYNC_GROUP_COMMIT.enqueue_opened(opened)?;
|
||||
if let Some(group) = worker {
|
||||
tokio::spawn(run_dst_dir_fsync_group_worker(group));
|
||||
}
|
||||
@@ -919,11 +804,8 @@ async fn fsync_dst_dir_group_commit_with_enabled(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dst_dir_group_commit(
|
||||
dir: impl AsRef<Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled(), namespace_owner).await
|
||||
pub(crate) async fn fsync_dst_dir_group_commit(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, dst_dir_fsync_group_commit_enabled()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
@@ -932,7 +814,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
admission: &FileSyncAdmission,
|
||||
) -> io::Result<()> {
|
||||
if dst_dir_fsync_group_commit_enabled() {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, true, Some(lease)).await
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, true).await
|
||||
} else {
|
||||
fsync_dir_with_namespace_file_sync_limit(dir, lease, admission).await
|
||||
}
|
||||
@@ -940,7 +822,7 @@ pub(crate) async fn fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn fsync_dst_dir_group_commit_for_test(dir: impl AsRef<Path>, enabled: bool) -> io::Result<()> {
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, enabled, None).await
|
||||
fsync_dst_dir_group_commit_with_enabled(dir, enabled).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1347,8 +1229,6 @@ pub(crate) struct NamespaceMutationLease {
|
||||
_namespace_guard: OwnedMutexGuard<()>,
|
||||
_volume_guard: Option<OwnedRwLockReadGuard<()>>,
|
||||
external_guard: Mutex<Option<Arc<dyn Send + Sync>>>,
|
||||
// Independent of the quota claim; both survive cancellation of the waiter.
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
impl NamespaceMutationLease {
|
||||
@@ -1358,18 +1238,10 @@ impl NamespaceMutationLease {
|
||||
}
|
||||
|
||||
async fn acquire_namespace_mutation_lease(path: &Path) -> Arc<NamespaceMutationLease> {
|
||||
acquire_namespace_mutation_lease_with_owner(path, None).await
|
||||
}
|
||||
|
||||
async fn acquire_namespace_mutation_lease_with_owner(
|
||||
path: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
Arc::new(NamespaceMutationLease {
|
||||
_namespace_guard: disk_namespace_mutation_lock(path).lock_owned().await,
|
||||
_volume_guard: None,
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1379,15 +1251,6 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
|
||||
root: &Path,
|
||||
volume: &str,
|
||||
destination_object: &Path,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
acquire_rename_data_mutation_lease_with_owner(root, volume, destination_object, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
|
||||
root: &Path,
|
||||
volume: &str,
|
||||
destination_object: &Path,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<NamespaceMutationLease> {
|
||||
let namespace_guard = disk_namespace_mutation_lock(destination_object).lock_owned().await;
|
||||
let volume_guard = disk_volume_mutation_lock(root, volume).read_owned().await;
|
||||
@@ -1395,7 +1258,6 @@ pub(crate) async fn acquire_rename_data_mutation_lease_with_owner(
|
||||
_namespace_guard: namespace_guard,
|
||||
_volume_guard: Some(volume_guard),
|
||||
external_guard: Mutex::new(None),
|
||||
_namespace_owner: namespace_owner,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1885,69 +1747,6 @@ pub async fn rename_all(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dir_with_owner(path: &Path, namespace_owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
if namespace_owner.is_none() {
|
||||
return fsync_dir(path).await;
|
||||
}
|
||||
let path = path.to_path_buf();
|
||||
fsync_spawn_blocking(move || {
|
||||
let _namespace_owner = namespace_owner;
|
||||
fsync_dir_std(path)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = namespace_owner;
|
||||
fsync_dir(path).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Retain namespace ownership in the actual filesystem executor after timeout.
|
||||
pub(crate) async fn remove_file_with_owner(
|
||||
path: impl AsRef<Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::remove_file(path).await;
|
||||
}
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(test, not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
std::fs::remove_file(path)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Retain namespace ownership in the actual filesystem executor after timeout.
|
||||
pub(crate) async fn remove_dir_with_owner(
|
||||
path: impl AsRef<Path>,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> io::Result<()> {
|
||||
if namespace_owner.is_none() {
|
||||
return tokio::fs::remove_dir(path).await;
|
||||
}
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
|
||||
pub(crate) async fn rename_all_with_owner(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
base_dir: impl AsRef<Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
|
||||
}
|
||||
|
||||
pub(crate) async fn rename_all_with_lease(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
@@ -2140,8 +1939,6 @@ pub(crate) async fn rename_all_with_prepared_source(
|
||||
move || {
|
||||
validate_prepared_rename_source(&prepared_source, &src_file_path)?;
|
||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
|
||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||
}
|
||||
};
|
||||
@@ -2180,32 +1977,6 @@ pub async fn rename_all_ignore_missing_source(
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(name = "rename_all_ignore_missing_source", level = "debug", skip_all)]
|
||||
pub(crate) async fn rename_all_ignore_missing_source_with_owner(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
base_dir: impl AsRef<Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let src_file_path = src_file_path.as_ref();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
|
||||
match reliable_rename_inner_with_lease(
|
||||
src_file_path.to_path_buf(),
|
||||
dst_file_path.as_ref().to_path_buf(),
|
||||
base_dir.as_ref().to_path_buf(),
|
||||
publication_root.clone(),
|
||||
false,
|
||||
lease,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound && rename_source_is_missing(src_file_path, publication_root) => Ok(()),
|
||||
Err(err) => Err(to_file_error(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn rename_source_is_missing(src_file_path: &Path, publication_root: &PublicationRoot) -> bool {
|
||||
let Some(source_parent) = src_file_path.parent() else {
|
||||
@@ -2271,11 +2042,6 @@ async fn reliable_rename_inner_with_lease(
|
||||
let base_dir = base_dir.clone();
|
||||
move || {
|
||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||
#[cfg(all(test, not(windows)))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
|
||||
}
|
||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||
}
|
||||
};
|
||||
@@ -6370,245 +6136,6 @@ mod tests {
|
||||
wait_for_dst_dir_fsync_group_commit_idle().await;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn grouped_fsync_physical_batch_keeps_all_owners_after_worker_cancellation() {
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let dir = temp_dir.path().canonicalize().expect("canonical fsync path");
|
||||
let first_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let second_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let first_owner = first_ctx.begin_namespace_commit();
|
||||
let second_owner = second_ctx.begin_namespace_commit();
|
||||
let first_probe = Arc::downgrade(&first_owner);
|
||||
let second_probe = Arc::downgrade(&second_owner);
|
||||
let (first_rx, group) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(&dir).expect("open first waiter directory"),
|
||||
Some(first_owner),
|
||||
)
|
||||
.expect("queue first real waiter");
|
||||
let group = group.expect("first waiter starts the group");
|
||||
let (second_rx, second_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(&dir).expect("open second waiter directory"),
|
||||
Some(second_owner),
|
||||
)
|
||||
.expect("queue second real waiter");
|
||||
assert!(second_worker.is_none(), "same directory must join the same batch");
|
||||
assert_eq!(group.inner.lock().pending.len(), 2);
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let _hook =
|
||||
prepared_publication_test_hooks::install_at(prepared_publication_test_hooks::Stage::DirFsync, &dir, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let worker = tokio::spawn(run_dst_dir_fsync_group_worker(group.clone()));
|
||||
tokio::time::timeout(Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("batch must reach its physical fsync")
|
||||
.expect("physical fsync entry");
|
||||
assert_eq!(fsync_dir_recorder::grouped_batch_sizes(&dir), vec![2]);
|
||||
assert!(
|
||||
group.inner.lock().pending.is_empty(),
|
||||
"both waiters were transferred into the physical batch"
|
||||
);
|
||||
let queued_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let queued_owner = queued_ctx.begin_namespace_commit();
|
||||
let queued_probe = Arc::downgrade(&queued_owner);
|
||||
let queued_generation = queued_ctx.namespace_commit_generation();
|
||||
let (queued_rx, queued_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(&dir).expect("open queued waiter directory"),
|
||||
Some(queued_owner),
|
||||
)
|
||||
.expect("queue a waiter after the physical batch was frozen");
|
||||
assert!(queued_worker.is_none());
|
||||
assert_eq!(group.inner.lock().pending.len(), 1);
|
||||
drop((first_rx, second_rx));
|
||||
worker.abort();
|
||||
assert!(worker.await.expect_err("cancel the async batch owner").is_cancelled());
|
||||
assert!(queued_rx.await.is_err(), "an undispatched waiter must observe worker cancellation");
|
||||
assert!(queued_probe.upgrade().is_none());
|
||||
assert!(!queued_ctx.namespace_commits_pending());
|
||||
assert!(queued_ctx.namespace_commit_generation() > queued_generation);
|
||||
assert!(group.inner.lock().pending.is_empty());
|
||||
assert!(!group.inner.lock().worker_running);
|
||||
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
|
||||
let first_pending = first_ctx.namespace_commits_pending() && first_probe.upgrade().is_some();
|
||||
let second_pending = second_ctx.namespace_commits_pending() && second_probe.upgrade().is_some();
|
||||
let generations = (first_ctx.namespace_commit_generation(), second_ctx.namespace_commit_generation());
|
||||
drop(release_tx);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while Arc::strong_count(&group.dir_file) != 1
|
||||
|| first_probe.upgrade().is_some()
|
||||
|| second_probe.upgrade().is_some()
|
||||
|| first_ctx.namespace_commits_pending()
|
||||
|| second_ctx.namespace_commits_pending()
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical fsync must release every batch owner");
|
||||
assert!(fsync_dir_recorder::was_fsynced(&dir), "the detached syscall must really execute");
|
||||
assert!(
|
||||
first_pending && second_pending,
|
||||
"one physical batch must preserve both independent namespace owners"
|
||||
);
|
||||
assert!(!first_ctx.namespace_commits_pending());
|
||||
assert!(!second_ctx.namespace_commits_pending());
|
||||
assert!(first_ctx.namespace_commit_generation() > generations.0);
|
||||
assert!(second_ctx.namespace_commit_generation() > generations.1);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn grouped_fsync_unpolled_worker_releases_queued_owner() {
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let owner = ctx.begin_namespace_commit();
|
||||
let probe = Arc::downgrade(&owner);
|
||||
let generation = ctx.namespace_commit_generation();
|
||||
let (rx, group) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open queued waiter directory"),
|
||||
Some(owner),
|
||||
)
|
||||
.expect("queue a real waiter");
|
||||
let group = group.expect("first waiter starts the group");
|
||||
let worker = run_dst_dir_fsync_group_worker(group.clone());
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
drop(worker);
|
||||
assert!(rx.await.is_err(), "shutdown before first poll must release the waiter");
|
||||
assert!(probe.upgrade().is_none());
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
assert!(ctx.namespace_commit_generation() > generation);
|
||||
assert!(group.inner.lock().pending.is_empty());
|
||||
assert!(!group.inner.lock().worker_running);
|
||||
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
|
||||
assert!(
|
||||
fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()).is_empty(),
|
||||
"the dropped future must not dispatch a physical batch"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn stale_idle_group_cleanup_preserves_successor_registration() {
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let registry = DstDirFsyncGroupCommit::default();
|
||||
let (mut first_rx, first_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue first worker");
|
||||
let old_group = first_worker.expect("first waiter starts a worker");
|
||||
// W1 has completed its batch and marked G idle, but has not cleaned G up.
|
||||
let first_waiter = old_group.inner.lock().pending.pop_front().expect("first batch waiter");
|
||||
registry.complete_batch(1);
|
||||
old_group.inner.lock().worker_running = false;
|
||||
|
||||
let (mut second_rx, second_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue second worker");
|
||||
let reused_group = second_worker.expect("idle G starts another worker");
|
||||
assert!(Arc::ptr_eq(&old_group, &reused_group));
|
||||
let second_waiter = reused_group.inner.lock().pending.pop_front().expect("second batch waiter");
|
||||
registry.complete_batch(1);
|
||||
reused_group.inner.lock().worker_running = false;
|
||||
registry.remove_idle_group(&reused_group);
|
||||
assert_eq!(registry.counts_for_test(), (0, 0), "normal idle cleanup must remove G");
|
||||
assert!(second_waiter.result_tx.send(Ok(())).is_ok());
|
||||
assert!(second_rx.try_recv().expect("second worker reports completion").is_ok());
|
||||
|
||||
let (mut successor_rx, successor_worker) = registry.enqueue_for_test(temp_dir.path()).expect("enqueue successor");
|
||||
let successor = successor_worker.expect("successor starts a new group");
|
||||
assert!(!Arc::ptr_eq(&old_group, &successor));
|
||||
assert_eq!(registry.counts_for_test(), (1, 1));
|
||||
// W1 resumes with its old Arc after W2 removed G and W3 installed G2.
|
||||
registry.remove_idle_group(&old_group);
|
||||
assert!(first_waiter.result_tx.send(Ok(())).is_ok());
|
||||
assert!(first_rx.try_recv().expect("first worker reports completion").is_ok());
|
||||
assert!(
|
||||
registry
|
||||
.inner
|
||||
.lock()
|
||||
.groups
|
||||
.get(&successor.key)
|
||||
.is_some_and(|registered| Arc::ptr_eq(registered, &successor)),
|
||||
"stale cleanup must retain the exact successor Arc"
|
||||
);
|
||||
assert_eq!(registry.counts_for_test(), (1, 1));
|
||||
assert!(successor.inner.lock().worker_running);
|
||||
assert_eq!(successor.inner.lock().pending.len(), 1);
|
||||
assert!(matches!(successor_rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)));
|
||||
|
||||
let (_joined_rx, new_worker) = registry.enqueue_for_test(temp_dir.path()).expect("join successor");
|
||||
assert!(new_worker.is_none(), "a later waiter must join G2 instead of creating G3");
|
||||
assert_eq!(successor.inner.lock().pending.len(), 2);
|
||||
assert_eq!(registry.counts_for_test(), (1, 2));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn stale_idle_cleanup_then_unpolled_worker_drop_releases_waiter_budget() {
|
||||
wait_for_dst_dir_fsync_group_commit_idle().await;
|
||||
let temp_dir = tempdir().expect("fixture directory");
|
||||
let (old_rx, old_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_for_test(temp_dir.path())
|
||||
.expect("enqueue old group");
|
||||
let old_group = old_worker.expect("old group starts a worker");
|
||||
tokio::time::timeout(Duration::from_secs(5), run_dst_dir_fsync_group_worker(old_group.clone()))
|
||||
.await
|
||||
.expect("old worker must finish its actual fsync");
|
||||
assert!(old_rx.await.expect("old worker reports completion").is_ok());
|
||||
assert!(fsync_dir_recorder::was_fsynced(temp_dir.path()));
|
||||
assert_eq!(fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()), vec![1]);
|
||||
assert_eq!(DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test(), (0, 0));
|
||||
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let owner = ctx.begin_namespace_commit();
|
||||
let probe = Arc::downgrade(&owner);
|
||||
let generation = ctx.namespace_commit_generation();
|
||||
let (rx, successor_worker) = DST_DIR_FSYNC_GROUP_COMMIT
|
||||
.enqueue_opened(
|
||||
OpenedDstDirFsyncGroup::open(temp_dir.path()).expect("open successor directory"),
|
||||
Some(owner),
|
||||
)
|
||||
.expect("enqueue successor owner");
|
||||
let successor = successor_worker.expect("successor starts a new group");
|
||||
assert!(!Arc::ptr_eq(&old_group, &successor));
|
||||
let worker = run_dst_dir_fsync_group_worker(successor.clone());
|
||||
// The stale Arc represents W1 resuming after another worker removed G.
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.remove_idle_group(&old_group);
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert!(probe.upgrade().is_some());
|
||||
drop(worker);
|
||||
let channel_closed = tokio::time::timeout(Duration::from_secs(5), rx)
|
||||
.await
|
||||
.expect("dropping the unpolled worker must release its channel")
|
||||
.is_err();
|
||||
let counts_after_drop = DST_DIR_FSYNC_GROUP_COMMIT.counts_for_test();
|
||||
let owner_released = probe.upgrade().is_none();
|
||||
let namespace_pending = ctx.namespace_commits_pending();
|
||||
let generation_after_drop = ctx.namespace_commit_generation();
|
||||
let successor_pending = successor.inner.lock().pending.len();
|
||||
let worker_running = successor.inner.lock().worker_running;
|
||||
// Preserve the observed result before cleanup, so a RED run cannot leak
|
||||
// its phantom count into unrelated tests in the same process.
|
||||
clear_dst_dir_fsync_group_commit_for_test();
|
||||
assert!(channel_closed);
|
||||
assert!(owner_released);
|
||||
assert!(!namespace_pending);
|
||||
assert!(generation_after_drop > generation);
|
||||
assert_eq!(successor_pending, 0);
|
||||
assert!(!worker_running);
|
||||
assert_eq!(
|
||||
fsync_dir_recorder::grouped_batch_sizes(temp_dir.path()),
|
||||
vec![1],
|
||||
"dropping the successor before its first poll must not dispatch another fsync"
|
||||
);
|
||||
assert_eq!(counts_after_drop, (0, 0), "stale cleanup must not strand a phantom waiter");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(dst_dir_fsync_group_commit)]
|
||||
async fn dst_dir_fsync_group_commit_cancellation_releases_waiter_state() {
|
||||
|
||||
@@ -13,15 +13,12 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::diagnostics::get::{
|
||||
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
GET_STAGE_READER_MMAP_ACCESS_CHECK, GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT,
|
||||
GET_STAGE_READER_MMAP_COPY_BUFFER, GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN,
|
||||
GET_STAGE_READER_MMAP_MAP, GET_STAGE_READER_MMAP_METADATA_LOOKUP, GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
|
||||
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
#[cfg(unix)]
|
||||
use crate::diagnostics::get::{
|
||||
GET_STAGE_READER_MMAP_BLOCKING_TASK, GET_STAGE_READER_MMAP_BLOCKING_WAIT, GET_STAGE_READER_MMAP_COPY_BUFFER,
|
||||
GET_STAGE_READER_MMAP_DIRECT_READ_COPY, GET_STAGE_READER_MMAP_FILE_OPEN, GET_STAGE_READER_MMAP_MAP,
|
||||
};
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::disk::FileWriter;
|
||||
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
|
||||
@@ -409,17 +406,11 @@ async fn open_disk_reader(
|
||||
path_resolve_stage: GET_STAGE_READER_MMAP_PATH_RESOLVE,
|
||||
metadata_lookup_stage: GET_STAGE_READER_MMAP_METADATA_LOOKUP,
|
||||
metadata_validate_stage: GET_STAGE_READER_MMAP_METADATA_VALIDATE,
|
||||
#[cfg(unix)]
|
||||
blocking_wait_stage: GET_STAGE_READER_MMAP_BLOCKING_WAIT,
|
||||
#[cfg(unix)]
|
||||
blocking_task_stage: GET_STAGE_READER_MMAP_BLOCKING_TASK,
|
||||
#[cfg(unix)]
|
||||
file_open_stage: GET_STAGE_READER_MMAP_FILE_OPEN,
|
||||
#[cfg(unix)]
|
||||
mmap_map_stage: GET_STAGE_READER_MMAP_MAP,
|
||||
#[cfg(unix)]
|
||||
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
|
||||
#[cfg(unix)]
|
||||
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
|
||||
});
|
||||
let mmap_result = {
|
||||
|
||||
@@ -956,9 +956,6 @@ pub struct ObjectOptions {
|
||||
pub preserve_etag: Option<String>,
|
||||
pub metadata_chg: bool,
|
||||
pub http_preconditions: Option<HTTPPreconditions>,
|
||||
/// Internal create-only writes may also preserve an acknowledged deletion.
|
||||
/// Evaluated with `http_preconditions` under the namespace commit lock.
|
||||
pub preserve_delete_marker: bool,
|
||||
|
||||
pub delete_replication: Option<ReplicationState>,
|
||||
pub delete_replication_config_snapshot: Option<Arc<DeleteReplicationConfigSnapshot>>,
|
||||
|
||||
@@ -78,21 +78,6 @@ pub(crate) struct ScannerPublicationLeaseEntry {
|
||||
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
|
||||
}
|
||||
|
||||
pub(crate) struct NamespaceCommitGuard {
|
||||
ctx: Arc<InstanceContext>,
|
||||
counted: bool,
|
||||
}
|
||||
|
||||
impl Drop for NamespaceCommitGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.counted {
|
||||
// Publish the new generation before a zero-pending publication probe.
|
||||
self.ctx.advance_namespace_commit_generation();
|
||||
self.ctx.namespace_commits.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime state owned by a single `ECStore` instance.
|
||||
///
|
||||
/// This is intentionally minimal in the first migration slice; subsequent
|
||||
@@ -224,13 +209,9 @@ pub struct InstanceContext {
|
||||
/// Last storage-owned movement snapshot observed under the operation
|
||||
/// gate. SetDisks cache writers fail closed until ECStore refreshes it.
|
||||
scanner_publication_state: AtomicU8,
|
||||
namespace_commits: AtomicU64,
|
||||
namespace_commit_generation: AtomicU64,
|
||||
/// Resolves object-encryption material at the application boundary.
|
||||
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
#[cfg(test)]
|
||||
suppress_tier_delete_journal_recovery: bool,
|
||||
transition_transaction_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
tier_delete_journal_recovery_wakeup: tokio::sync::Notify,
|
||||
}
|
||||
@@ -275,12 +256,8 @@ impl InstanceContext {
|
||||
data_movement_generation_exhausted: AtomicBool::new(false),
|
||||
data_movement_generation_notify: Arc::new(Notify::new()),
|
||||
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
|
||||
namespace_commits: AtomicU64::new(0),
|
||||
namespace_commit_generation: AtomicU64::new(0),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
#[cfg(test)]
|
||||
suppress_tier_delete_journal_recovery: false,
|
||||
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
tier_delete_journal_recovery_wakeup: tokio::sync::Notify::new(),
|
||||
}
|
||||
@@ -408,36 +385,6 @@ impl InstanceContext {
|
||||
&& self.scanner_publication_state.load(Ordering::Acquire) == SCANNER_PUBLICATION_STATE_ALLOWED
|
||||
}
|
||||
|
||||
pub(crate) fn begin_namespace_commit(self: &Arc<Self>) -> Arc<NamespaceCommitGuard> {
|
||||
let counted = self
|
||||
.namespace_commits
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1))
|
||||
.is_ok();
|
||||
if counted {
|
||||
self.advance_namespace_commit_generation();
|
||||
} else {
|
||||
self.namespace_commit_generation.store(u64::MAX, Ordering::Release);
|
||||
}
|
||||
Arc::new(NamespaceCommitGuard {
|
||||
ctx: Arc::clone(self),
|
||||
counted,
|
||||
})
|
||||
}
|
||||
|
||||
fn advance_namespace_commit_generation(&self) {
|
||||
let _ = self
|
||||
.namespace_commit_generation
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |generation| Some(generation.saturating_add(1)));
|
||||
}
|
||||
|
||||
pub(crate) fn namespace_commit_generation(&self) -> u64 {
|
||||
self.namespace_commit_generation.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn namespace_commits_pending(&self) -> bool {
|
||||
self.namespace_commits.load(Ordering::Acquire) != 0 || self.namespace_commit_generation() == u64::MAX
|
||||
}
|
||||
|
||||
pub(crate) fn set_scanner_publication_state(&self, blocked: bool) {
|
||||
self.scanner_publication_state.store(
|
||||
if blocked {
|
||||
@@ -693,21 +640,12 @@ impl InstanceContext {
|
||||
}
|
||||
|
||||
pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool {
|
||||
#[cfg(test)]
|
||||
if self.suppress_tier_delete_journal_recovery {
|
||||
return false;
|
||||
}
|
||||
self.tier_delete_journal_recovery_stores
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(store_id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn suppress_tier_delete_journal_recovery_for_test(&mut self) {
|
||||
self.suppress_tier_delete_journal_recovery = true;
|
||||
}
|
||||
|
||||
pub(crate) fn mark_transition_transaction_recovery_started(&self, store_id: Uuid) -> bool {
|
||||
self.transition_transaction_recovery_stores
|
||||
.lock()
|
||||
@@ -818,50 +756,6 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn namespace_commit_guards_are_instance_local_and_count_until_last_owner() {
|
||||
let first = Arc::new(InstanceContext::new());
|
||||
let other = Arc::new(InstanceContext::new());
|
||||
first.set_scanner_publication_state(false);
|
||||
other.set_scanner_publication_state(false);
|
||||
assert!(first.scanner_publication_state_allowed());
|
||||
let one = first.begin_namespace_commit();
|
||||
let shared_owner = Arc::clone(&one);
|
||||
let two = first.begin_namespace_commit();
|
||||
assert!(first.namespace_commits_pending());
|
||||
assert!(first.scanner_publication_state_allowed(), "pending writes must not block scan admission");
|
||||
assert_eq!(first.namespace_commit_generation(), 2);
|
||||
assert!(!other.namespace_commits_pending());
|
||||
assert_eq!(other.namespace_commit_generation(), 0);
|
||||
assert!(other.scanner_publication_state_allowed());
|
||||
drop(one);
|
||||
assert_eq!(first.namespace_commit_generation(), 2);
|
||||
drop(shared_owner);
|
||||
assert!(first.namespace_commits_pending());
|
||||
assert_eq!(first.namespace_commit_generation(), 3);
|
||||
drop(two);
|
||||
assert!(!first.namespace_commits_pending());
|
||||
assert_eq!(first.namespace_commit_generation(), 4);
|
||||
assert!(first.scanner_publication_state_allowed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_commit_counter_exhaustion_keeps_publication_blocked() {
|
||||
for (count, generation) in [(0, u64::MAX - 1), (u64::MAX, 0)] {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.set_scanner_publication_state(false);
|
||||
ctx.namespace_commits.store(count, Ordering::Release);
|
||||
ctx.namespace_commit_generation.store(generation, Ordering::Release);
|
||||
let guard = ctx.begin_namespace_commit();
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
|
||||
drop(guard);
|
||||
assert!(ctx.namespace_commits_pending());
|
||||
assert_eq!(ctx.namespace_commit_generation(), u64::MAX);
|
||||
assert_eq!(ctx.namespace_commits.load(Ordering::Acquire), count);
|
||||
}
|
||||
}
|
||||
|
||||
// The SetupType inputs must derive the exact (is_erasure,
|
||||
// is_dist_erasure, is_erasure_sd) triples that the original three
|
||||
// process-global erasure bools produced via update_erasure_type().
|
||||
@@ -1179,12 +1073,6 @@ mod tests {
|
||||
assert!(!ctx_a.mark_tier_delete_journal_recovery_started(store_a));
|
||||
assert!(ctx_a.mark_tier_delete_journal_recovery_started(store_b));
|
||||
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_a));
|
||||
|
||||
let mut manual_ctx = InstanceContext::new();
|
||||
manual_ctx.suppress_tier_delete_journal_recovery_for_test();
|
||||
assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_a));
|
||||
assert!(!manual_ctx.mark_tier_delete_journal_recovery_started(store_b));
|
||||
assert!(ctx_b.mark_tier_delete_journal_recovery_started(store_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -25,7 +25,6 @@ pub(crate) mod tier_probe_intent;
|
||||
pub mod warm_backend;
|
||||
pub mod warm_backend_aliyun;
|
||||
pub mod warm_backend_azure;
|
||||
#[cfg(feature = "gcs")]
|
||||
pub mod warm_backend_gcs;
|
||||
pub mod warm_backend_huaweicloud;
|
||||
pub mod warm_backend_minio;
|
||||
|
||||
@@ -56,7 +56,6 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
#[cfg(feature = "test-util")]
|
||||
use std::path::Path;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
@@ -69,28 +68,21 @@ use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Mutex, Notify, RwLock};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::format::FormatV3;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
|
||||
use crate::services::tier::warm_backend::{
|
||||
TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_s3_client::transition_api::{ReadCloser, ReaderImpl};
|
||||
#[cfg(feature = "test-util")]
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
|
||||
/// One-shot barrier before rejected transition cleanup resolves its ECStore.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct TransitionCleanupStoreBarrier(crate::set_disk::SetDiskTransitionCleanupStoreBarrier);
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl TransitionCleanupStoreBarrier {
|
||||
/// Install the barrier for the next rejected transition cleanup.
|
||||
pub fn install() -> Self {
|
||||
@@ -104,7 +96,6 @@ impl TransitionCleanupStoreBarrier {
|
||||
}
|
||||
|
||||
/// Default polling cadence used by the `wait_for_*` helpers.
|
||||
#[cfg(feature = "test-util")]
|
||||
const POLL_INTERVAL: Duration = Duration::from_millis(50);
|
||||
|
||||
/// A fault to inject into [`MockWarmBackend`] operations.
|
||||
@@ -217,12 +208,10 @@ impl Drop for MockRemoveOperationGuard {
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses a mock tier PUT after storing its remote body.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct MockPutBarrier {
|
||||
state: Arc<MockPutBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl MockPutBarrier {
|
||||
/// Wait until the remote body is stored and the PUT is paused before returning.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
@@ -237,7 +226,6 @@ impl MockPutBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl Drop for MockPutBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -270,12 +258,10 @@ impl Drop for MockGetBarrier {
|
||||
}
|
||||
|
||||
/// One-shot barrier that pauses and then fails a mock tier DELETE.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct MockRemoveBarrier {
|
||||
state: Arc<MockRemoveBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl MockRemoveBarrier {
|
||||
/// Wait until DELETE reaches the deterministic failure point.
|
||||
pub async fn wait_until_paused(&self) {
|
||||
@@ -297,7 +283,6 @@ impl MockRemoveBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
impl Drop for MockRemoveBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
@@ -321,7 +306,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Arm a one-shot pause after the next tier PUT stores its remote body.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_put_barrier(&self) -> MockPutBarrier {
|
||||
let state = Arc::new(MockPutBarrierState::default());
|
||||
*self.inner.put_barrier.lock().await = Some(Arc::clone(&state));
|
||||
@@ -329,7 +313,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Pause and then fail the next DELETE after it reaches the backend.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_failing_remove_barrier(&self) -> MockRemoveBarrier {
|
||||
let state = Arc::new(MockRemoveBarrierState::default());
|
||||
let mut barrier = self.inner.remove_barrier.lock().await;
|
||||
@@ -340,7 +323,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Arm a one-shot pause before the next tier GET, then return an error
|
||||
/// after the test releases it.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn arm_failing_get_barrier(&self) -> MockGetBarrier {
|
||||
let state = Arc::new(MockGetBarrierState {
|
||||
fail_after_release: true,
|
||||
@@ -361,7 +343,6 @@ impl MockWarmBackend {
|
||||
// ---- fault injection -------------------------------------------------
|
||||
|
||||
/// Replace the entire fault configuration.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_faults(&self, faults: FaultConfig) {
|
||||
*self.inner.faults.lock().await = faults;
|
||||
}
|
||||
@@ -372,7 +353,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Toggle "HTTP 5xx" server errors on every operation.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_server_error(&self, server_error: bool) {
|
||||
self.inner.faults.lock().await.server_error = server_error;
|
||||
}
|
||||
@@ -383,13 +363,11 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Set (or clear, with `None`) injected latency applied before each op.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_latency(&self, latency: Option<Duration>) {
|
||||
self.inner.faults.lock().await.latency = latency;
|
||||
}
|
||||
|
||||
/// Clear all injected faults, restoring healthy behaviour.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn clear_faults(&self) {
|
||||
*self.inner.faults.lock().await = FaultConfig::default();
|
||||
}
|
||||
@@ -397,7 +375,6 @@ impl MockWarmBackend {
|
||||
/// Limit how many body bytes a successful mock PUT consumes. `None` drains
|
||||
/// the complete body. This models a backend that incorrectly accepts a
|
||||
/// truncated stream while still returning success.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn set_put_read_limit(&self, limit: Option<usize>) {
|
||||
*self.inner.put_read_limit.lock().await = limit;
|
||||
}
|
||||
@@ -418,14 +395,12 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Reject non-empty remote versions before transition metadata is committed.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub fn set_reject_non_empty_remote_versions(&self, reject: bool) {
|
||||
self.inner.reject_non_empty_remote_versions.store(reject, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Reject the next non-empty remote version validation without changing
|
||||
/// subsequent exact-version backend cleanup behavior.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub fn reject_next_non_empty_remote_version_validation(&self) {
|
||||
self.inner
|
||||
.reject_non_empty_remote_version_validations
|
||||
@@ -463,7 +438,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Clear the operation log without touching stored objects or faults.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn clear_op_log(&self) {
|
||||
self.inner.op_log.lock().await.clear();
|
||||
}
|
||||
@@ -485,13 +459,11 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs produced by successful tier PUTs.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn put_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.put_versions.lock().await.clone()
|
||||
}
|
||||
|
||||
/// Return the exact object/version pairs passed to successful tier removes.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn remove_versions(&self) -> Vec<(String, String)> {
|
||||
self.inner.remove_versions.lock().await.clone()
|
||||
}
|
||||
@@ -503,7 +475,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Number of `get` calls recorded — useful to assert restore reads hit the
|
||||
/// local copy rather than the remote tier.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn get_count(&self) -> usize {
|
||||
self.inner
|
||||
.op_log
|
||||
@@ -515,7 +486,6 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// Number of `put` calls recorded.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn put_count(&self) -> usize {
|
||||
self.inner
|
||||
.op_log
|
||||
@@ -529,7 +499,6 @@ impl MockWarmBackend {
|
||||
// ---- storage inspection ---------------------------------------------
|
||||
|
||||
/// Whether the backend currently stores `object`.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn contains(&self, object: &str) -> bool {
|
||||
self.inner.objects.lock().await.contains_key(object)
|
||||
}
|
||||
@@ -540,13 +509,11 @@ impl MockWarmBackend {
|
||||
}
|
||||
|
||||
/// A clone of the stored object, if present.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn stored(&self, object: &str) -> Option<MockStoredObject> {
|
||||
self.inner.objects.lock().await.get(object).cloned()
|
||||
}
|
||||
|
||||
/// A clone of the raw bytes stored for `object`, if present.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn bytes(&self, object: &str) -> Option<Vec<u8>> {
|
||||
self.inner.objects.lock().await.get(object).map(|o| o.bytes.clone())
|
||||
}
|
||||
@@ -571,7 +538,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Poll until `object` is absent from the backend, or `timeout` elapses.
|
||||
/// Returns `true` if the object disappeared within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_remote_absence(&self, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -587,7 +553,6 @@ impl MockWarmBackend {
|
||||
|
||||
/// Poll until the backend holds exactly `expected` objects, or `timeout`
|
||||
/// elapses. Returns `true` if the count was reached within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_object_count(&self, expected: usize, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -882,7 +847,6 @@ pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tie
|
||||
/// The transition-state tuple read from an on-disk `xl.meta`, plus the object's
|
||||
/// free-version count.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
#[cfg(feature = "test-util")]
|
||||
pub struct TransitionMeta {
|
||||
/// `transition_status` (e.g. `"complete"`), empty when not transitioned.
|
||||
pub status: String,
|
||||
@@ -896,7 +860,6 @@ pub struct TransitionMeta {
|
||||
pub free_version_count: usize,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
|
||||
// `LocalDisk::new` rejects an endpoint whose (set_idx, disk_idx) disagrees
|
||||
// with the position recorded in the disk's own format.json, so derive the
|
||||
@@ -927,7 +890,6 @@ async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
|
||||
/// The free-version metadata removal lands asynchronously after the remote
|
||||
/// object disappears, so callers typically poll via
|
||||
/// [`wait_for_free_version_absence`] instead of asserting a single read.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) -> usize {
|
||||
let Some(disk) = open_disk(disk_path).await else {
|
||||
return 0;
|
||||
@@ -952,7 +914,6 @@ pub async fn free_version_count(disk_path: &Path, bucket: &str, object: &str) ->
|
||||
/// fields are taken from the newest version that carries a transition record;
|
||||
/// if no version is transitioned, they are taken from the current version (and
|
||||
/// will be empty).
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str) -> Option<TransitionMeta> {
|
||||
let disk = open_disk(disk_path).await?;
|
||||
let data = disk
|
||||
@@ -986,7 +947,6 @@ pub async fn read_transition_meta(disk_path: &Path, bucket: &str, object: &str)
|
||||
/// disk is missing the object or disagrees — this is the shard-consistency
|
||||
/// check required by ilm-6 (the `(status, tier, remote key, remote version id)`
|
||||
/// four-tuple plus free-version count must match across all erasure shards).
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P], bucket: &str, object: &str) -> TransitionMeta {
|
||||
assert!(!disk_paths.is_empty(), "assert_transition_meta_consistent needs at least one disk");
|
||||
|
||||
@@ -1012,7 +972,6 @@ pub async fn assert_transition_meta_consistent<P: AsRef<Path>>(disk_paths: &[P],
|
||||
|
||||
/// Poll until `object` retains no free versions on `disk_path`, or `timeout`
|
||||
/// elapses. Returns `true` if the free versions drained within the budget.
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn wait_for_free_version_absence(disk_path: &Path, bucket: &str, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
@@ -1081,44 +1040,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_metadata_survives_put_and_external_delete_is_distinct() {
|
||||
let backend = MockWarmBackend::new();
|
||||
let metadata = HashMap::from([
|
||||
("content-type".to_string(), "text/plain".to_string()),
|
||||
("project".to_string(), "archive".to_string()),
|
||||
]);
|
||||
let version = backend
|
||||
.put_with_meta("object", ReaderImpl::Body(Bytes::from_static(b"body")), 4, metadata.clone())
|
||||
.await
|
||||
.expect("mock PUT should preserve remote metadata");
|
||||
assert_eq!(backend.metadata("object").await, Some(metadata));
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("object")
|
||||
.await
|
||||
.expect("probe stored object"),
|
||||
TransitionCandidateProbe::VersionedPresent(version)
|
||||
);
|
||||
|
||||
backend.external_remove("object").await;
|
||||
assert_eq!(backend.metadata("object").await, None);
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_transition_candidate_state("object")
|
||||
.await
|
||||
.expect("probe removed object"),
|
||||
TransitionCandidateProbe::Missing
|
||||
);
|
||||
let operations = backend.op_log().await;
|
||||
assert!(
|
||||
operations
|
||||
.iter()
|
||||
.any(|op| matches!(op, MockWarmOp::ExternalRemove { object } if object == "object"))
|
||||
);
|
||||
assert!(!operations.iter().any(|op| matches!(op, MockWarmOp::Remove { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mock_probe_preserves_fault_fail_closed_behavior() {
|
||||
let backend = MockWarmBackend::new();
|
||||
|
||||
@@ -143,12 +143,11 @@ struct TierDriverBuildBarrier {
|
||||
static TIER_DRIVER_BUILD_BARRIER: LazyLock<Mutex<Option<Arc<TierDriverBuildBarrier>>>> = LazyLock::new(|| Mutex::new(None));
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) type TierDriverTestFactory =
|
||||
Arc<dyn Fn(&TierConfig) -> std::result::Result<WarmBackendImpl, AdminError> + Send + Sync + 'static>;
|
||||
type TierDriverTestFactory = Arc<dyn Fn(&TierConfig) -> std::result::Result<WarmBackendImpl, AdminError> + Send + Sync + 'static>;
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
pub(crate) static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory;
|
||||
static TIER_DRIVER_TEST_FACTORY: TierDriverTestFactory;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1378,42 +1377,27 @@ async fn ensure_no_authoritative_persisted_references<S>(
|
||||
where
|
||||
S: TierReferenceProofStore,
|
||||
{
|
||||
ensure_no_authoritative_persisted_references_with(
|
||||
api.clone(),
|
||||
TIER_DELETE_JOURNAL_PREFIX,
|
||||
"tier-delete journal",
|
||||
|_object, data| {
|
||||
let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?;
|
||||
Ok((
|
||||
journal.tier_name.clone(),
|
||||
tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets),
|
||||
))
|
||||
},
|
||||
)
|
||||
ensure_no_authoritative_persisted_references_with(api.clone(), TIER_DELETE_JOURNAL_PREFIX, |_object, data| {
|
||||
let journal = decode_tier_delete_journal_entry(data).map_err(io::Error::other)?;
|
||||
Ok((
|
||||
journal.tier_name.clone(),
|
||||
tier_persisted_reference_blocks_any_target(&journal.tier_name, journal.backend_identity, targets),
|
||||
))
|
||||
})
|
||||
.await?;
|
||||
ensure_no_authoritative_persisted_references_with(
|
||||
api,
|
||||
TRANSITION_TRANSACTION_RECORD_PREFIX,
|
||||
"transition transaction",
|
||||
|object, data| {
|
||||
let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?;
|
||||
Ok((
|
||||
transaction.tier_name.clone(),
|
||||
tier_persisted_reference_blocks_any_target(
|
||||
&transaction.tier_name,
|
||||
Some(transaction.backend_fingerprint),
|
||||
targets,
|
||||
),
|
||||
))
|
||||
},
|
||||
)
|
||||
ensure_no_authoritative_persisted_references_with(api, TRANSITION_TRANSACTION_RECORD_PREFIX, |object, data| {
|
||||
let transaction = decode_transition_transaction_record(object, data).map_err(io::Error::other)?;
|
||||
Ok((
|
||||
transaction.tier_name.clone(),
|
||||
tier_persisted_reference_blocks_any_target(&transaction.tier_name, Some(transaction.backend_fingerprint), targets),
|
||||
))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn ensure_no_authoritative_persisted_references_with<S, F>(
|
||||
api: Arc<S>,
|
||||
prefix: &str,
|
||||
reference_kind: &str,
|
||||
blocks_target: F,
|
||||
) -> std::result::Result<(), AdminError>
|
||||
where
|
||||
@@ -1442,7 +1426,7 @@ where
|
||||
.map_err(tier_reference_proof_admin_error)?;
|
||||
let (tier_name, blocks) = blocks_target(&object.name, &data).map_err(tier_reference_proof_admin_error)?;
|
||||
if blocks {
|
||||
return Err(tier_reference_proof_persisted_in_use_error(&tier_name, reference_kind, &object.name));
|
||||
return Err(tier_reference_proof_persisted_in_use_error(&tier_name, &object.name));
|
||||
}
|
||||
}
|
||||
if !page.is_truncated {
|
||||
@@ -1504,21 +1488,16 @@ fn tier_persisted_reference_blocks_target(
|
||||
|
||||
fn tier_reference_proof_in_use_error(tier_name: &str, object: &ObjectInfo) -> AdminError {
|
||||
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
|
||||
let reference_kind = if object.transitioned_object.free_version {
|
||||
"free-version ownership"
|
||||
} else {
|
||||
"transitioned-object"
|
||||
};
|
||||
err.message = format!(
|
||||
"Remote tier {tier_name} still has a {reference_kind} reference, for example {}/{}",
|
||||
"Remote tier {tier_name} still has object references, for example {}/{}",
|
||||
object.bucket, object.name
|
||||
);
|
||||
err
|
||||
}
|
||||
|
||||
fn tier_reference_proof_persisted_in_use_error(tier_name: &str, reference_kind: &str, object: &str) -> AdminError {
|
||||
fn tier_reference_proof_persisted_in_use_error(tier_name: &str, object: &str) -> AdminError {
|
||||
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
|
||||
err.message = format!("Remote tier {tier_name} still has a {reference_kind} reference, for example {object}");
|
||||
err.message = format!("Remote tier {tier_name} still has a persisted reference, for example {object}");
|
||||
err
|
||||
}
|
||||
|
||||
@@ -3562,7 +3541,7 @@ impl TierConfigMgr {
|
||||
// Get tier configuration and create new driver
|
||||
let tier_config = self.tiers.get(tier_name).ok_or_else(|| ERR_TIER_NOT_FOUND.clone())?;
|
||||
|
||||
let driver = construct_warm_backend(tier_config).await?;
|
||||
let driver = new_warm_backend(tier_config, false).await?;
|
||||
|
||||
self.replace_driver(tier_name, driver)?;
|
||||
Ok(self
|
||||
@@ -3706,17 +3685,13 @@ impl TierConfigMgr {
|
||||
let manager = handle.read().await;
|
||||
let runtime = tier_driver_runtime(handle, &manager);
|
||||
let runtime = lock_unpoisoned(&runtime);
|
||||
let prepared = runtime
|
||||
.prepared_mutation_blocks
|
||||
.values()
|
||||
.any(|blocked_mutation_id| *blocked_mutation_id == mutation_id);
|
||||
let committed = runtime
|
||||
if !runtime
|
||||
.committed_mutation_blocks
|
||||
.values()
|
||||
.any(|mutation_ids| mutation_ids.contains(&mutation_id));
|
||||
if !prepared && !committed {
|
||||
.any(|mutation_ids| mutation_ids.contains(&mutation_id))
|
||||
{
|
||||
let mut err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
err.message = "Remote tier mutation fence was not installed".to_string();
|
||||
err.message = "Remote tier committed mutation fence was not installed".to_string();
|
||||
return Err(err);
|
||||
}
|
||||
Ok(MutationBlockAllowance {
|
||||
@@ -3923,6 +3898,14 @@ impl TierConfigMgr {
|
||||
Self::begin_tier_transition_with_destinations(handle, manager, changed, replaced_destinations, mutation_block_allowance)
|
||||
}
|
||||
|
||||
fn begin_tier_transition(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
manager: &mut Self,
|
||||
changed: HashSet<String>,
|
||||
) -> std::result::Result<TierPublishTransition, AdminError> {
|
||||
Self::begin_tier_transition_with_destinations(handle, manager, changed, HashMap::new(), None)
|
||||
}
|
||||
|
||||
fn begin_tier_transition_with_destinations(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
manager: &mut Self,
|
||||
@@ -4193,7 +4176,7 @@ impl TierConfigMgr {
|
||||
let mut config_lock = config_lock;
|
||||
let coordinated_config_update = config_lock.is_some();
|
||||
let mut update = Some(update);
|
||||
let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, target_tiers) =
|
||||
let (mutation_kind, explicit_tier_name, mutation_force, current_for_targets, driver_tier, mut transition) =
|
||||
match mutation {
|
||||
TierCandidateMutation::Prevalidated(prepared) => {
|
||||
if version != prepared.version {
|
||||
@@ -4209,8 +4192,9 @@ impl TierConfigMgr {
|
||||
)));
|
||||
}
|
||||
candidate = prepared.candidate;
|
||||
let target_tiers = {
|
||||
let manager = handle.read().await;
|
||||
let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
|
||||
let mut transition = {
|
||||
let mut manager = handle.write().await;
|
||||
let mut target_tiers = changed_tier_names(&manager, &candidate);
|
||||
if let Some(tier_name) = prepared.explicit_tier_name.as_ref()
|
||||
&& (manager.tiers.contains_key(tier_name)
|
||||
@@ -4219,7 +4203,8 @@ impl TierConfigMgr {
|
||||
{
|
||||
target_tiers.insert(tier_name.clone());
|
||||
}
|
||||
target_tiers
|
||||
Self::begin_tier_transition(&handle, &mut manager, target_tiers)
|
||||
.map_err(TierConfigUpdateError::Publish)?
|
||||
};
|
||||
(
|
||||
prepared.kind,
|
||||
@@ -4227,7 +4212,7 @@ impl TierConfigMgr {
|
||||
prepared.force,
|
||||
prepared.current,
|
||||
prepared.driver_tier,
|
||||
target_tiers,
|
||||
transition,
|
||||
)
|
||||
}
|
||||
mutation => {
|
||||
@@ -4252,9 +4237,11 @@ impl TierConfigMgr {
|
||||
last_refreshed_at: candidate.last_refreshed_at,
|
||||
};
|
||||
let validation_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
|
||||
let target_tiers = {
|
||||
let manager = handle.read().await;
|
||||
mutation.target_tiers(&manager, &candidate)
|
||||
let mut transition = {
|
||||
let mut manager = handle.write().await;
|
||||
let target_tiers = mutation.target_tiers(&manager, &candidate);
|
||||
Self::begin_tier_transition(&handle, &mut manager, target_tiers)
|
||||
.map_err(TierConfigUpdateError::Publish)?
|
||||
};
|
||||
let driver_tier = apply_tier_candidate_mutation(mutation, &mut candidate, validation_deadline)
|
||||
.await
|
||||
@@ -4265,7 +4252,7 @@ impl TierConfigMgr {
|
||||
mutation_force,
|
||||
current_for_targets,
|
||||
driver_tier,
|
||||
target_tiers,
|
||||
transition,
|
||||
)
|
||||
}
|
||||
};
|
||||
@@ -4288,83 +4275,28 @@ impl TierConfigMgr {
|
||||
save_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref())
|
||||
.await
|
||||
.map_err(TierConfigUpdateError::Save)?;
|
||||
let mut blocked_target_tiers = target_tiers.clone();
|
||||
if let Some(intent) = coordinator_intent.as_ref() {
|
||||
blocked_target_tiers.extend(intent.affected_targets.iter().map(|target| target.tier_name.clone()));
|
||||
}
|
||||
if let Some(intent) = coordinator_intent.as_ref() {
|
||||
// `target_tiers` may include a stale local-only manager
|
||||
// entry that is absent from the persisted proof
|
||||
// snapshot. Fence that local transition under the same
|
||||
// mutation ID as well; recovery may discard this
|
||||
// process-local superset, which advances the revision
|
||||
// and makes the deferred transition fail closed.
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block_for_tiers(&handle, intent, &blocked_target_tiers)
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&handle, intent)
|
||||
.await
|
||||
.map_err(TierConfigUpdateError::Publish)?;
|
||||
}
|
||||
let prepared_mutation_block_allowance = match coordinator_intent.as_ref() {
|
||||
Some(intent) => Some(
|
||||
TierConfigMgr::mutation_block_allowance_for(&handle, intent.mutation_id)
|
||||
.await
|
||||
.map_err(TierConfigUpdateError::Publish)?,
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
// A durable coordinator intent supplies the admission
|
||||
// fence that lets us defer generation revocation. Keep the
|
||||
// original early transition for no-intent paths (for
|
||||
// example, reconciling a stale local manager to an
|
||||
// idempotently removed persisted tier), where there is no
|
||||
// Prepared record capable of blocking a new lease.
|
||||
let (mut transition, deferred_target_tiers) = if coordinator_intent.is_some() {
|
||||
(None, Some(target_tiers))
|
||||
} else {
|
||||
let transition = {
|
||||
let mut manager = handle.write().await;
|
||||
Self::begin_tier_transition_with_destinations(
|
||||
&handle,
|
||||
&mut manager,
|
||||
target_tiers,
|
||||
HashMap::new(),
|
||||
None,
|
||||
)
|
||||
.map_err(TierConfigUpdateError::Publish)?
|
||||
};
|
||||
(Some(transition), None)
|
||||
};
|
||||
if coordinated_config_update {
|
||||
drop(update.take());
|
||||
drop(config_lock.take());
|
||||
}
|
||||
// The durable Prepared block closes admission before the
|
||||
// zero-reference proof, but deliberately leaves already
|
||||
// issued generations current. In particular, an exact
|
||||
// free-version cleanup that has completed remote DELETE
|
||||
// must still be able to remove its local ownership marker;
|
||||
// revoking its generation here would strand that marker
|
||||
// and make this mutation reject its own interrupted work.
|
||||
if let Some(intent) = coordinator_intent.as_ref()
|
||||
&& let Err(drain_error) =
|
||||
TierConfigMgr::wait_for_blocked_tier_operation_leases_for_tiers(&handle, &blocked_target_tiers).await
|
||||
{
|
||||
if !abort_prepared_tier_mutation(&handle, api.clone(), Some(intent), Vec::new()).await {
|
||||
let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
|
||||
if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await {
|
||||
if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), Vec::new()).await {
|
||||
warn!(
|
||||
event = "tier_mutation_abort",
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_TIER,
|
||||
result = "prepared_intent_retained",
|
||||
mutation_id = %intent.mutation_id,
|
||||
coordinator_intent = coordinator_intent.is_some(),
|
||||
"tier mutation lease drain failed and abort was incomplete"
|
||||
);
|
||||
}
|
||||
return Err(TierConfigUpdateError::Publish(drain_error));
|
||||
} else if let Some(transition) = transition.as_ref() {
|
||||
let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
|
||||
transition
|
||||
.wait_for_active_leases_until(drain_deadline)
|
||||
.await
|
||||
.map_err(TierConfigUpdateError::Publish)?;
|
||||
}
|
||||
let prepared_peers = if let Some(intent) = coordinator_intent.as_ref() {
|
||||
let peers = match remote_tier_mutation_peers().await {
|
||||
@@ -4431,71 +4363,6 @@ impl TierConfigMgr {
|
||||
}
|
||||
return Err(TierConfigUpdateError::Publish(proof_error));
|
||||
}
|
||||
// No affected-tier lease can start after Prepared, and the
|
||||
// existing set was drained above. It is now safe to revoke
|
||||
// the generation for publication without invalidating a
|
||||
// cleanup between its remote and local commit boundaries.
|
||||
if transition.is_none() {
|
||||
let target_tiers = deferred_target_tiers.ok_or_else(|| {
|
||||
let mut err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
err.message = "Remote tier mutation lost its deferred transition targets".to_string();
|
||||
TierConfigUpdateError::Publish(err)
|
||||
})?;
|
||||
let mut manager = handle.write().await;
|
||||
transition = Some(
|
||||
match Self::begin_tier_transition_with_destinations(
|
||||
&handle,
|
||||
&mut manager,
|
||||
target_tiers,
|
||||
HashMap::new(),
|
||||
prepared_mutation_block_allowance.as_ref(),
|
||||
) {
|
||||
Ok(transition) => transition,
|
||||
Err(transition_error) => {
|
||||
drop(manager);
|
||||
if !abort_prepared_tier_mutation(
|
||||
&handle,
|
||||
api.clone(),
|
||||
coordinator_intent.as_ref(),
|
||||
prepared_peers,
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
event = "tier_mutation_abort",
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_TIER,
|
||||
result = "prepared_intent_retained",
|
||||
coordinator_intent = coordinator_intent.is_some(),
|
||||
"tier mutation publish transition failed and abort was incomplete"
|
||||
);
|
||||
}
|
||||
return Err(TierConfigUpdateError::Publish(transition_error));
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
let mut transition = transition.ok_or_else(|| {
|
||||
let mut err = ERR_TIER_INVALID_CONFIG.clone();
|
||||
err.message = "Remote tier mutation lost its publish transition".to_string();
|
||||
TierConfigUpdateError::Publish(err)
|
||||
})?;
|
||||
let drain_deadline = Instant::now() + TIER_REMOTE_VALIDATION_TIMEOUT;
|
||||
if let Err(drain_error) = transition.wait_for_active_leases_until(drain_deadline).await {
|
||||
drop(transition);
|
||||
if !abort_prepared_tier_mutation(&handle, api.clone(), coordinator_intent.as_ref(), prepared_peers).await
|
||||
{
|
||||
warn!(
|
||||
event = "tier_mutation_abort",
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_TIER,
|
||||
result = "prepared_intent_retained",
|
||||
coordinator_intent = coordinator_intent.is_some(),
|
||||
"tier mutation publish drain failed and abort was incomplete"
|
||||
);
|
||||
}
|
||||
return Err(TierConfigUpdateError::Publish(drain_error));
|
||||
}
|
||||
let candidate_digest = tier_config_candidate_digest(&candidate).map_err(TierConfigUpdateError::Save)?;
|
||||
if coordinated_config_update {
|
||||
config_lock = match Self::acquire_tier_config_write_lock(api.clone()).await {
|
||||
@@ -4619,11 +4486,6 @@ impl TierConfigMgr {
|
||||
let committed_coordinator_intent =
|
||||
committed_tier_mutation_intent(coordinator_intent.as_ref(), &committed_config_etag)
|
||||
.map_err(TierConfigUpdateError::Save)?;
|
||||
// Persist Committed before notifying refresh; a Prepared disk record
|
||||
// would restore the prepared block and invalidate our publish allowance.
|
||||
let coordinator_commit =
|
||||
commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag)
|
||||
.await;
|
||||
if let Some(intent) = committed_coordinator_intent.as_ref() {
|
||||
TierConfigMgr::apply_committed_mutation_intent_block(&handle, intent)
|
||||
.await
|
||||
@@ -4634,9 +4496,9 @@ impl TierConfigMgr {
|
||||
.map_err(TierConfigUpdateError::Publish)?,
|
||||
);
|
||||
}
|
||||
// Config is already saved: retain the committed fence and wake recovery
|
||||
// even when the coordinator commit failed or its outcome is unknown.
|
||||
coordinator_commit.map_err(TierConfigUpdateError::Save)?;
|
||||
commit_coordinator_tier_mutation_intent(api.clone(), coordinator_intent.as_ref(), &committed_config_etag)
|
||||
.await
|
||||
.map_err(TierConfigUpdateError::Save)?;
|
||||
if coordinated_config_update {
|
||||
drop(update.take());
|
||||
drop(config_lock.take());
|
||||
@@ -5547,40 +5409,11 @@ impl TierConfigMgr {
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
intent: &TierMutationIntent,
|
||||
) -> std::result::Result<(), AdminError> {
|
||||
let target_tiers = intent
|
||||
.affected_targets
|
||||
.iter()
|
||||
.map(|target| target.tier_name.clone())
|
||||
.collect();
|
||||
Self::apply_prepared_mutation_intent_block_for_tiers(handle, intent, &target_tiers).await
|
||||
}
|
||||
|
||||
async fn apply_prepared_mutation_intent_block_for_tiers(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
intent: &TierMutationIntent,
|
||||
target_tiers: &HashSet<String>,
|
||||
) -> std::result::Result<(), AdminError> {
|
||||
if intent.state != TierMutationIntentState::Prepared {
|
||||
return Ok(());
|
||||
}
|
||||
let manager = handle.read().await;
|
||||
let runtime = tier_driver_runtime(handle, &manager);
|
||||
let mut runtime = lock_unpoisoned(&runtime);
|
||||
let mut prepared_mutation_blocks = runtime.prepared_mutation_blocks.clone();
|
||||
Self::collect_prepared_mutation_intent_block(&mut prepared_mutation_blocks, intent)?;
|
||||
for tier_name in target_tiers {
|
||||
match prepared_mutation_blocks.entry(tier_name.clone()) {
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(intent.mutation_id);
|
||||
}
|
||||
Entry::Occupied(entry) if *entry.get() == intent.mutation_id => {}
|
||||
Entry::Occupied(_) => {
|
||||
let mut err = ERR_TIER_BACKEND_IN_USE.clone();
|
||||
err.message = format!("Remote tier {tier_name} already has another prepared mutation");
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
if prepared_mutation_blocks == runtime.prepared_mutation_blocks {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -5596,18 +5429,6 @@ impl TierConfigMgr {
|
||||
pub(crate) async fn wait_for_blocked_tier_operation_leases(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
intent: &TierMutationIntent,
|
||||
) -> std::result::Result<(), AdminError> {
|
||||
let target_tiers = intent
|
||||
.affected_targets
|
||||
.iter()
|
||||
.map(|target| target.tier_name.clone())
|
||||
.collect();
|
||||
Self::wait_for_blocked_tier_operation_leases_for_tiers(handle, &target_tiers).await
|
||||
}
|
||||
|
||||
async fn wait_for_blocked_tier_operation_leases_for_tiers(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
target_tiers: &HashSet<String>,
|
||||
) -> std::result::Result<(), AdminError> {
|
||||
let generations = {
|
||||
let manager = handle.read().await;
|
||||
@@ -5615,9 +5436,10 @@ impl TierConfigMgr {
|
||||
return Ok(());
|
||||
};
|
||||
let runtime = lock_unpoisoned(&runtime);
|
||||
target_tiers
|
||||
intent
|
||||
.affected_targets
|
||||
.iter()
|
||||
.filter_map(|tier_name| runtime.generations.get(tier_name).cloned())
|
||||
.filter_map(|target| runtime.generations.get(&target.tier_name).cloned())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let drain = async {
|
||||
@@ -10781,11 +10603,6 @@ mod tests {
|
||||
.expect_err("coordinator committed-state CAS failure must be observable");
|
||||
assert!(matches!(err, TierConfigUpdateError::Save(_)));
|
||||
assert!(manager.read().await.tiers.contains_key("COLD-A"));
|
||||
assert!(TierConfigMgr::has_committed_mutation_block(&manager).await);
|
||||
let refresh = TierConfigMgr::mutation_refresh_notifier(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(1), refresh.notified())
|
||||
.await
|
||||
.expect("failed coordinator commit must notify recovery after saving config");
|
||||
let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
|
||||
Ok(_) => panic!("failed coordinator commit CAS must retain the local committed fence"),
|
||||
Err(err) => err,
|
||||
@@ -14512,12 +14329,6 @@ mod tests {
|
||||
after_commit: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct CasCoordinatorCommitBarrier {
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CasConfigStore {
|
||||
objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>,
|
||||
@@ -14530,7 +14341,6 @@ mod tests {
|
||||
fail_delete_prefix: tokio::sync::Mutex<Option<(String, usize)>>,
|
||||
delete_log: tokio::sync::Mutex<Vec<String>>,
|
||||
list_barrier: tokio::sync::Mutex<Option<Arc<CasListBarrier>>>,
|
||||
coordinator_commit_barrier: tokio::sync::Mutex<Option<Arc<CasCoordinatorCommitBarrier>>>,
|
||||
intent_list_calls: AtomicUsize,
|
||||
fail_reference_walk: AtomicBool,
|
||||
reference_walk_send_count: AtomicUsize,
|
||||
@@ -14553,7 +14363,6 @@ mod tests {
|
||||
fail_delete_prefix: tokio::sync::Mutex::new(None),
|
||||
delete_log: tokio::sync::Mutex::new(Vec::new()),
|
||||
list_barrier: tokio::sync::Mutex::new(None),
|
||||
coordinator_commit_barrier: tokio::sync::Mutex::new(None),
|
||||
intent_list_calls: AtomicUsize::new(0),
|
||||
fail_reference_walk: AtomicBool::new(false),
|
||||
reference_walk_send_count: AtomicUsize::new(0),
|
||||
@@ -14587,13 +14396,6 @@ mod tests {
|
||||
.push(object);
|
||||
}
|
||||
|
||||
fn remove_listed_version(&self, bucket: &str, object: &str) {
|
||||
self.listed_versions
|
||||
.lock()
|
||||
.expect("tier reference fixture should not poison")
|
||||
.retain(|version| version.bucket != bucket || version.name != object);
|
||||
}
|
||||
|
||||
fn add_lifecycle_config(&self, bucket: &str, config: BucketLifecycleConfiguration) {
|
||||
self.lifecycle_configs
|
||||
.lock()
|
||||
@@ -14752,19 +14554,6 @@ mod tests {
|
||||
}
|
||||
let mut payload = Vec::new();
|
||||
tokio::io::AsyncReadExt::read_to_end(&mut data.stream, &mut payload).await?;
|
||||
if object.starts_with(crate::services::tier::tier_mutation_intent::TIER_COORDINATOR_MUTATION_INTENT_RECORD_PREFIX)
|
||||
&& opts
|
||||
.http_preconditions
|
||||
.as_ref()
|
||||
.and_then(HTTPPreconditions::if_match_value)
|
||||
.is_some()
|
||||
{
|
||||
let barrier = self.coordinator_commit_barrier.lock().await.take();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
let race_rewrite = if opts
|
||||
.http_preconditions
|
||||
.as_ref()
|
||||
@@ -15863,152 +15652,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_remove_prepared_fence_allows_inflight_free_version_cleanup_to_finish() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let tier = build_rustfs_tier("COLD-A");
|
||||
let identity = tier_backend_identity(&tier).expect("test tier identity should encode");
|
||||
let mut persisted = empty_mgr();
|
||||
persisted.tiers.insert("COLD-A".to_string(), tier.clone_with_credentials());
|
||||
persisted
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("free-version drain fixture should persist");
|
||||
|
||||
let manager = TierConfigMgr::new();
|
||||
{
|
||||
let mut guard = manager.write().await;
|
||||
guard.tiers.insert("COLD-A".to_string(), tier);
|
||||
guard.tiers.insert("COLD-B".to_string(), build_rustfs_tier("COLD-B"));
|
||||
guard
|
||||
.replace_driver("COLD-A", Box::new(LeaseTestBackend::ready("cleanup")))
|
||||
.expect("cleanup driver generation should install");
|
||||
guard
|
||||
.replace_driver("COLD-B", Box::new(LeaseTestBackend::ready("stale-local")))
|
||||
.expect("stale local driver generation should install");
|
||||
}
|
||||
let cleanup_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
|
||||
.await
|
||||
.expect("in-flight cleanup lease should be available");
|
||||
let stale_local_lease = TierConfigMgr::acquire_operation_lease(&manager, "COLD-B")
|
||||
.await
|
||||
.expect("stale local tier lease should be available");
|
||||
let mut free_version = transitioned_tier_object("photos", "2026/free-version.jpg", "COLD-A", Some(identity));
|
||||
free_version.transitioned_object.status = "pending".to_string();
|
||||
free_version.transitioned_object.free_version = true;
|
||||
store.add_listed_version(free_version);
|
||||
|
||||
let remove_manager = manager.clone();
|
||||
let remove_store = store.clone();
|
||||
let remove = tokio::spawn(async move {
|
||||
TIER_MUTATION_TEST_PEERS
|
||||
.scope(
|
||||
Vec::new(),
|
||||
TierConfigMgr::remove_and_save_with(&remove_manager, remove_store, "COLD-A", true),
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let guard = manager.read().await;
|
||||
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
|
||||
let prepared = {
|
||||
let runtime = lock_unpoisoned(&runtime);
|
||||
runtime.prepared_mutation_blocks.contains_key("COLD-A")
|
||||
&& runtime.prepared_mutation_blocks.contains_key("COLD-B")
|
||||
};
|
||||
if prepared {
|
||||
break;
|
||||
}
|
||||
drop(guard);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("tier remove should install its durable prepared fence");
|
||||
|
||||
assert!(
|
||||
cleanup_lease.is_current(&manager).await,
|
||||
"the prepared fence must let the already leased cleanup finish its exact local marker deletion"
|
||||
);
|
||||
assert!(
|
||||
stale_local_lease.is_current(&manager).await,
|
||||
"the local superset fence must also let an already leased stale-manager operation finish"
|
||||
);
|
||||
let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
|
||||
Ok(_) => panic!("the prepared fence must reject new tier operations"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked));
|
||||
let stale_blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-B").await {
|
||||
Ok(_) => panic!("the local superset fence must reject new stale-manager operations"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&stale_blocked));
|
||||
|
||||
store.remove_listed_version("photos", "2026/free-version.jpg");
|
||||
drop(cleanup_lease);
|
||||
drop(stale_local_lease);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(5), remove)
|
||||
.await
|
||||
.expect("tier remove should finish after both in-flight operations release their leases")
|
||||
.expect("tier remove task should join")
|
||||
.expect("tier remove should pass once the in-flight cleanup removes its marker");
|
||||
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
|
||||
assert!(!manager.read().await.tiers.contains_key("COLD-B"));
|
||||
assert!(
|
||||
!load_tier_config_for_update(store)
|
||||
.await
|
||||
.expect("removed tier config should reload")
|
||||
.0
|
||||
.tiers
|
||||
.contains_key("COLD-A")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_intent_stale_manager_removal_keeps_early_generation_drain() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
empty_mgr()
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("empty persisted tier config should exist");
|
||||
let manager = TierConfigMgr::new();
|
||||
{
|
||||
let mut guard = manager.write().await;
|
||||
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("stale"));
|
||||
}
|
||||
let old = TierConfigMgr::acquire_operation_lease(&manager, "COLD-A")
|
||||
.await
|
||||
.expect("stale manager lease should be available");
|
||||
|
||||
let remove_manager = manager.clone();
|
||||
let remove_store = store.clone();
|
||||
let remove =
|
||||
tokio::spawn(async move { TierConfigMgr::remove_and_save_with(&remove_manager, remove_store, "COLD-A", true).await });
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while old.is_current(&manager).await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("no-intent stale-manager reconciliation should revoke before its proof");
|
||||
let blocked = match TierConfigMgr::acquire_operation_lease(&manager, "COLD-A").await {
|
||||
Ok(_) => panic!("stale-manager reconciliation must not admit a new operation"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(TierConfigMgr::operation_lease_blocked_by_mutation(&blocked));
|
||||
|
||||
drop(old);
|
||||
remove
|
||||
.await
|
||||
.expect("stale-manager removal task should join")
|
||||
.expect("stale-manager removal should converge to the persisted empty config");
|
||||
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
|
||||
}
|
||||
|
||||
async fn assert_lifecycle_only_reference_obeys_force(clear: bool, force: bool) {
|
||||
async fn force_remove_and_save_bypasses_lifecycle_only_reference() {
|
||||
// rustfs/rustfs#6832: reproduces the admin RemoveTier path (not just the lower-level
|
||||
// reference-proof function) for a tier with zero transitioned objects but a lifecycle
|
||||
// rule still pointing at it — the exact shape of
|
||||
// `test_manual_transition_async_tier_failure_reports_terminal_partial` in e2e_test,
|
||||
// which force-removes a tier a lifecycle rule still references to simulate a
|
||||
// decommissioned backend.
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let tier = build_rustfs_tier("COLD-A");
|
||||
let mut persisted = empty_mgr();
|
||||
@@ -16049,55 +15699,22 @@ mod tests {
|
||||
|
||||
let manager = TierConfigMgr::new();
|
||||
manager.write().await.tiers.insert("COLD-A".to_string(), tier);
|
||||
let mutation = if clear {
|
||||
TierCandidateMutation::Clear(force)
|
||||
} else {
|
||||
TierCandidateMutation::Remove("COLD-A".to_string(), force)
|
||||
};
|
||||
let result = TIER_DRIVER_TEST_FACTORY
|
||||
.scope(
|
||||
healthy_driver_factory(),
|
||||
TierConfigMgr::update_candidate_with_config_lock(&manager, store.clone(), mutation),
|
||||
)
|
||||
.await;
|
||||
if force {
|
||||
result.expect("force mutation must bypass a lifecycle-config-only reference");
|
||||
} else {
|
||||
let err = result.expect_err("non-force mutation must reject a lifecycle-only reference");
|
||||
let TierConfigUpdateError::Publish(err) = err else {
|
||||
panic!("non-force mutation must fail during reference proof: {err:?}");
|
||||
};
|
||||
assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code);
|
||||
assert!(err.message.contains("move-current"), "{err}");
|
||||
}
|
||||
TierConfigMgr::remove_and_save_with(&manager, store.clone(), "COLD-A", true)
|
||||
.await
|
||||
.expect("force remove must bypass a lifecycle-config-only reference");
|
||||
|
||||
assert_eq!(manager.read().await.tiers.contains_key("COLD-A"), !force);
|
||||
assert_eq!(
|
||||
load_tier_config_for_update(store)
|
||||
assert!(!manager.read().await.tiers.contains_key("COLD-A"));
|
||||
assert!(
|
||||
!load_tier_config_for_update(store)
|
||||
.await
|
||||
.expect("config should still reload")
|
||||
.0
|
||||
.tiers
|
||||
.contains_key("COLD-A"),
|
||||
!force,
|
||||
"persisted state must match the force mutation result"
|
||||
"force removal must persist the empty candidate"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remove_with_config_lock_obeys_force_for_lifecycle_only_reference() {
|
||||
for force in [false, true] {
|
||||
assert_lifecycle_only_reference_obeys_force(false, force).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clear_with_config_lock_obeys_force_for_lifecycle_only_reference() {
|
||||
for force in [false, true] {
|
||||
assert_lifecycle_only_reference_obeys_force(true, force).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn zero_reference_proof_blocks_clear_before_config_save() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
@@ -16911,23 +16528,12 @@ mod tests {
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let guard = manager.read().await;
|
||||
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
|
||||
let prepared = lock_unpoisoned(&runtime).prepared_mutation_blocks.contains_key("COLD-A");
|
||||
if prepared {
|
||||
break;
|
||||
}
|
||||
drop(guard);
|
||||
while old.is_current(&manager).await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned update should install its prepared fence before caller cancellation");
|
||||
assert!(
|
||||
old.is_current(&manager).await,
|
||||
"an already leased operation must remain current until it can finish"
|
||||
);
|
||||
.expect("owned update should revoke before caller cancellation");
|
||||
caller.abort();
|
||||
drop(old);
|
||||
|
||||
@@ -16972,23 +16578,12 @@ mod tests {
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let guard = manager.read().await;
|
||||
let runtime = registered_tier_driver_runtime(&guard).expect("runtime should remain registered");
|
||||
let prepared = lock_unpoisoned(&runtime).prepared_mutation_blocks.contains_key("COLD-A");
|
||||
if prepared {
|
||||
break;
|
||||
}
|
||||
drop(guard);
|
||||
while old.is_current(&manager).await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("owned update should install its prepared fence before caller cancellation");
|
||||
assert!(
|
||||
old.is_current(&manager).await,
|
||||
"the prepared fence must not invalidate an already leased operation"
|
||||
);
|
||||
.expect("owned update should revoke before caller cancellation");
|
||||
caller.abort();
|
||||
|
||||
let config_file = tier_config_lock_path();
|
||||
@@ -17374,71 +16969,6 @@ mod tests {
|
||||
assert!(current.tiers.contains_key("COLD-B"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn reference_proof_rejects_a_changed_prepared_fence_revision_before_publish() {
|
||||
let manager = TierConfigMgr::new();
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
let mut persisted = empty_mgr();
|
||||
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
persisted
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("prepared-fence revision fixture should persist");
|
||||
{
|
||||
let mut guard = manager.write().await;
|
||||
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
|
||||
}
|
||||
|
||||
let barrier = tier_reference_proof_test_barrier();
|
||||
let scoped_barrier = barrier.clone();
|
||||
let update_manager = manager.clone();
|
||||
let update_store = store.clone();
|
||||
let update = tokio::spawn(async move {
|
||||
TIER_REFERENCE_PROOF_TEST_BARRIER
|
||||
.scope(
|
||||
scoped_barrier,
|
||||
TIER_MUTATION_TEST_PEERS.scope(
|
||||
Vec::new(),
|
||||
TierConfigMgr::update_candidate_with_config_lock(
|
||||
&update_manager,
|
||||
update_store,
|
||||
TierCandidateMutation::Remove("COLD-A".to_string(), true),
|
||||
),
|
||||
),
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.arrived.notified().await;
|
||||
|
||||
let unrelated = prepared_remove_intent("COLD-B", uuid::Uuid::from_u128(0x2237));
|
||||
TierConfigMgr::apply_prepared_mutation_intent_block(&manager, &unrelated)
|
||||
.await
|
||||
.expect("an unrelated prepared fence should advance the runtime revision");
|
||||
barrier.release.add_permits(1);
|
||||
|
||||
let err = update
|
||||
.await
|
||||
.expect("tier update task should join")
|
||||
.expect_err("a reference proof cannot authorize publication across a fence revision change");
|
||||
let TierConfigUpdateError::Publish(err) = err else {
|
||||
panic!("the stale prepared-fence allowance should fail publication: {err:?}");
|
||||
};
|
||||
assert!(err.message.contains("changed before replacement"), "{err}");
|
||||
assert!(manager.read().await.tiers.contains_key("COLD-A"));
|
||||
assert!(
|
||||
load_tier_config_for_update(store)
|
||||
.await
|
||||
.expect("rejected tier config should remain readable")
|
||||
.0
|
||||
.tiers
|
||||
.contains_key("COLD-A")
|
||||
);
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&manager, unrelated.mutation_id)
|
||||
.await
|
||||
.expect("unrelated test fence should clear");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn caller_cancellation_after_durable_prepare_does_not_hide_the_mutation() {
|
||||
@@ -17725,98 +17255,6 @@ mod tests {
|
||||
assert_ne!(manager_a.read().await.empty(), manager_b.read().await.empty());
|
||||
}
|
||||
|
||||
async fn assert_coordinator_commit_refresh_succeeds(mutation: TierCandidateMutation) {
|
||||
let adding = matches!(mutation, TierCandidateMutation::Add(..));
|
||||
let manager = TierConfigMgr::new();
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
if !adding {
|
||||
let mut persisted = empty_mgr();
|
||||
persisted.tiers.insert("COLD-A".to_string(), build_rustfs_tier("COLD-A"));
|
||||
persisted
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("existing tier fixture should persist");
|
||||
let mut guard = manager.write().await;
|
||||
install_lease_backend(&mut guard, "COLD-A", LeaseTestBackend::ready("old"));
|
||||
}
|
||||
let barrier = Arc::new(CasCoordinatorCommitBarrier::default());
|
||||
*store.coordinator_commit_barrier.lock().await = Some(barrier.clone());
|
||||
let update_manager = manager.clone();
|
||||
let update_store = store.clone();
|
||||
let update = tokio::spawn(async move {
|
||||
TIER_DRIVER_TEST_FACTORY
|
||||
.scope(
|
||||
healthy_driver_factory(),
|
||||
TIER_MUTATION_TEST_PEERS.scope(
|
||||
Vec::new(),
|
||||
TierConfigMgr::update_candidate_with_config_lock(&update_manager, update_store, mutation),
|
||||
),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), barrier.arrived.notified())
|
||||
.await
|
||||
.expect("mutation should reach coordinator commit after saving config");
|
||||
assert_eq!(
|
||||
load_tier_config_for_update(store.clone())
|
||||
.await
|
||||
.expect("saved config should be readable before coordinator commit")
|
||||
.0
|
||||
.tiers
|
||||
.contains_key("COLD-A"),
|
||||
adding
|
||||
);
|
||||
assert_eq!(
|
||||
TierConfigMgr::load_coordinator_mutation_intents(store.clone())
|
||||
.await
|
||||
.expect("coordinator intent should remain readable")[0]
|
||||
.state,
|
||||
TierMutationIntentState::Prepared
|
||||
);
|
||||
|
||||
let lock_requests = lock_unpoisoned(&store.lock_requests).len();
|
||||
// Also exercise an independently scheduled refresh while the durable
|
||||
// coordinator record is still Prepared, before its commit notification.
|
||||
TierConfigMgr::request_committed_mutation_refresh(&manager).await;
|
||||
TIER_MUTATION_TEST_PEERS
|
||||
.scope(Vec::new(), async {
|
||||
let worker = TierConfigMgr::refresh_tier_config_handle_with(manager.clone(), store.clone());
|
||||
tokio::pin!(worker);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while lock_unpoisoned(&store.lock_requests).len() == lock_requests {
|
||||
tokio::select! {
|
||||
_ = &mut worker => panic!("refresh worker must remain available"),
|
||||
_ = tokio::task::yield_now() => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("refresh should reconcile the Prepared record before waiting for the config lock");
|
||||
barrier.release.notify_one();
|
||||
let result = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
tokio::select! {
|
||||
_ = &mut worker => panic!("refresh worker must remain available"),
|
||||
result = update => result.expect("tier mutation task should join"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("tier mutation should finish with refresh running");
|
||||
result.expect("saved tier mutation must publish successfully on the first attempt");
|
||||
})
|
||||
.await;
|
||||
assert_eq!(manager.read().await.tiers.contains_key("COLD-A"), adding);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_add_succeeds_with_refresh_during_coordinator_commit() {
|
||||
assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Add(build_rustfs_tier("COLD-A"), true)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_remove_succeeds_with_refresh_during_coordinator_commit() {
|
||||
assert_coordinator_commit_refresh_succeeds(TierCandidateMutation::Remove("COLD-A".to_string(), true)).await;
|
||||
}
|
||||
|
||||
async fn committed_refresh_fixture(fail_cleanup: bool) -> (Arc<RwLock<TierConfigMgr>>, Arc<CasConfigStore>, uuid::Uuid) {
|
||||
let manager = TierConfigMgr::new();
|
||||
{
|
||||
|
||||
@@ -19,14 +19,13 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
#[cfg(feature = "gcs")]
|
||||
use crate::services::tier::warm_backend_gcs::WarmBackendGCS;
|
||||
use crate::services::tier::{
|
||||
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED},
|
||||
tier_config::{TierConfig, TierType},
|
||||
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR},
|
||||
warm_backend_aliyun::WarmBackendAliyun,
|
||||
warm_backend_azure::WarmBackendAzure,
|
||||
warm_backend_gcs::WarmBackendGCS,
|
||||
warm_backend_huaweicloud::WarmBackendHuaweicloud,
|
||||
warm_backend_minio::WarmBackendMinIO,
|
||||
warm_backend_r2::WarmBackendR2,
|
||||
@@ -38,7 +37,7 @@ use crate::services::tier::{
|
||||
use bytes::Bytes;
|
||||
use http::StatusCode;
|
||||
use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
|
||||
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
|
||||
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore};
|
||||
use rustfs_s3_client::{
|
||||
admin_handler_utils::AdminError,
|
||||
api_error_response::to_error_response,
|
||||
@@ -321,27 +320,6 @@ pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Erro
|
||||
}
|
||||
}
|
||||
|
||||
fn transition_timeout_from_env(env_key: &str, default_secs: u64) -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(env_key, default_secs))
|
||||
}
|
||||
|
||||
pub(crate) fn transition_client_timeouts_from_env() -> TransitionClientTimeouts {
|
||||
TransitionClientTimeouts::new(
|
||||
transition_timeout_from_env(
|
||||
rustfs_config::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
|
||||
),
|
||||
transition_timeout_from_env(
|
||||
rustfs_config::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
|
||||
),
|
||||
transition_timeout_from_env(
|
||||
rustfs_config::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
/// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
|
||||
///
|
||||
/// Credential, bucket, and endpoint validation run in this order because the
|
||||
@@ -372,7 +350,6 @@ pub(crate) async fn new_s3_compatible_warm_backend(
|
||||
signer_type: SignatureType::SignatureV4,
|
||||
..Default::default()
|
||||
}));
|
||||
let timeouts = transition_client_timeouts_from_env();
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
@@ -385,7 +362,7 @@ pub(crate) async fn new_s3_compatible_warm_backend(
|
||||
// Run the SSRF guard after the host-presence check so a host-less endpoint
|
||||
// keeps this constructor's stable error text.
|
||||
(params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
|
||||
let client = TransitionClient::new_with_timeouts(&endpoint, opts, params.provider_tag, timeouts).await?;
|
||||
let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
@@ -913,15 +890,6 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
|
||||
});
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "gcs"))]
|
||||
TierType::GCS => {
|
||||
return Err(AdminError {
|
||||
code: ERR_TIER_TYPE_UNSUPPORTED.code.clone(),
|
||||
message: "This build does not include the GCS backend; rebuild with the gcs feature".to_string(),
|
||||
status_code: StatusCode::NOT_IMPLEMENTED,
|
||||
});
|
||||
}
|
||||
#[cfg(feature = "gcs")]
|
||||
TierType::GCS => {
|
||||
if let Some(gcs_config) = tier.gcs.as_ref() {
|
||||
let dd = WarmBackendGCS::new(gcs_config, &tier.name).await;
|
||||
@@ -1038,27 +1006,6 @@ mod tests {
|
||||
|
||||
const PROBE_VERSION: &str = "remote-v2";
|
||||
|
||||
#[cfg(not(feature = "gcs"))]
|
||||
#[tokio::test]
|
||||
async fn gcs_backend_not_compiled_preserves_config() {
|
||||
let json = r#"{"name":"ARCHIVE","type":"gcs","gcs":{"bucket":"archive","creds":"secret"}}"#;
|
||||
let tier: TierConfig = serde_json::from_str(json).expect("GCS config remains readable without the backend");
|
||||
assert_eq!(tier.tier_type, TierType::GCS);
|
||||
let encoded = serde_json::to_vec(&tier).expect("GCS config remains writable");
|
||||
let restored: TierConfig = serde_json::from_slice(&encoded).expect("GCS config round trips");
|
||||
assert_eq!(restored.tier_type, TierType::GCS);
|
||||
let restored_gcs = restored.gcs.as_ref().expect("GCS settings preserved");
|
||||
assert_eq!(restored_gcs.bucket, "archive");
|
||||
assert_eq!(restored_gcs.creds, "secret");
|
||||
assert_eq!(tier.redacted().gcs.expect("redacted GCS settings").creds, "REDACTED");
|
||||
let error = match new_warm_backend(&tier, false).await {
|
||||
Ok(_) => panic!("an excluded GCS backend cannot be constructed"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code, ERR_TIER_TYPE_UNSUPPORTED.code);
|
||||
assert_eq!(error.status_code, StatusCode::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
struct CountingBackend {
|
||||
put_result: fn() -> Result<String, std::io::Error>,
|
||||
removes: Arc<AtomicUsize>,
|
||||
|
||||
@@ -26,7 +26,7 @@ use crate::services::tier::{
|
||||
tier_config::TierS3,
|
||||
warm_backend::{
|
||||
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
|
||||
build_transition_put_options, endpoint_authority, transition_client_timeouts_from_env,
|
||||
build_transition_put_options, endpoint_authority,
|
||||
},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
@@ -139,7 +139,6 @@ impl WarmBackendS3 {
|
||||
} else {
|
||||
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
|
||||
}
|
||||
let timeouts = transition_client_timeouts_from_env();
|
||||
let opts = Options {
|
||||
creds,
|
||||
secure: u.scheme() == "https",
|
||||
@@ -148,7 +147,7 @@ impl WarmBackendS3 {
|
||||
..Default::default()
|
||||
};
|
||||
let endpoint = endpoint_authority(&u)?;
|
||||
let client = TransitionClient::new_with_timeouts(&endpoint, opts, tier_type, timeouts).await?;
|
||||
let client = TransitionClient::new(&endpoint, opts, tier_type).await?;
|
||||
|
||||
let client = Arc::new(client);
|
||||
let core = TransitionCore(Arc::clone(&client));
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user