Compare commits

..

3 Commits

Author SHA1 Message Date
唐小鸭 a7e5181e2e test: make app fixtures portable to native Windows
- Token- and cache-shaped keys use Windows-legal spellings at the same
  sort position, so pagination still runs without relaxing key admission.
- The TLS root test falls back to a directory junction only when a
  symlink needs a privilege the runner lacks, and asserts the root is a link.
- Snapshot timestamps step by 1 us, which Windows SystemTime can represent.
- The receiver disk URL uses a slash-led path instead of a raw drive path.
2026-11-07 22:26:09 +08:00
唐小鸭 d45468ecab fix(ecstore): name the target record when a migration write fails
The existence probe reports a blocked record with its path, but the
commit path returns bare disk errors. On Windows a regular file in the
record's place reads as missing, so migration only fails at the write,
and startup reported "File access denied" without naming the record.
2026-11-07 22:26:09 +08:00
唐小鸭 4a69fd2e2e test: bypass the disk independence check in path-only topology tests
These tests parse endpoints whose local paths are never created. Unix
leaves missing paths out of the physical-disk independence check, but
Windows resolves them to their volume, so every disk counted as one
shared device and the parsing assertions never ran. The check keeps its
own coverage in the dedicated shared-volume tests.
2026-11-07 22:26:09 +08:00
568 changed files with 37828 additions and 61678 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=0374e6b9b061dd5d622686301386232a4aa1cb22eedc18af70d2a7411bc66f8d
sha256-linux=f2cee240f7a744ae991971820f48a6c9c28d19f525e8b2345fcb76eeaaf1e382
sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26
+1 -1
View File
@@ -1 +1 @@
sha256=4eac269f9a59b2f9be4e7a87ea6aa38454f5ebe10ede35c9b8d4218137d2cd35
sha256=e4ee3898b840a8374cb02c64342cc15f8d7ecd1becaf4b4a2a4657290f9e7ff1
+1 -1
View File
@@ -1 +1 @@
sha256=fbabc660e6e01fd8ff6b9d1de7d68e58242d60c942f85aed5074b0ec2918c154
sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
-5
View File
@@ -70,11 +70,6 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
@echo "📦 Checking s3s footprint ratchet..."
./scripts/check_s3s_footprint.sh
.PHONY: test-count-ratchet-check
test-count-ratchet-check: ## Check no workspace crate lost tests or gained #[ignore] against the committed baseline
@echo "🧪 Checking per-crate test-count ratchet..."
./scripts/check_test_count_ratchet.sh
.PHONY: fips-wording-check
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
@echo "📣 Checking cryptographic capability wording guard..."
+2 -2
View File
@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
./scripts/check_no_planning_docs.sh
.PHONY: pre-commit
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
@echo "✅ All pre-commit checks passed!"
.PHONY: pre-pr
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
@echo "✅ All pre-PR checks passed!"
.PHONY: dev-check
-3
View File
@@ -52,11 +52,8 @@ script-tests: ## Run shell script tests
./scripts/test_python_bin.sh
$(RUSTFS_PYTHON_BIN) ./scripts/test_check_doc_paths.py
./scripts/check_embedded_secrets.sh --self-test
./scripts/check_s3s_footprint.sh --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
+2 -23
View File
@@ -110,7 +110,7 @@ threads-required = "num-test-threads"
# Reserve capacity so sibling disk-heavy tests cannot consume its 2s
# registration budget; retain the queued backup and deadlock assertions.
[[profile.default.overrides]]
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
threads-required = "num-test-threads"
[[profile.default.overrides]]
@@ -356,12 +356,6 @@ threads-required = "num-test-threads"
filter = 'package(rustfs-ecstore) & test(=store::init::tests::dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member)'
threads-required = "num-test-threads"
# These short-deadline real-disk progress probes need the test run's capacity
# for scanner and object-lock handoffs; keep their assertions and deadlines.
[[profile.default.overrides]]
filter = '(package(rustfs-scanner) & (test(=scanner::tests::cycle_persist_failure::later_cycle_post_commit_failure_reloads_durable_state) | test(=scanner::tests::running_main_loop_catches_up_pause_cleared_after_startup_observe))) | (package(rustfs-ecstore) & test(=store::init::tests::restored_transitioned_delete_causally_enqueues_free_version))'
threads-required = "num-test-threads"
# ---------------------------------------------------------------------------
# ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
# ---------------------------------------------------------------------------
@@ -416,7 +410,7 @@ threads-required = "num-test-threads"
# Reserve capacity so sibling disk-heavy tests cannot consume its 2s
# registration budget; retain the queued backup and deadlock assertions.
[[profile.ci.overrides]]
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
threads-required = "num-test-threads"
[[profile.ci.overrides]]
@@ -430,12 +424,6 @@ threads-required = "num-test-threads"
filter = 'package(rustfs-scanner) & (test(/^scanner::backlog::tests::native_/) | test(=scanner_folder::tests::mrf_ownership::mrf_ownership_manager_completion_preserves_scanner_pending) | test(/^scanner_io::tests::service_cohort::/))'
threads-required = "num-test-threads"
# Keep short-deadline real-disk progress probes from competing with unrelated
# tests for scanner and object-lock capacity. Assertions and deadlines stay on.
[[profile.ci.overrides]]
filter = '(package(rustfs-scanner) & (test(=scanner::tests::cycle_persist_failure::later_cycle_post_commit_failure_reloads_durable_state) | test(=scanner::tests::running_main_loop_catches_up_pause_cleared_after_startup_observe))) | (package(rustfs-ecstore) & test(=store::init::tests::restored_transitioned_delete_causally_enqueues_free_version))'
threads-required = "num-test-threads"
# Emit this expensive fixture's stage timings in CI and use tmpfs when safe.
[[profile.ci.overrides]]
filter = 'package(rustfs-ecstore) & test(=store::init::tests::decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page)'
@@ -710,14 +698,6 @@ test-group = 'ecstore-serial-flaky'
# the PR-lane e2e coverage for scanner usage snapshots consumed by quota and
# admin surfaces. It uses the same single-node, random-port, isolated-temp-dir
# fixture as the existing smoke modules.
#
# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
# starts two single-node servers from the same binary, one per
# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
# SDK script covering every operation of `impl S3 for FS` against both. Its
# comparator unit tests need no server. It meets the admission criteria
# unchanged; the module clause admits a new differential test without another
# filter edit.
[profile.e2e-smoke]
default-filter = """
package(e2e_test) & (
@@ -726,7 +706,6 @@ default-filter = """
| test(/^reliant::lifecycle::/)
| test(/^conditional_delete_objects_test::/)
| test(/^reliant::tiering::/)
| test(/^dual_stack_differential::/)
| test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)
| test(/^on_demand_migration::(get_basic_test::(get_miss_pulls_inline_and_serves_locally_afterwards|head_miss_answers_from_the_source_without_persisting)|interaction_test::test_odm_admin_config_is_redacted_and_status_counts_match_the_source)$/)
)
-37
View File
@@ -1,37 +0,0 @@
# Files that may still reference s3s (`s3s::` or `s3s_sigv4`) when Phase 1 of
# the gateway migration ends (rustfs/backlog#2734 task T0.6, rustfs/backlog#2741).
# Consumed only by `scripts/check_s3s_footprint.sh --mode allowlist`.
#
# One entry per line: a repository-relative file path or a git `:(glob)`
# pathspec. `**` matches across directories, `*` stays within one path
# component, and a directory entry matches everything under it. Blank lines
# and lines starting with `#` are ignored; an entry may not contain whitespace,
# so a note belongs on its own comment line.
#
# Every Rust file listed by `git grep -l '\bs3s\(::\|_sigv4\)' -- '*.rs'` that
# no entry matches fails the allowlist mode. Task T1.9 switches CI to that
# mode; until then `--dry-run` prints the files still outside this list, which
# is the Phase 1 tracking list.
#
# Never add crates/** other than crates/e2e_test: the engine crates leave s3s
# entirely, and the script rejects such an entry. Only the s3s-facing edge of
# the rustfs binary and the e2e oracle keep it.
# rustfs binary: the s3s-facing edge that survives Phase 1
rustfs/src/storage/ecfs.rs
rustfs/src/storage/s3_compat/**
rustfs/src/storage/access.rs
rustfs/src/auth.rs
rustfs/src/error.rs
rustfs/src/storage_api.rs
rustfs/src/app/storage_api.rs
rustfs/src/admin/router.rs
rustfs/src/app/metadata_route.rs
rustfs/src/server/http.rs
rustfs/src/server/layer.rs
rustfs/src/server/hybrid.rs
rustfs/src/protocols/client.rs
# e2e oracle: the test target that verifies S3 behaviour against s3s
crates/e2e_test/src/fake_s3_target/**
crates/e2e_test/src/**
-17
View File
@@ -1,17 +0,0 @@
# Exact baseline for the direct s3s footprint, consumed only by
# scripts/check_s3s_footprint.sh (rustfs/backlog#1677; rustfs/backlog#2734 T0.4).
#
# Each value is the exact measurement of the tree, not a ceiling. CI fails when a
# measured counter differs from its line here in either direction:
# - a larger measurement means new code widened the s3s surface: route it
# through the gateway abstractions instead; never raise a value here;
# - a smaller measurement means the PR shrank the surface: lower the matching
# value to the measured number in the same PR.
#
# files Rust files referencing s3s paths, repo-wide minus crates/e2e_test
# s3_error_lines lines holding s3_error!, same scope, minus those the script
# header's rule clears (rustfs_s3_types's macro); fails closed
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
files=65
s3_error_lines=219
ecstore_files=0
-64
View File
@@ -1,64 +0,0 @@
# Per-crate test-count baseline consumed by scripts/check_test_count_ratchet.sh
# (rustfs/backlog#2734 section 1 rule 3; T0.3 in rustfs/backlog#2738).
#
# One row per workspace member: <package>, tests, ignored, tab-separated.
# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)]
# attribute lines under the package directory; ignored counts #[ignore] and
# #[ignore = "..."]. Both columns are exact: CI fails when a crate's tests fall
# below or rise above its row, or when its ignored count changes. Raise a row in
# the PR that adds tests; lowering one is a reviewed statement that tests were
# intentionally removed. Regenerate with:
# scripts/check_test_count_ratchet.sh --update
e2e_test 1026 20
rustfs 6438 64
rustfs-audit 70 0
rustfs-checksums 26 0
rustfs-common 36 0
rustfs-concurrency 87 0
rustfs-config 43 0
rustfs-connect-symbol-catalog 6 0
rustfs-credentials 24 0
rustfs-crypto 67 0
rustfs-data-usage 58 0
rustfs-ecstore 7850 2
rustfs-extension-schema 23 0
rustfs-filemeta 361 0
rustfs-heal 1128 3
rustfs-heal-contracts 8 0
rustfs-iam 415 0
rustfs-io-core 52 0
rustfs-io-metrics 157 0
rustfs-keystone 31 0
rustfs-kms 716 25
rustfs-license 2 0
rustfs-lifecycle 156 0
rustfs-lock 182 0
rustfs-log-analyzer 91 0
rustfs-madmin 210 0
rustfs-notify 140 0
rustfs-object-capacity 92 0
rustfs-object-data-cache 141 0
rustfs-obs 387 0
rustfs-policy 758 0
rustfs-protocols 770 1
rustfs-protos 75 0
rustfs-replication 209 0
rustfs-rio 223 0
rustfs-rio-v2 34 6
rustfs-s3-client 71 0
rustfs-s3-ops 8 0
rustfs-s3-types 105 0
rustfs-s3select-api 232 0
rustfs-s3select-query 235 2
rustfs-scanner 1386 20
rustfs-scanner-metrics 90 0
rustfs-security-governance 20 0
rustfs-signer 34 0
rustfs-storage-api 34 0
rustfs-targets 612 26
rustfs-test-utils 1 0
rustfs-tls-runtime 44 0
rustfs-trusted-proxies 72 0
rustfs-utils 251 0
rustfs-zip 30 0
zstd 11 0
+3 -8
View File
@@ -28,12 +28,11 @@ runs:
- name: Install actionlint
shell: bash
env:
GH_TOKEN: ${{ github.token }}
ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
run: |
actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
python3 "$ASSET_DOWNLOADER" --repository rhysd/actionlint --tag v1.7.12 --asset actionlint_1.7.12_linux_amd64.tar.gz --output "$actionlint_dir/actionlint.tar.gz" --sha256 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8
curl --fail --location --silent --show-error \
--output "$actionlint_dir/actionlint.tar.gz" \
https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
rm "$actionlint_dir/actionlint.tar.gz"
@@ -97,10 +96,6 @@ runs:
shell: bash
run: ./scripts/check_s3s_footprint.sh
- name: Check per-crate test-count ratchet
shell: bash
run: ./scripts/check_test_count_ratchet.sh
- name: Check cryptographic capability wording
shell: bash
run: ./scripts/check_fips_wording.sh
+5 -23
View File
@@ -66,7 +66,10 @@ inputs:
runs:
using: "composite"
steps:
# Use the pinned code generators rather than Ubuntu's older packages.
# protobuf-compiler is deliberately absent: the setup-protoc step below
# installs 35.1 into the tool cache and prepends it to PATH, so the apt
# build (older, and never version-matched) was shadowed on every run and
# simply never used.
- name: Install system dependencies (Ubuntu)
if: runner.os == 'Linux'
shell: bash
@@ -76,9 +79,7 @@ runs:
build-essential \
pkg-config \
libssl-dev \
ripgrep \
python3 \
gh
ripgrep
# musl-gcc is needed by the native musl release leg, and zip/unzip by the
# release packaging steps. No CI test lane touches any of them.
@@ -87,32 +88,13 @@ runs:
shell: bash
run: sudo apt-get install -y musl-tools zip unzip
- name: Install pinned code generators through GitHub API (Linux x64)
if: runner.os == 'Linux' && runner.arch == 'X64'
shell: bash
env:
GH_TOKEN: ${{ github.token }}
ASSET_DOWNLOADER: ${{ github.action_path }}/download_release_asset.py
run: |
python3 "$ASSET_DOWNLOADER" --repository protocolbuffers/protobuf --tag v35.1 --asset protoc-35.1-linux-x86_64.zip --output "$RUNNER_TEMP/protoc.zip"
python3 "$ASSET_DOWNLOADER" --repository google/flatbuffers --tag v25.12.19 --asset 'Linux.flatc.binary.g++-13.zip' --output "$RUNNER_TEMP/flatc.zip"
python3 -m zipfile -e "$RUNNER_TEMP/protoc.zip" "$RUNNER_TEMP/protoc"
python3 -m zipfile -e "$RUNNER_TEMP/flatc.zip" "$RUNNER_TEMP/flatc"
chmod +x "$RUNNER_TEMP/protoc/bin/protoc" "$RUNNER_TEMP/flatc/flatc"
"$RUNNER_TEMP/protoc/bin/protoc" --version
"$RUNNER_TEMP/flatc/flatc" --version
echo "$RUNNER_TEMP/protoc/bin" >> "$GITHUB_PATH"
echo "$RUNNER_TEMP/flatc" >> "$GITHUB_PATH"
- name: Install protoc
if: runner.os != 'Linux' || runner.arch != 'X64'
uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
with:
version: "35.1"
repo-token: ${{ github.token }}
- name: Install flatc
if: runner.os != 'Linux' || runner.arch != 'X64'
uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
with:
version: "25.12.19"
@@ -1,84 +0,0 @@
#!/usr/bin/env python3
"""Download a named GitHub release asset through its API and verify its digest."""
import argparse
import hashlib
import json
from pathlib import Path
import re
import subprocess
import sys
import tempfile
from urllib.parse import quote
def download(repository, tag, name, output, expected_sha256=None):
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", repository):
raise ValueError("invalid release repository")
if not tag or not name or "/" in name or "\\" in name:
raise ValueError("invalid release tag or asset name")
if expected_sha256 is not None and not re.fullmatch(r"[0-9a-f]{64}", expected_sha256):
raise ValueError("invalid pinned asset digest")
result = subprocess.run(
["gh", "api", "--hostname", "github.com", f"repos/{repository}/releases/tags/{quote(tag, safe='')}"],
check=True, capture_output=True, text=True, timeout=60,
)
release = json.loads(result.stdout)
if not isinstance(release, dict) or release.get("tag_name") != tag:
raise ValueError("release metadata does not match the requested tag")
assets = release.get("assets")
if not isinstance(assets, list) or any(not isinstance(asset, dict) for asset in assets):
raise ValueError("invalid release asset inventory")
matches = [asset for asset in assets if asset.get("name") == name]
if len(matches) != 1:
raise ValueError("release must contain exactly one named asset")
asset = matches[0]
asset_id, size, digest = asset.get("id"), asset.get("size"), asset.get("digest")
if type(asset_id) is not int or asset_id <= 0 or type(size) is not int or not 0 < size <= 512 * 1024 * 1024:
raise ValueError("invalid release asset identity or size")
if not isinstance(digest, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
raise ValueError("release asset has no valid SHA-256 digest")
sha256 = digest.removeprefix("sha256:")
if expected_sha256 is not None and expected_sha256 != sha256:
raise ValueError("release digest differs from the pinned digest")
output = Path(output)
output.parent.mkdir(parents=True, exist_ok=True)
temporary = None
try:
with tempfile.NamedTemporaryFile(dir=output.parent, prefix=".release-asset-", delete=False) as stream:
temporary = Path(stream.name)
# gh authenticates to the API; its HTTP client strips credentials on
# cross-host redirects. No browser URL or token is supplied in argv.
subprocess.run(
["gh", "api", "--hostname", "github.com", "--header", "Accept: application/octet-stream", f"repos/{repository}/releases/assets/{asset_id}"],
check=True, stdout=stream, timeout=300,
)
with temporary.open("rb") as stream:
actual = hashlib.file_digest(stream, "sha256").hexdigest()
if temporary.stat().st_size != size or actual != sha256:
raise ValueError("downloaded release asset failed size or SHA-256 verification")
temporary.replace(output)
print(f"Verified release asset: {repository} {tag} {name} sha256={actual}")
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repository", required=True)
parser.add_argument("--tag", required=True)
parser.add_argument("--asset", required=True)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--sha256")
args = parser.parse_args()
try:
download(args.repository, args.tag, args.asset, args.output, args.sha256)
except (ValueError, KeyError, TypeError, json.JSONDecodeError, subprocess.SubprocessError, OSError) as error:
print(f"Release asset download failed: {error}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
+8 -129
View File
@@ -16,9 +16,7 @@ name: Continuous Integration
on:
push:
# gateway/integration receives the daily merge of main as a direct push
# (rustfs/backlog#2734), so its pushes must prove the result like main's.
branches: [ main, gateway/integration ]
branches: [ main ]
paths-ignore:
- "**.md"
- "docs/**"
@@ -144,23 +142,15 @@ jobs:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Install pinned typos through GitHub API
env:
GH_TOKEN: ${{ github.token }}
TYPOS_VERSION: 1.50.2
run: |
python3 .github/actions/setup/download_release_asset.py --repository crate-ci/typos --tag "v${TYPOS_VERSION}" --asset "typos-v${TYPOS_VERSION}-x86_64-unknown-linux-musl.tar.gz" --output "$RUNNER_TEMP/typos.tar.gz"
mkdir -p "$RUNNER_TEMP/typos"
tar -xzf "$RUNNER_TEMP/typos.tar.gz" -C "$RUNNER_TEMP/typos"
- name: Typos check with custom config file
run: '"$RUNNER_TEMP/typos/typos" .'
uses: crate-ci/typos@512fc24f32f44ab01972217aaaf3dc86ec234d53 # v1.50.2
# Fail early with compile-free checks for every pull request.
quick-checks:
name: Quick Checks
if: github.event_name != 'pull_request' || github.event.action != 'closed'
runs-on: ubuntu-latest
timeout-minutes: 20
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -967,8 +957,6 @@ jobs:
# Retain the producer image and fixture paths until the runtime pilot passes.
runs-on: sm-standard-4
timeout-minutes: 30
env:
RUSTFS_S3_STACK: legacy
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -1089,122 +1077,18 @@ jobs:
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
retention-days: 3
# The same e2e-smoke suite against the same debug binary with the gateway
# S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
# Test and Lint aggregate and becomes required at T2.10. The test-binary
# download, identity, membership, run and completeness steps must stay
# identical to e2e-tests; scripts/ci_gate.py enforces that.
e2e-tests-gateway:
name: e2e-smoke (gateway, report-only until T2.10)
needs: [ build-rustfs-debug-binary ]
runs-on: sm-standard-2
timeout-minutes: 30
continue-on-error: true
env:
RUSTFS_S3_STACK: gateway
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Install verification runtime
uses: ./.github/actions/test-runtime
- name: Download E2E test binaries
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: e2e-test-binaries
path: ${{ runner.temp }}
- name: Verify E2E archive identity
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install awscurl
run: |
python3 -m pip install --user --upgrade pip "awscurl==0.44"
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
- name: Verify awscurl
run: test -x "$AWSCURL_PATH"
# Download after the cache restore so the freshly built binary from the
# build job always wins over anything restored into target/debug.
- name: Download debug binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: rustfs-debug-binary
path: target/debug
- name: Make binary executable
run: chmod +x ./target/debug/rustfs
# Build the e2e test graph once. The archive is reused by the smoke
# selection guard, security exact-count check, and run below, avoiding a
# second compile of the same e2e_test target on cold runners (backlog#1645).
- name: Verify e2e smoke test membership
env:
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
run: |
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
# profile.e2e-smoke default-filter in .config/nextest.toml is the single
# wiring mechanism for e2e tests in CI — extend that filter instead of
# adding new e2e jobs here. Each test spawns its own rustfs server on a
# random port and reuses the downloaded debug binary above.
- name: Run e2e smoke suite
env:
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
SAMPLER_INTERVAL_SECS: "15"
run: |
./scripts/ci/resource_sampler.sh start e2e-smoke
trap './scripts/ci/resource_sampler.sh stop' EXIT
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
--status-level all --final-status-level all --failure-output final
- name: Verify e2e smoke execution completeness
run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-smoke-list.json" target/nextest/e2e-smoke/junit.xml
- name: Upload e2e smoke diagnostics
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
path: |
${{ runner.temp }}/rustfs-e2e-smoke-logs/
${{ runner.temp }}/rustfs-e2e-smoke-list.json
if-no-files-found: warn
- name: Upload e2e smoke JUnit report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-smoke-gateway-junit-${{ github.run_number }}-${{ github.run_attempt }}
path: target/nextest/e2e-smoke/junit.xml
if-no-files-found: warn
e2e-full:
name: End-to-End Tests (full merge gate)
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
# every PR, so it is gated to main and gateway/integration pushes, the merge
# queue, and manual dispatch. protocols / the 7 cluster suites / replication
# / #[ignore] are owned by other lanes (see .config/nextest.toml profile.e2e-full).
# every PR, so it is gated to main pushes, the merge queue, and manual
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
if: >-
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/gateway/integration'))
(github.event_name == 'push' && github.ref == 'refs/heads/main')
needs: [ build-rustfs-debug-binary ]
# Retain the producer image and fixture paths until the runtime pilot passes.
runs-on: sm-standard-4
@@ -1253,16 +1137,11 @@ jobs:
- name: Install mc
env:
GH_TOKEN: ${{ github.token }}
MC_VERSION: RELEASE.2025-08-13T08-35-41Z
MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
run: |
if ! command -v gh >/dev/null 2>&1; then
sudo apt-get update
sudo apt-get install -y gh
fi
MC_BINARY="mc.linux-amd64.${MC_VERSION}"
python3 .github/actions/setup/download_release_asset.py --repository minio/mc --tag "$MC_VERSION" --asset "$MC_BINARY" --output "$RUNNER_TEMP/mc" --sha256 "$MC_SHA256"
curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
echo "${MC_SHA256} $RUNNER_TEMP/mc" | sha256sum --check --status
chmod +x "$RUNNER_TEMP/mc"
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
+1 -1
View File
@@ -155,7 +155,7 @@ default build (lifecycle:
- ⚠️ PARTIALLY VIOLATED: serving-side `s3s` references remain in ecstore
(bucket metadata/replication/lifecycle DTOs and error mapping). The
count is ratcheted shrink-only by `scripts/check_s3s_footprint.sh`
(the `ecstore_files` line of `.config/s3s-footprint-baseline.txt`; the `object_lock` module was converted to
(`S3S_ECSTORE_FILES_BASELINE`; the `object_lock` module was converted to
storage-level types as the first ratchet step). Target state: the
baseline reaches zero and ecstore's `Cargo.toml` drops `s3s`.
Generated
+13 -87
View File
@@ -8697,15 +8697,6 @@ version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quick-xml"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
dependencies = [
"memchr",
]
[[package]]
name = "quick-xml"
version = "0.42.0"
@@ -9590,7 +9581,7 @@ dependencies = [
"proptest",
"prost 0.14.4",
"pyroscope",
"quick-xml 0.42.0",
"quick-xml",
"quinn",
"rand 0.10.3",
"rcgen",
@@ -9609,7 +9600,6 @@ dependencies = [
"rustfs-ecstore",
"rustfs-extension-schema",
"rustfs-filemeta",
"rustfs-gateway-types",
"rustfs-heal",
"rustfs-heal-contracts",
"rustfs-iam",
@@ -9847,7 +9837,6 @@ dependencies = [
"google-cloud-auth",
"google-cloud-storage",
"h2",
"hashbrown 0.17.1",
"hex-simd",
"hmac 0.13.0",
"hostname",
@@ -9874,7 +9863,7 @@ dependencies = [
"path-absolutize",
"pin-project-lite",
"proptest",
"quick-xml 0.42.0",
"quick-xml",
"rand 0.10.3",
"ratelimit",
"rcgen",
@@ -9891,7 +9880,6 @@ dependencies = [
"rustfs-data-usage",
"rustfs-erasure-codec",
"rustfs-filemeta",
"rustfs-gateway-types",
"rustfs-heal-contracts",
"rustfs-io-metrics",
"rustfs-lifecycle",
@@ -9999,48 +9987,6 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "rustfs-gateway-stream"
version = "0.2.0"
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
dependencies = [
"bitflags 2.13.2",
"bytes",
"http 1.5.0",
"http-body",
]
[[package]]
name = "rustfs-gateway-types"
version = "0.32.0+aws.2026-10-02"
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
dependencies = [
"bytes",
"crc-fast",
"futures-core",
"hex",
"http 1.5.0",
"md-5 0.11.0",
"percent-encoding",
"rustfs-gateway-stream",
"rustfs-gateway-xml",
"s3s",
"sha1 0.11.0",
"sha2 0.11.1",
"time",
"twox-hash",
"unicode-normalization",
"zeroize",
]
[[package]]
name = "rustfs-gateway-xml"
version = "0.4.0"
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
dependencies = [
"quick-xml 0.41.0",
]
[[package]]
name = "rustfs-heal"
version = "1.0.2"
@@ -10312,10 +10258,10 @@ dependencies = [
"metrics-util",
"proptest",
"rustfs-config",
"rustfs-gateway-types",
"rustfs-replication",
"rustfs-scanner-metrics",
"rustfs-storage-api",
"s3s",
"serial_test",
"temp-env",
"time",
@@ -10379,6 +10325,7 @@ dependencies = [
"reqwest",
"rmp-serde",
"rustfs-signer",
"s3s",
"serde",
"serde_json",
"sysinfo",
@@ -10418,7 +10365,7 @@ dependencies = [
"jiff",
"metrics",
"percent-encoding",
"quick-xml 0.42.0",
"quick-xml",
"rayon",
"rustc-hash",
"rustfs-config",
@@ -10600,13 +10547,10 @@ dependencies = [
"rustfs-config",
"rustfs-credentials",
"rustfs-ecstore",
"rustfs-gateway-stream",
"rustfs-gateway-types",
"rustfs-iam",
"rustfs-keystone",
"rustfs-policy",
"rustfs-rio",
"rustfs-s3-types",
"rustfs-storage-api",
"rustfs-test-utils",
"rustfs-tls-runtime",
@@ -10614,13 +10558,13 @@ dependencies = [
"rustfs-trusted-proxies",
"rustfs-utils",
"rustls",
"s3s",
"serde",
"serde_json",
"sha1 0.11.0",
"sha2 0.11.1",
"socket2",
"subtle",
"sync_wrapper",
"tempfile",
"thiserror 2.0.21",
"time",
@@ -10670,7 +10614,7 @@ dependencies = [
"regex",
"rmp",
"rmp-serde",
"rustfs-gateway-types",
"s3s",
"serde",
"time",
"tokio",
@@ -10765,13 +10709,11 @@ dependencies = [
"hyper-util",
"lazy_static",
"md-5 0.11.0",
"quick-xml 0.42.0",
"quick-xml",
"rand 0.10.3",
"rustfs-checksums",
"rustfs-config",
"rustfs-gateway-types",
"rustfs-rio",
"rustfs-s3-types",
"rustfs-signer",
"rustfs-storage-api",
"rustfs-tls-runtime",
@@ -10805,19 +10747,9 @@ dependencies = [
name = "rustfs-s3-types"
version = "1.0.2"
dependencies = [
"bytes",
"futures-core",
"hotpath",
"http 1.5.0",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"pin-project-lite",
"s3s",
"serde",
"serde_json",
"tokio",
]
[[package]]
@@ -10827,29 +10759,23 @@ dependencies = [
"arc-swap",
"async-compression",
"async-trait",
"base64-simd",
"brotli 9.1.1",
"bytes",
"chrono",
"crc-fast",
"datafusion",
"flatbuffers",
"flate2",
"futures",
"futures-core",
"hotpath",
"http 1.5.0",
"lz4",
"lz4_flex",
"memchr",
"metrics",
"parking_lot 0.13.0",
"rustfs-common",
"rustfs-ecstore",
"rustfs-gateway-types",
"rustfs-s3-types",
"rustfs-storage-api",
"rustfs-test-utils",
"s3s",
"serde_json",
"serial_test",
"thiserror 2.0.21",
@@ -10877,6 +10803,7 @@ dependencies = [
"parking_lot 0.13.0",
"rustfs-s3select-api",
"rustfs-test-utils",
"s3s",
"serde",
"serde_json",
"tokio",
@@ -10908,7 +10835,6 @@ dependencies = [
"rustfs-data-usage",
"rustfs-ecstore",
"rustfs-filemeta",
"rustfs-gateway-types",
"rustfs-heal",
"rustfs-heal-contracts",
"rustfs-lifecycle",
@@ -10966,10 +10892,9 @@ dependencies = [
"bytes",
"hotpath",
"http 1.5.0",
"http-body",
"hyper",
"rustfs-s3-types",
"rustfs-utils",
"s3s",
"serde_urlencoded",
"thiserror 2.0.21",
"time",
@@ -11028,6 +10953,7 @@ dependencies = [
"rustfs-utils",
"rustls",
"rustls-native-certs",
"s3s",
"serde",
"serde_json",
"sha2 0.11.1",
@@ -11419,7 +11345,7 @@ dependencies = [
"numeric_cast",
"phf 0.14.0",
"pin-project-lite",
"quick-xml 0.42.0",
"quick-xml",
"regex",
"s3s-multipart",
"s3s-rfc2047",
+1 -14
View File
@@ -48,7 +48,7 @@ members = [
"crates/replication", # Replication contracts and wire formats
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
"crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets)
"crates/s3-types", # S3 event and error type definitions
"crates/s3-types", # S3 event type definitions
"crates/s3-ops", # S3 operation definitions and mapping
"crates/s3select-api", # S3 Select API interface
"crates/s3select-query", # S3 Select query engine
@@ -143,13 +143,6 @@ rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.2" }
rustfs-utils = { path = "crates/utils", version = "1.0.2" }
rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
# The gateway's S3 types: the DTOs and persisted bucket-configuration codecs the
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
# the s3s-facing edge of the rustfs binary.
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
# Async Runtime and Networking
async-channel = "2.5.0"
async_zip = { default-features = false, version = "0.0.19" }
@@ -303,7 +296,6 @@ local-ip-address = "0.6.13"
log = "0.4"
memmap2 = "0.9.12"
lz4 = "1.28.1"
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
matchit = "0.9.2"
md-5 = "0.11.0"
mime_guess = "2.0.5"
@@ -336,10 +328,6 @@ rust-embed = { version = "8.13.0" }
rustc-hash = { version = "2.1.3" }
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
# The same s3s pin under the name the legacy edge of the gateway migration uses
# (crates/s3-types, feature `compat-s3s`). Keep `rev` identical to `s3s` above.
# DELETE BY T4.2 (rustfs/backlog#2784).
legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.4"
@@ -348,7 +336,6 @@ compact_str = "0.10.0"
snap = "1.1.2"
starshard = { version = "2.4.0" }
strum = { version = "0.28.0" }
sync_wrapper = "1.0.2"
sysinfo = "0.40.1"
temp-env = "0.3.6"
tempfile = "3.27.0"
+9 -21
View File
@@ -184,27 +184,15 @@ const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
/// one-time consumption of authenticated RPC signatures.
///
/// Nonces remain protected until their signed timestamp is outside the observed wall-clock
/// validity window, including after clock rollback. Current-time signatures live about five
/// minutes; the maximum accepted forward clock skew can extend that to ten minutes. A body-bound
/// v3 request consumes both its v3 and compatibility-v2 nonces, so size for receiver nonce intake,
/// not S3 request rate or CPU count.
///
/// Automatic admission budgets 13% of effective process memory for signed-time
/// hash tables, allocator overhead and simultaneous old/new tables during growth.
/// The reported count is an absolute UUID payload ceiling, not guaranteed throughput:
/// actual byte admission can reject earlier depending on timestamp distribution.
/// Tables grow with demand and are released together when their signed time expires.
/// CPU count, a historical minimum, and a fixed 32M ceiling do not constrain this budget.
/// If memory detection fails, the historical default remains a bounded count fallback.
///
/// Explicit values remain count ceilings with the historical minimum of 1,048,576
/// for configuration compatibility, and bypass automatic byte admission. That
/// compatibility floor does not apply to the automatically detected memory budget.
/// Zero never disables replay protection.
/// Overflow rejects the request without evicting a still-valid identity, increments
/// `rustfs_system_network_internode_replay_cache_overflow_total`, and returns resource
/// exhaustion rather than invalid credentials. Measure headroom before overriding.
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
/// state holds roughly `authenticated RPC RPS x 601s` entries. This default is the minimum floor:
/// explicit operator values and resource-aware auto sizing both clamp upward to at least this
/// value. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
/// the shared secret) — and increments
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
+2 -3
View File
@@ -32,9 +32,8 @@ pub const ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: &str = "RUSTFS_OBJECT_MEDIUM_
/// Environment variable name for maximum concurrent disk reads for object operations.
/// - Purpose: Limit the number of concurrent disk read operations for object reads to prevent I/O saturation.
/// - Unit: request count (usize) for GET; estimated drive fanout for native LIST.
/// - Semantics: GET bodies and native LIST walks use separate per-instance budgets of this size.
/// A LIST consumes up to one budget, including its disk-producer cleanup. `0` disables both caps.
/// - Unit: request count (usize).
/// - Semantics: Throttling disk reads helps maintain overall system responsiveness under load.
/// - Example: `export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=16`
/// - Note: This setting may interact with OS-level I/O scheduling and should be tuned based on hardware capabilities.
pub const ENV_OBJECT_MAX_CONCURRENT_DISK_READS: &str = "RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS";
+1 -106
View File
@@ -17,8 +17,7 @@
//! when PublicAccessBlock configuration is missing or explicitly set.
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
use crate::sts_query_compat_test::{OPA_AUTH_TOKEN, OpaMock};
use aws_sdk_s3::types::{PublicAccessBlockConfiguration, Tag, Tagging};
use aws_sdk_s3::types::PublicAccessBlockConfiguration;
use tracing::info;
async fn setup_public_bucket(
@@ -70,110 +69,6 @@ async fn anonymous_get_object(
local_http_client().get(&url).send().await
}
#[tokio::test]
async fn test_plain_http_matches_secure_transport_false_bucket_policy() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
assert!(env.url.starts_with("http://"), "fixture must exercise plaintext HTTP: {}", env.url);
let bucket = "secure-transport-policy";
let key = "plain-http.txt";
let admin = env.create_s3_client();
admin.create_bucket().bucket(bucket).send().await?;
admin
.put_object()
.bucket(bucket)
.key(key)
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"plain http"))
.send()
.await?;
admin
.put_bucket_policy()
.bucket(bucket)
.policy(
serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": format!("arn:aws:s3:::{bucket}/*"),
"Condition": {"Bool": {"aws:SecureTransport": "false"}}
}]
})
.to_string(),
)
.send()
.await?;
let response = anonymous_get_object(&env, bucket, key).await?;
assert_eq!(response.status().as_u16(), 200);
assert_eq!(response.bytes().await?.as_ref(), b"plain http");
Ok(())
}
/// Root credentials have no attached IAM policy, so these requests depend on
/// the owner flag reaching policy evaluation unchanged.
#[tokio::test]
async fn test_root_requests_keep_policy_owner_flag() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
let opa = OpaMock::start().await?;
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(
vec![],
&[
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
],
)
.await?;
let bucket = "root-owner-flag";
let key = "owner.txt";
let client = env.create_s3_client();
client.create_bucket().bucket(bucket).send().await?;
client
.put_object()
.bucket(bucket)
.key(key)
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"owner access"))
.send()
.await?;
// Duplicate tag keys are ambiguous, so PutBucketTagging rejects them
// before they can reach stored metadata or the OPA tag-loading path.
let duplicate = client
.put_bucket_tagging()
.bucket(bucket)
.tagging(
Tagging::builder()
.tag_set(Tag::builder().key("department").value("finance").build()?)
.tag_set(Tag::builder().key("department").value("engineering").build()?)
.build()?,
)
.send()
.await
.expect_err("duplicate bucket tag keys must be rejected");
assert_eq!(duplicate.into_service_error().meta().code(), Some("InvalidTag"));
client
.put_bucket_tagging()
.bucket(bucket)
.tagging(
Tagging::builder()
.tag_set(Tag::builder().key("department").value("finance").build()?)
.build()?,
)
.send()
.await?;
let object = client.get_object().bucket(bucket).key(key).send().await?;
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"owner access");
Ok(())
}
/// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it
/// and no PublicAccessBlock configuration exists (ConfigNotFound).
#[tokio::test]
@@ -1,512 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! What one stack answered, and how two answers are compared.
//!
//! An [`Observation`] keeps only what a client acts on: the HTTP status, the S3 error code, the
//! allowlisted response headers and a few typed fields decoded from the response. The error
//! message is kept for diagnostics and never compared: message wording is judged by the wire
//! corpus, not here. The comparator is pure so it can be tested without a server.
use serde::Serialize;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
/// Response headers compared exactly (after [`normalized_header_value`]).
const HEADER_ALLOWLIST: [&str; 7] = [
"content-type",
"x-amz-version-id",
"etag",
"x-amz-delete-marker",
"content-range",
"accept-ranges",
"x-amz-request-charged",
];
/// Every response header with this prefix is compared as well.
const HEADER_ALLOWLIST_PREFIX: &str = "x-amz-server-side-encryption";
/// Placeholder for a field one side did not send.
const ABSENT: &str = "<absent>";
/// The lowercase header name if the comparator looks at this header.
pub(crate) fn compared_header(name: &str) -> Option<String> {
let name = name.to_ascii_lowercase();
(HEADER_ALLOWLIST.contains(&name.as_str()) || name.starts_with(HEADER_ALLOWLIST_PREFIX)).then_some(name)
}
/// A version ID is minted per process, so its value never matches across two servers; only
/// whether it is the literal `null` (an unversioned write) carries meaning.
fn normalized_header_value(name: &str, value: &str) -> String {
if name == "x-amz-version-id" && value != "null" {
"<version-id>".to_owned()
} else {
value.to_owned()
}
}
/// One request's outcome as a client sees it.
#[derive(Clone, Debug, Serialize)]
pub(crate) struct Observation {
/// The S3 operation, named as in the `s3s::S3` trait (`GetObject`).
pub op: &'static str,
/// Which request of the script this was (`normal`, `negative`, or a setup label).
pub case: &'static str,
/// HTTP status; 0 when no response arrived.
pub status: u16,
/// The S3 error code, or the SDK's error kind when there was no service error.
pub error_code: Option<String>,
/// Allowlisted headers only, lowercase names, normalized values.
pub headers: BTreeMap<String, String>,
/// Typed values decoded from the response body, never free text.
pub fields: BTreeMap<String, String>,
/// Diagnostic only; never compared.
#[serde(skip)]
pub message: Option<String>,
}
impl Observation {
/// Builds an observation, keeping only the allowlisted headers.
pub(crate) fn new<'h>(
op: &'static str,
case: &'static str,
status: u16,
error_code: Option<String>,
headers: impl IntoIterator<Item = (&'h str, &'h str)>,
) -> Self {
let headers = headers
.into_iter()
.filter_map(|(name, value)| compared_header(name).map(|name| (name, value)))
.map(|(name, value)| {
let value = normalized_header_value(&name, value);
(name, value)
})
.collect();
Self {
op,
case,
status,
error_code,
headers,
fields: BTreeMap::new(),
message: None,
}
}
pub(crate) fn with_field(mut self, name: &str, value: impl Into<String>) -> Self {
self.fields.insert(name.to_owned(), value.into());
self
}
pub(crate) fn is_success(&self) -> bool {
(200..300).contains(&self.status)
}
}
/// One field on which the two stacks answered differently.
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct Difference {
pub op: String,
pub case: String,
pub field: String,
pub a: String,
pub b: String,
}
/// Every difference between two transcripts of the same script, matched by `(op, case)`.
pub(crate) fn compare(a: &[Observation], b: &[Observation]) -> Vec<Difference> {
let mut differences = Vec::new();
let (index_a, duplicates_a) = index(a);
let (index_b, duplicates_b) = index(b);
for (op, case) in duplicates_a.union(&duplicates_b) {
let count = |side: &[Observation]| {
side.iter()
.filter(|observation| (observation.op, observation.case) == (*op, *case))
.count()
.to_string()
};
differences.push(difference(op, case, "duplicate_case", count(a), count(b)));
}
let keys: BTreeSet<_> = index_a.keys().chain(index_b.keys()).copied().collect();
for key @ (op, case) in keys {
match (index_a.get(&key), index_b.get(&key)) {
(Some(a), Some(b)) => compare_one(a, b, &mut differences),
(a, b) => {
let presence = |side: Option<&&Observation>| if side.is_some() { "present" } else { ABSENT }.to_owned();
differences.push(difference(op, case, "observation", presence(a), presence(b)));
}
}
}
differences
}
type Key = (&'static str, &'static str);
fn index(side: &[Observation]) -> (BTreeMap<Key, &Observation>, BTreeSet<Key>) {
let mut index = BTreeMap::new();
let mut duplicates = BTreeSet::new();
for observation in side {
let key = (observation.op, observation.case);
if index.insert(key, observation).is_some() {
duplicates.insert(key);
}
}
(index, duplicates)
}
fn difference(op: &str, case: &str, field: &str, a: String, b: String) -> Difference {
Difference {
op: op.to_owned(),
case: case.to_owned(),
field: field.to_owned(),
a,
b,
}
}
fn compare_one(a: &Observation, b: &Observation, differences: &mut Vec<Difference>) {
let mut field = |name: String, value_a: Option<&str>, value_b: Option<&str>| {
if value_a != value_b {
let show = |value: Option<&str>| value.unwrap_or(ABSENT).to_owned();
differences.push(difference(a.op, a.case, &name, show(value_a), show(value_b)));
}
};
field(
"status".to_owned(),
Some(a.status.to_string().as_str()),
Some(b.status.to_string().as_str()),
);
field("error_code".to_owned(), a.error_code.as_deref(), b.error_code.as_deref());
let headers: BTreeSet<_> = a.headers.keys().chain(b.headers.keys()).collect();
for name in headers {
field(
format!("header.{name}"),
a.headers.get(name).map(String::as_str),
b.headers.get(name).map(String::as_str),
);
}
let fields: BTreeSet<_> = a.fields.keys().chain(b.fields.keys()).collect();
for name in fields {
field(
format!("body.{name}"),
a.fields.get(name).map(String::as_str),
b.fields.get(name).map(String::as_str),
);
}
}
/// Operations observed on both sides.
pub(crate) fn compared_ops(a: &[Observation], b: &[Observation]) -> BTreeSet<&'static str> {
let ops_b: BTreeSet<_> = b.iter().map(|observation| observation.op).collect();
a.iter()
.map(|observation| observation.op)
.filter(|op| ops_b.contains(op))
.collect()
}
/// `(op, case)` pairs observed on both sides.
pub(crate) fn compared_cases(a: &[Observation], b: &[Observation]) -> usize {
let cases_b: BTreeSet<_> = b.iter().map(|observation| (observation.op, observation.case)).collect();
a.iter()
.map(|observation| (observation.op, observation.case))
.collect::<BTreeSet<_>>()
.intersection(&cases_b)
.count()
}
/// The JSON report written for every run.
#[derive(Debug, Serialize)]
pub(crate) struct Report {
pub schema: u32,
pub run: String,
pub stack_a: String,
pub stack_b: String,
pub region_a: Option<String>,
pub region_b: Option<String>,
pub compared_ops: usize,
pub compared_cases: usize,
pub differences: Vec<Difference>,
}
impl Report {
/// Writes the report as `<directory>/<run>.json` and returns its path.
pub(crate) fn write_to(&self, directory: &Path) -> std::io::Result<PathBuf> {
std::fs::create_dir_all(directory)?;
let path = directory.join(format!("{}.json", self.run));
let json = serde_json::to_vec_pretty(self).map_err(std::io::Error::other)?;
std::fs::write(&path, json)?;
Ok(path)
}
}
/// One line per difference, for a failing assertion.
pub(crate) fn render(differences: &[Difference]) -> String {
differences
.iter()
.map(|d| format!("({}, {}, {}, a={:?}, b={:?})", d.op, d.case, d.field, d.a, d.b))
.collect::<Vec<_>>()
.join("\n")
}
#[cfg(test)]
mod tests {
use super::*;
fn ok(op: &'static str) -> Observation {
Observation::new(op, "normal", 200, None, [("content-type", "application/xml")])
}
fn fields(differences: &[Difference]) -> Vec<&str> {
differences.iter().map(|d| d.field.as_str()).collect()
}
#[test]
fn identical_transcripts_have_no_difference() {
let a = vec![
ok("HeadBucket"),
ok("GetBucketLocation").with_field("location_constraint", "us-east-1"),
];
let b = a.clone();
assert_eq!(compare(&a, &b), Vec::new());
assert_eq!(compared_ops(&a, &b).len(), 2);
}
#[test]
fn a_status_difference_is_reported() {
let a = vec![ok("PutBucketAcl")];
let b = vec![Observation::new(
"PutBucketAcl",
"normal",
403,
None,
[("content-type", "application/xml")],
)];
assert_eq!(
compare(&a, &b),
vec![Difference {
op: "PutBucketAcl".into(),
case: "normal".into(),
field: "status".into(),
a: "200".into(),
b: "403".into(),
}]
);
}
#[test]
fn an_error_code_difference_is_reported() {
let a = vec![Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), [])];
let b = vec![Observation::new(
"GetObject",
"negative",
404,
Some("NoSuchBucket".into()),
[],
)];
let differences = compare(&a, &b);
assert_eq!(fields(&differences), ["error_code"]);
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("NoSuchKey", "NoSuchBucket"));
}
#[test]
fn an_error_code_on_one_side_only_is_reported() {
let a = vec![Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), [])];
let b = vec![Observation::new("HeadBucket", "negative", 404, None, [])];
let differences = compare(&a, &b);
assert_eq!(fields(&differences), ["error_code"]);
assert_eq!(differences[0].b, ABSENT);
}
#[test]
fn an_allowlisted_header_difference_is_reported() {
let a = vec![Observation::new("HeadObject", "normal", 200, None, [("ETag", "\"aa\"")])];
let b = vec![Observation::new("HeadObject", "normal", 200, None, [("etag", "\"bb\"")])];
assert_eq!(fields(&compare(&a, &b)), ["header.etag"]);
}
#[test]
fn a_header_sent_by_one_side_only_is_reported() {
let a = vec![Observation::new(
"DeleteObject",
"normal",
204,
None,
[("x-amz-delete-marker", "true")],
)];
let b = vec![Observation::new("DeleteObject", "normal", 204, None, [])];
let differences = compare(&a, &b);
assert_eq!(fields(&differences), ["header.x-amz-delete-marker"]);
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("true", ABSENT));
}
#[test]
fn every_server_side_encryption_header_is_compared() {
let header = "x-amz-server-side-encryption-customer-key-md5";
let a = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "one")])];
let b = vec![Observation::new("PutObject", "sse-c", 200, None, [(header, "two")])];
assert_eq!(fields(&compare(&a, &b)), [format!("header.{header}")]);
}
#[test]
fn headers_outside_the_allowlist_are_not_compared() {
let a = vec![Observation::new(
"ListBuckets",
"normal",
200,
None,
[
("date", "Mon"),
("x-amz-request-id", "1"),
("server", "a"),
("content-length", "10"),
],
)];
let b = vec![Observation::new(
"ListBuckets",
"normal",
200,
None,
[
("date", "Tue"),
("x-amz-request-id", "2"),
("server", "b"),
("content-length", "11"),
],
)];
assert_eq!(compare(&a, &b), Vec::new());
assert!(a[0].headers.is_empty(), "non-allowlisted headers were kept: {:?}", a[0].headers);
}
#[test]
fn distinct_version_ids_are_not_a_difference() {
let a = vec![Observation::new(
"PutObject",
"normal",
200,
None,
[("x-amz-version-id", "0b1c")],
)];
let b = vec![Observation::new(
"PutObject",
"normal",
200,
None,
[("x-amz-version-id", "9f8e")],
)];
assert_eq!(compare(&a, &b), Vec::new());
}
#[test]
fn a_null_version_id_against_a_real_one_is_reported() {
let a = vec![Observation::new(
"PutObject",
"normal",
200,
None,
[("x-amz-version-id", "null")],
)];
let b = vec![Observation::new(
"PutObject",
"normal",
200,
None,
[("x-amz-version-id", "9f8e")],
)];
assert_eq!(fields(&compare(&a, &b)), ["header.x-amz-version-id"]);
}
#[test]
fn a_body_field_difference_is_reported() {
let a = vec![ok("GetBucketLocation").with_field("location_constraint", "us-east-1")];
let b = vec![ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")];
let differences = compare(&a, &b);
assert_eq!(fields(&differences), ["body.location_constraint"]);
assert_eq!((differences[0].a.as_str(), differences[0].b.as_str()), ("us-east-1", "eu-west-1"));
}
#[test]
fn a_body_field_on_one_side_only_is_reported() {
let a = vec![ok("ListObjectsV2").with_field("keys", "[]")];
let b = vec![ok("ListObjectsV2")];
assert_eq!(fields(&compare(&a, &b)), ["body.keys"]);
}
#[test]
fn error_messages_are_never_compared() {
let mut a = Observation::new("GetObject", "negative", 404, Some("NoSuchKey".into()), []);
let mut b = a.clone();
a.message = Some("The specified key does not exist.".into());
b.message = Some("Object not found".into());
assert_eq!(compare(&[a], &[b]), Vec::new());
}
#[test]
fn a_case_observed_on_one_side_only_is_reported() {
let a = vec![ok("HeadBucket"), ok("ListBuckets")];
let b = vec![ok("HeadBucket")];
let differences = compare(&a, &b);
assert_eq!(fields(&differences), ["observation"]);
assert_eq!(differences[0].op, "ListBuckets");
assert_eq!(compared_ops(&a, &b).into_iter().collect::<Vec<_>>(), ["HeadBucket"]);
}
#[test]
fn a_case_recorded_twice_is_reported() {
let a = vec![ok("HeadBucket"), ok("HeadBucket")];
let b = vec![ok("HeadBucket")];
assert_eq!(fields(&compare(&a, &b)), ["duplicate_case"]);
}
#[test]
fn compared_ops_counts_operations_not_cases() {
let negative = Observation::new("HeadBucket", "negative", 404, Some("NotFound".into()), []);
let a = vec![ok("HeadBucket"), negative, ok("ListBuckets")];
let b = a.clone();
assert_eq!(compared_ops(&a, &b).len(), 2);
}
#[test]
fn the_report_lists_each_difference_as_op_field_a_b() {
let directory = std::env::temp_dir().join(format!("dual-stack-report-{}", uuid::Uuid::new_v4()));
let report = Report {
schema: 1,
run: "unit".into(),
stack_a: "legacy".into(),
stack_b: "legacy".into(),
region_a: None,
region_b: Some("eu-west-1".into()),
compared_ops: 1,
compared_cases: 1,
differences: compare(
&[ok("GetBucketLocation").with_field("location_constraint", "us-east-1")],
&[ok("GetBucketLocation").with_field("location_constraint", "eu-west-1")],
),
};
let path = report.write_to(&directory).expect("write the report");
let json: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).expect("read the report")).expect("parse");
let _ = std::fs::remove_dir_all(&directory);
assert_eq!(path.file_name().and_then(|name| name.to_str()), Some("unit.json"));
assert_eq!(json["compared_ops"], 1);
assert_eq!(
json["differences"],
serde_json::json!([{
"op": "GetBucketLocation",
"case": "normal",
"field": "body.location_constraint",
"a": "us-east-1",
"b": "eu-west-1",
}])
);
}
}
@@ -1,199 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
//!
//! Stack A always runs `RUSTFS_S3_STACK=legacy`. Stack B runs the value of `RUSTFS_E2E_STACK_B`
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
//! set, starts B in another region so that a difference can be injected by hand. Both servers
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
//! is process-global, so two stacks in one process would read each other's buckets.
mod compare;
mod scenarios;
use crate::common::{RustFSTestEnvironment, init_logging, workspace_root};
use compare::{Difference, Observation, Report, compare, compared_cases, compared_ops, render};
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
type BoxError = Box<dyn std::error::Error + Send + Sync>;
const STACK_A: &str = "legacy";
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
const DEFAULT_STACK_B: &str = "legacy";
/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
/// cannot erase the injected difference. Not `us-east-1`: that region answers GetBucketLocation
/// with an empty LocationConstraint and re-creating an owned bucket with 200, so it would not
/// report its own name and would add CreateBucket differences to the control.
const CONTROL_REGION_A: &str = "us-west-2";
const CONTROL_REGION_B: &str = "eu-west-1";
/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
/// script that exercises real behavior from one in which both stacks fail the same way. Five of
/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
/// notification target configured), RestoreObject (the object is not archived) and
/// GetObjectTorrent (legacy answers `NoSuchKey`).
const MIN_NORMAL_SUCCESSES: usize = 68;
struct PairConfig {
stack_b: String,
region_a: Option<String>,
region_b: Option<String>,
}
impl PairConfig {
fn from_env() -> Self {
Self {
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
region_a: None,
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
}
}
}
/// Starts one server on `stack`, overriding its region only when asked.
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
let mut env = RustFSTestEnvironment::new().await?;
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
if let Some(region) = region {
vars.push(("RUSTFS_REGION", region));
}
env.start_rustfs_server_without_cleanup_with_env(&vars).await?;
Ok(env)
}
/// Starts stack A and stack B side by side, each with its own port and data directory.
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
tokio::try_join!(
start_stack(STACK_A, config.region_a.as_deref()),
start_stack(&config.stack_b, config.region_b.as_deref())
)
}
fn report_directory() -> PathBuf {
let target = std::env::var_os("CARGO_TARGET_DIR").map_or_else(|| PathBuf::from("target"), PathBuf::from);
let target = if target.is_absolute() {
target
} else {
workspace_root().join(target)
};
target.join("dual-stack")
}
struct Run {
report: Report,
path: PathBuf,
a: Vec<Observation>,
}
/// Runs the script against both stacks concurrently and writes the report.
async fn run_differential(label: &str, config: &PairConfig) -> Run {
let (server_a, server_b) = spawn_pair(config).await.expect("start stack A and stack B");
let (a, b) = tokio::join!(Box::pin(scenarios::run(&server_a)), Box::pin(scenarios::run(&server_b)));
let millis = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |elapsed| elapsed.as_millis());
let report = Report {
schema: 1,
run: format!("{label}-{millis}-{}", std::process::id()),
stack_a: STACK_A.to_owned(),
stack_b: config.stack_b.clone(),
region_a: config.region_a.clone(),
region_b: config.region_b.clone(),
compared_ops: compared_ops(&a, &b).len(),
compared_cases: compared_cases(&a, &b),
differences: compare(&a, &b),
};
let path = report.write_to(&report_directory()).expect("write the dual-stack report");
Run { report, path, a }
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn both_stacks_answer_every_operation_identically() {
init_logging();
let config = PairConfig::from_env();
let run = run_differential("self-comparison", &config).await;
let report = &run.report;
let mut observed: Vec<_> = run.a.iter().map(|observation| observation.op).collect();
observed.sort_unstable();
observed.dedup();
assert_eq!(
observed,
scenarios::OPERATIONS,
"the script must drive exactly the operations of impl S3 for FS"
);
assert_eq!(report.compared_ops, scenarios::OPERATIONS.len(), "report: {}", run.path.display());
assert!(
report.differences.is_empty(),
"stack {} and stack {} differ on {} of {} cases (report: {}):\n{}",
report.stack_a,
report.stack_b,
report.differences.len(),
report.compared_cases,
run.path.display(),
render(&report.differences)
);
let not_4xx: Vec<_> = run
.a
.iter()
.filter(|observation| observation.case == "negative" && !(400..500).contains(&observation.status))
.collect();
assert!(not_4xx.is_empty(), "negative cases must fail with a 4xx on {STACK_A}: {not_4xx:#?}");
let successes = run
.a
.iter()
.filter(|observation| observation.case == "normal" && observation.is_success())
.count();
assert!(
successes >= MIN_NORMAL_SUCCESSES,
"only {successes} normal cases succeeded on {STACK_A}: {:#?}",
run.a
);
}
#[tokio::test]
async fn differential_detects_an_injected_difference() {
init_logging();
let config = PairConfig {
stack_b: STACK_A.to_owned(),
region_a: Some(CONTROL_REGION_A.to_owned()),
region_b: Some(CONTROL_REGION_B.to_owned()),
};
let run = run_differential("injected-region", &config).await;
let expected = Difference {
op: "GetBucketLocation".to_owned(),
case: "normal".to_owned(),
field: "body.location_constraint".to_owned(),
a: CONTROL_REGION_A.to_owned(),
b: CONTROL_REGION_B.to_owned(),
};
assert!(
run.report.differences.contains(&expected),
"B started in {CONTROL_REGION_B} but GetBucketLocation was not reported (report: {}):\n{}",
run.path.display(),
render(&run.report.differences)
);
}
}
File diff suppressed because it is too large Load Diff
-6
View File
@@ -31,12 +31,6 @@ pub mod chaos;
#[cfg(test)]
pub mod fake_s3_target;
// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
// compared on status, error code, allowlisted headers and typed body fields.
#[cfg(test)]
mod dual_stack_differential;
// On-demand migration (backlog#2147): shared two-server environment, admin
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
// added by later ODM tasks.
+1 -78
View File
@@ -32,9 +32,7 @@
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx>
use crate::common::rustfs_binary_path_with_features;
use crate::common::{
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, signed_s3_request,
};
use crate::common::{AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via};
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, admin_request};
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
use anyhow::Result;
@@ -776,81 +774,6 @@ async fn test_webdav_core_operations_direct() -> Result<()> {
test_webdav_core_operations().await
}
#[tokio::test]
async fn test_webdav_content_type_directory_marker() -> Result<()> {
let env = ProtocolTestEnvironment::new().map_err(|e| anyhow::anyhow!("{}", e))?;
let admin_base_url = format!("http://{}", S3_TEST_ADDRESS);
let binary_path = rustfs_binary_path_with_features(Some("webdav"));
let mut server_process = Command::new(&binary_path)
.arg("--address")
.arg(S3_TEST_ADDRESS)
.env("RUSTFS_CONSOLE_ENABLE", "false")
.env("RUSTFS_WEBDAV_ENABLE", "true")
.env("RUSTFS_WEBDAV_ADDRESS", WEBDAV_ADDRESS)
.env("RUSTFS_WEBDAV_TLS_ENABLED", "false")
.arg(&env.temp_dir)
.spawn()?;
let result = async {
ProtocolTestEnvironment::wait_for_port_ready(WEBDAV_PORT, 30)
.await
.map_err(|e| anyhow::anyhow!("{}", e))?;
let client = create_client();
let bucket_name = "webdav-content-type-bucket";
let directory_name = "content-type-directory";
let webdav_base_url = format!("http://{}", WEBDAV_ADDRESS);
let mkcol_bucket = client
.request(
reqwest::Method::from_bytes(b"MKCOL").expect("MKCOL is a valid HTTP method"),
format!("{}/{}", webdav_base_url, bucket_name),
)
.header("Authorization", basic_auth_header())
.send()
.await?;
anyhow::ensure!(
mkcol_bucket.status().is_success() || mkcol_bucket.status().as_u16() == 201,
"MKCOL bucket should succeed, got: {}",
mkcol_bucket.status()
);
let mkcol_directory = client
.request(
reqwest::Method::from_bytes(b"MKCOL").expect("MKCOL is a valid HTTP method"),
format!("{}/{}/{}", webdav_base_url, bucket_name, directory_name),
)
.header("Authorization", basic_auth_header())
.send()
.await?;
anyhow::ensure!(
mkcol_directory.status().is_success() || mkcol_directory.status().as_u16() == 201,
"MKCOL directory should succeed, got: {}",
mkcol_directory.status()
);
let head_url = format!("{}/{}/{}/", admin_base_url, bucket_name, directory_name);
let head = signed_s3_request(http::Method::HEAD, &head_url, None, None, DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY)
.await
.map_err(|error| anyhow::anyhow!("{error}"))?;
anyhow::ensure!(head.status().is_success(), "HEAD directory marker should succeed, got: {}", head.status());
let content_type = head
.headers()
.get("content-type")
.and_then(|value| value.to_str().ok())
.unwrap_or_default();
anyhow::ensure!(
content_type == "application/x-directory",
"WebDAV directory marker must preserve its content type, got: {content_type}"
);
Ok(())
}
.await;
let _ = server_process.kill().await;
let _ = server_process.wait().await;
result
}
/// Extract the numeric value of the first occurrence of a DAV property in a
/// PROPFIND multistatus body. Namespace-prefix agnostic; a self-closing
/// (unreported) property yields None.
@@ -114,10 +114,10 @@ fn staged_commit(metadata: &Value, location: &str) -> Value {
})
}
async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -> TestResult {
async fn location_claim_preserves_other_users_objects_after_drop_and_restart(backing: &str) -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
let server_env = [("RUSTFS_TABLE_CATALOG_BACKING", "object")];
let server_env = [("RUSTFS_TABLE_CATALOG_BACKING", backing)];
env.start_rustfs_server_without_cleanup_with_env(&[("RUSTFS_TABLE_CATALOG_BACKING", "object")])
.await?;
env.wait_for_server_ready().await?;
@@ -147,6 +147,39 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -
StatusCode::OK,
)
.await?;
if backing == "durable-strong" {
catalog_request(
&env,
&root,
Method::POST,
&format!("/iceberg/v1/{bucket}/catalog/warehouse-index/backfill"),
None,
StatusCode::NO_CONTENT,
)
.await?;
let preflight = catalog_request(
&env,
&root,
Method::GET,
&format!("/iceberg/v1/{bucket}/catalog/migration"),
None,
StatusCode::OK,
)
.await?;
assert_eq!(preflight["blockers"], json!([]), "{preflight}");
let migration = catalog_request(
&env,
&root,
Method::POST,
&format!("/iceberg/v1/{bucket}/catalog/migration"),
None,
StatusCode::OK,
)
.await?;
assert_eq!(migration["ready_to_enable_durable_strong"], json!(true), "{migration}");
env.restart_server_preserving_data(vec![], &server_env).await?;
env.wait_for_server_ready().await?;
}
admin_create_user(&env, &creator.0, &creator.1).await?;
admin_create_user(&env, &reader.0, &reader.1).await?;
let create_statement = json!({
@@ -308,7 +341,7 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -
StatusCode::FORBIDDEN,
)
.await?;
{
if backing == "object" {
catalog_request(
&env,
&creator,
@@ -375,5 +408,10 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -
#[tokio::test]
async fn object_catalog_location_claim_preserves_objects_after_drop_and_restart() -> TestResult {
location_claim_preserves_other_users_objects_after_drop_and_restart().await
location_claim_preserves_other_users_objects_after_drop_and_restart("object").await
}
#[tokio::test]
async fn durable_strong_catalog_location_claim_preserves_objects_after_drop_and_restart() -> TestResult {
location_claim_preserves_other_users_objects_after_drop_and_restart("durable-strong").await
}
+1 -3
View File
@@ -121,7 +121,6 @@ test-util = []
e2e-test-hooks = []
[dependencies]
hashbrown.workspace = true
hotpath.workspace = true
rustfs-filemeta.workspace = true
rustfs-utils = { workspace = true, features = ["full"] }
@@ -141,7 +140,6 @@ rustfs-lifecycle.workspace = true
rustfs-s3-client = { workspace = true }
rustfs-s3-types = { workspace = true }
rustfs-data-usage.workspace = true
rustfs-gateway-types.workspace = true
rustfs-object-capacity.workspace = true
rustfs-object-data-cache = { workspace = true, features = ["runtime-memory"] }
arc-swap.workspace = true
@@ -160,6 +158,7 @@ serde = { workspace = true, features = ["derive"] }
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
bytesize.workspace = true
serde_json = { workspace = true, features = ["raw_value"] }
s3s = { workspace = true, features = ["minio"] }
http.workspace = true
opentelemetry.workspace = true
http-body = { workspace = true }
@@ -253,7 +252,6 @@ windows-sys = { workspace = true, features = [
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
aws-smithy-async.workspace = true
rustfs-s3-ops.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
+17 -19
View File
@@ -66,8 +66,6 @@ pub mod bucket {
}
pub mod bucket_lifecycle_ops {
pub use rustfs_lifecycle::{RestoreOutputLocation, RestoreRequest, RestoreS3Location, RestoreTag, RestoreTagging};
pub use crate::bucket::lifecycle::bucket_lifecycle_ops::{
ExpiryState, LifecycleOps, ManualTransitionCancelCheck, ManualTransitionProgressSink,
ManualTransitionQueueSnapshot, ManualTransitionRunExecution, ManualTransitionRunOptions,
@@ -139,7 +137,7 @@ pub mod bucket {
pub use crate::bucket::lifecycle::lifecycle::{
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate,
ObjectOpts, RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time,
format_lifecycle_date, object_opts_from_object_info, parse_lifecycle_date,
object_opts_from_object_info,
};
}
@@ -242,7 +240,7 @@ pub mod bucket {
}
pub mod object_lock {
pub use crate::bucket::object_lock::ObjectLockApi;
pub use crate::bucket::object_lock::{ObjectLockApi, ObjectLockStatusExt};
pub mod types {
pub use crate::bucket::object_lock::types::{
@@ -337,9 +335,9 @@ pub mod bucket {
pub mod utils {
pub use crate::bucket::utils::{
BucketConfigXml, BucketConfigXmlError, check_bucket_and_object_names, check_list_objs_args,
check_object_name_for_length_and_slash, check_valid_bucket_name_strict, deserialize, has_bad_path_component,
has_xml_invalid_control_characters, is_meta_bucketname, is_valid_object_prefix, serialize,
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
is_meta_bucketname, is_valid_object_prefix, serialize,
};
}
@@ -366,7 +364,7 @@ pub mod capacity {
}
pub mod object_api_utils {
pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader};
pub use crate::object_api::object_api_utils::{ObjReaderFn, PutObjReader, get_raw_etag, new_getobjectreader, to_s3s_etag};
}
pub mod cluster {
@@ -642,17 +640,17 @@ pub mod rpc {
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, RpcReplayCacheAdmissionError, S3PeerSys,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerDirtyUsageAcknowledgement,
ScannerPeerActivity, ScannerPeerDirtyUsageBucket, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease,
ScannerScopedDirtyUsageAckEntry, TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer,
check_and_record_signed_rpc_nonce, decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers,
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
rpc_auth_is_resource_exhausted, rpc_background_admission, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerDirtyUsageAcknowledgement, ScannerPeerActivity,
ScannerPeerDirtyUsageBucket, ScannerPeerDirtyUsageSnapshot, ScannerPublicationLease, ScannerScopedDirtyUsageAckEntry,
TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, rpc_background_admission,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
@@ -88,14 +88,12 @@ use rustfs_filemeta::{
FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatus, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info,
is_restored_object_on_disk,
};
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
use rustfs_gateway_types::persistence::{PersistedLifecycleConfiguration, PersistedLifecycleFilter};
use rustfs_lifecycle::{RestoreRequest, RestoreS3Location};
use rustfs_lock::{NamespaceLockGuard, error::LockError};
use rustfs_scanner_metrics::metrics::{
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
};
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::parse_bool};
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, ObjectLockConfiguration, RestoreRequest, RestoreRequestType};
use sha2::{Digest, Sha256};
use std::any::Any;
use std::collections::{BTreeMap, HashMap, HashSet};
@@ -302,7 +300,7 @@ impl LifecycleSys {
Arc::new(Self)
}
pub async fn get(&self, bucket: &str) -> Option<PersistedLifecycleConfiguration> {
pub async fn get(&self, bucket: &str) -> Option<BucketLifecycleConfiguration> {
match metadata_boundary::get_lifecycle_config(bucket).await {
Ok((lc, _)) => Some(lc),
Err(Error::ConfigNotFound) => None,
@@ -3559,7 +3557,7 @@ async fn stale_upload_lifecycle_due(
Err(_) => return None,
};
let has_size_filter = |filter: &PersistedLifecycleFilter| {
let has_size_filter = |filter: &s3s::dto::LifecycleRuleFilter| {
filter.object_size_greater_than.is_some()
|| filter.object_size_less_than.is_some()
|| filter
@@ -3568,7 +3566,7 @@ async fn stale_upload_lifecycle_due(
.is_some_and(|and| and.object_size_greater_than.is_some() || and.object_size_less_than.is_some())
};
let needs_size = lifecycle.rules.iter().any(|rule| {
rule.status.as_str() != "Disabled"
rule.status.as_str() != ExpirationStatus::DISABLED
&& rule.abort_incomplete_multipart_upload.is_some()
&& rule.filter.as_ref().is_some_and(has_size_filter)
});
@@ -4002,7 +4000,7 @@ pub fn init_background_stale_multipart_upload_cleanup(api: Arc<ECStore>) {
});
}
pub async fn validate_transition_tier(lc: &PersistedLifecycleConfiguration) -> Result<(), std::io::Error> {
pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Result<(), std::io::Error> {
for rule in &lc.rules {
if let Some(transitions) = &rule.transitions {
for transition in transitions {
@@ -4119,9 +4117,8 @@ pub(crate) async fn enqueue_immediate_expiry(api: Arc<ECStore>, oi: &ObjectInfo,
.map(lifecycle::object_opts_from_object_info)
.collect::<Vec<ObjectOpts>>();
let lock_config = configs.object_lock;
let evaluator_lock_config = lock_config.clone();
let events = match Evaluator::new(lifecycle)
.with_lock_retention(evaluator_lock_config)
.with_lock_retention(lock_config.clone())
.eval(&object_opts)
.await
{
@@ -4601,26 +4598,19 @@ pub async fn enqueue_transition_for_existing_objects_scoped(
}
}
fn lifecycle_rule_has_date_expiration(lc: &PersistedLifecycleConfiguration, rule_id: &str) -> bool {
fn lifecycle_rule_has_date_expiration(lc: &BucketLifecycleConfiguration, rule_id: &str) -> bool {
lc.rules.iter().any(|rule| {
rule.status == "Enabled"
rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
&& rule.id.as_deref() == Some(rule_id)
&& rule.expiration.as_ref().is_some_and(|expiration| expiration.date.is_some())
})
}
fn should_defer_date_expiry_for_recent_config_update(lc: &PersistedLifecycleConfiguration, now: OffsetDateTime) -> bool {
lc.expiry_updated_at
.as_ref()
.is_some_and(|updated_at| match rustfs_lifecycle::parse_lifecycle_date(updated_at) {
Ok(updated_at) => {
now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
}
Err(error) => {
warn!(error = %error, "Deferring date expiry with an unreadable update timestamp");
true
}
})
fn should_defer_date_expiry_for_recent_config_update(lc: &BucketLifecycleConfiguration, now: OffsetDateTime) -> bool {
lc.expiry_updated_at.as_ref().is_some_and(|updated_at| {
let updated_at = OffsetDateTime::from(updated_at.clone());
now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
})
}
async fn apply_existing_object_expiry(
@@ -4640,7 +4630,7 @@ async fn apply_existing_object_expiry(
struct ExistingObjectExpiryContext<'a> {
api: Arc<ECStore>,
bucket: &'a str,
lc: Arc<PersistedLifecycleConfiguration>,
lc: Arc<BucketLifecycleConfiguration>,
lock_config: Option<Arc<ObjectLockConfiguration>>,
bucket_incarnation_id: Uuid,
src: &'a LcEventSrc,
@@ -4660,9 +4650,8 @@ async fn enqueue_expiry_for_existing_object_group(
.iter()
.map(lifecycle::object_opts_from_object_info)
.collect::<Vec<ObjectOpts>>();
let evaluator_lock_config = context.lock_config.clone();
let events = match Evaluator::new(context.lc.clone())
.with_lock_retention(evaluator_lock_config)
.with_lock_retention(context.lock_config.clone())
.eval(&object_opts)
.await
{
@@ -4856,7 +4845,7 @@ fn manual_transition_version_marker(oi: &ObjectInfo) -> String {
async fn enqueue_transition_with_lifecycle_report(
api: Option<Arc<ECStore>>,
oi: &ObjectInfo,
lc: &PersistedLifecycleConfiguration,
lc: &BucketLifecycleConfiguration,
src: &LcEventSrc,
options: &ManualTransitionRunOptions,
report: &mut ManualTransitionRunReport,
@@ -4922,7 +4911,7 @@ async fn enqueue_transition_with_lifecycle_report(
false
}
async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &PersistedLifecycleConfiguration, src: &LcEventSrc) -> bool {
async fn enqueue_transition_with_lifecycle(oi: &ObjectInfo, lc: &BucketLifecycleConfiguration, src: &LcEventSrc) -> bool {
let event = lc.eval(&oi.to_lifecycle_opts()).await;
match event.action {
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
@@ -5436,8 +5425,12 @@ pub async fn post_restore_opts(version_id: &str, bucket: &str, object: &str) ->
})
}
fn select_restore_s3_location(rreq: &RestoreRequest) -> Result<Option<&RestoreS3Location>, std::io::Error> {
if rreq.type_.as_ref().is_none_or(|type_| type_.as_str() != "SELECT") {
fn select_restore_s3_location(rreq: &RestoreRequest) -> Result<Option<&s3s::dto::S3Location>, std::io::Error> {
if rreq
.type_
.as_ref()
.is_none_or(|type_| type_.as_str() != RestoreRequestType::SELECT)
{
return Ok(None);
}
let output_location = rreq
@@ -5479,20 +5472,21 @@ pub async fn put_restore_opts(
// the OutputLocation metadata instead of a write to `OutputLocation.S3`
// (backlog#1341). RestoreObject rejects SELECT at the API boundary; this
// is the fail-closed backstop for any other caller.
if rreq.type_.as_ref().is_some_and(|type_| type_.as_str() == "SELECT") {
if rreq
.type_
.as_ref()
.is_some_and(|type_| type_.as_str() == RestoreRequestType::SELECT)
{
return Err(std::io::Error::other("SELECT restore requests are not supported"));
}
for (k, v) in oi.user_defined.iter() {
meta.insert(k.to_string(), v.clone());
}
// The fleet capability gate guarantees physical copy-back support. Keep
// the completed transition identity for orphan recovery and remote cleanup.
rustfs_utils::http::metadata_compat::remove_str(&mut meta, rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID);
rustfs_utils::http::metadata_compat::remove_str(&mut meta, rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK);
if !oi.user_tags.is_empty() {
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
}
// Finalization publishes completion and its expiry under the object lock.
// The local copy-back commit must not make unfinished work eligible for expiry.
if let Some(expires) = oi.expires {
meta.insert(
"expires".to_string(),
@@ -5501,11 +5495,12 @@ pub async fn put_restore_opts(
.map_err(std::io::Error::other)?,
);
}
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
meta.insert(
metadata_keys::RESTORE.to_string(),
RestoreStatus {
is_restore_in_progress: Some(true),
restore_expiry_date: None,
is_restore_in_progress: Some(false),
restore_expiry_date: Some(restore_expiry),
}
.to_string(),
);
@@ -5519,7 +5514,6 @@ pub async fn put_restore_opts(
// computed MD5 is not the object's public plaintext ETag.
preserve_etag: oi.etag.clone(),
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
restore_copyback: true,
..Default::default()
})
}
@@ -5538,18 +5532,6 @@ impl LifecycleOps for ObjectInfo {
if self.transitioned_object.status != lifecycle::TRANSITION_COMPLETE {
return false;
}
if rustfs_utils::http::metadata_compat::get_consistent_str(
&self.user_defined,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
) == Some(rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1)
&& self
.user_defined
.get(metadata_keys::RESTORE)
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
.is_some_and(|status| status.is_restore_in_progress == Some(true))
{
return false;
}
!is_restored_object_on_disk(&self.user_defined)
}
}
@@ -5561,25 +5543,25 @@ pub trait RestoreRequestOps {
impl RestoreRequestOps for RestoreRequest {
fn validate(&self, _api: Arc<ECStore>) -> Result<(), std::io::Error> {
// SELECT type requires select_parameters, and vice versa
if self.type_.as_ref().is_none_or(|t| t.as_str() != "SELECT") && self.select_parameters.is_some() {
if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.select_parameters.is_some() {
return Err(std::io::Error::other("Select parameters can only be specified with SELECT request type"));
}
if let Some(type_) = &self.type_
&& type_.as_str() == "SELECT"
&& type_.as_str() == RestoreRequestType::SELECT
&& self.select_parameters.is_none()
{
return Err(std::io::Error::other("SELECT restore request requires select parameters to be specified"));
}
// OutputLocation is only valid for SELECT requests
if self.type_.as_ref().is_none_or(|t| t.as_str() != "SELECT") && self.output_location.is_some() {
if self.type_.as_ref().is_none_or(|t| t.as_str() != RestoreRequestType::SELECT) && self.output_location.is_some() {
return Err(std::io::Error::other("OutputLocation can only be specified with SELECT request type"));
}
select_restore_s3_location(self)?;
// Days must not be specified with SELECT requests
if let Some(type_) = &self.type_
&& type_.as_str() == "SELECT"
&& type_.as_str() == RestoreRequestType::SELECT
&& self.days.is_some_and(|d| d > 0)
{
return Err(std::io::Error::other("Days cannot be specified with SELECT restore request"));
@@ -5621,7 +5603,7 @@ pub fn lifecycle_version_delete_target(oi: &ObjectInfo) -> Option<ObjectToDelete
}
pub async fn eval_action_from_lifecycle(
lc: &PersistedLifecycleConfiguration,
lc: &BucketLifecycleConfiguration,
lock_config: Option<&ObjectLockConfiguration>,
oi: &ObjectInfo,
) -> lifecycle::Event {
@@ -6326,19 +6308,14 @@ mod tests {
use rustfs_data_usage::TierStats;
use rustfs_filemeta::metadata_keys;
use rustfs_filemeta::{FileInfo, FileMeta};
use rustfs_gateway_types::dto::MetadataEntry;
use rustfs_gateway_types::persistence::{
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockConfiguration as ObjectLockConfiguration,
PersistedObjectLockRule as ObjectLockRule,
};
use rustfs_gateway_types::persistence::{
PersistedLifecycleConfiguration, PersistedLifecycleExpiration, PersistedLifecycleRule,
PersistedNoncurrentVersionExpiration, PersistedTransition,
};
use rustfs_lifecycle::{RestoreOutputLocation as OutputLocation, RestoreRequest, RestoreS3Location as S3Location};
#[cfg(feature = "test-util")]
use rustfs_s3_client::transition_api::ReaderImpl;
use rustfs_scanner_metrics::metrics::{IlmAction, global_metrics};
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry,
NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
OutputLocation, RestoreRequest, RestoreRequestType, S3Location, Timestamp, Transition, TransitionStorageClass,
};
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
@@ -9364,7 +9341,7 @@ mod tests {
output_location,
select_parameters: None,
tier: None,
type_: Some("SELECT".to_owned()),
type_: Some(RestoreRequestType::from_static(RestoreRequestType::SELECT)),
}
}
@@ -9681,12 +9658,12 @@ mod tests {
#[test]
fn lifecycle_rule_has_date_expiration_detects_enabled_date_rule() {
let lc = PersistedLifecycleConfiguration {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
expiration: Some(PersistedLifecycleExpiration {
date: Some(lifecycle::format_lifecycle_date(OffsetDateTime::now_utc()).unwrap()),
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
date: Some(Timestamp::from(OffsetDateTime::now_utc())),
..Default::default()
}),
id: Some("rule-date".to_string()),
@@ -10203,15 +10180,14 @@ mod tests {
#[test]
fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
let now = OffsetDateTime::now_utc();
let recent = PersistedLifecycleConfiguration {
expiry_updated_at: Some(lifecycle::format_lifecycle_date(now - time::Duration::seconds(1)).unwrap()),
let recent = BucketLifecycleConfiguration {
expiry_updated_at: Some(Timestamp::from(now - time::Duration::seconds(1))),
rules: Vec::new(),
};
let stale = PersistedLifecycleConfiguration {
expiry_updated_at: Some(
lifecycle::format_lifecycle_date(now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1))
.unwrap(),
),
let stale = BucketLifecycleConfiguration {
expiry_updated_at: Some(Timestamp::from(
now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1),
)),
rules: Vec::new(),
};
@@ -10339,12 +10315,12 @@ mod tests {
assert_eq!(deleted.version_id, None);
}
fn expired_delete_marker_lifecycle() -> PersistedLifecycleConfiguration {
PersistedLifecycleConfiguration {
fn expired_delete_marker_lifecycle() -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
expiration: Some(PersistedLifecycleExpiration {
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
expired_object_delete_marker: Some(true),
..Default::default()
}),
@@ -10360,12 +10336,12 @@ mod tests {
}
}
fn latest_expiration_lifecycle() -> PersistedLifecycleConfiguration {
PersistedLifecycleConfiguration {
fn latest_expiration_lifecycle() -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
expiration: Some(PersistedLifecycleExpiration {
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
@@ -10381,12 +10357,12 @@ mod tests {
}
}
fn current_and_noncurrent_expiration_lifecycle() -> PersistedLifecycleConfiguration {
PersistedLifecycleConfiguration {
fn current_and_noncurrent_expiration_lifecycle() -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
expiration: Some(PersistedLifecycleExpiration {
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(30),
..Default::default()
}),
@@ -10394,7 +10370,7 @@ mod tests {
del_marker_expiration: None,
filter: None,
id: Some("expire-current-and-noncurrent".to_string()),
noncurrent_version_expiration: Some(PersistedNoncurrentVersionExpiration {
noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
}),
@@ -10405,12 +10381,12 @@ mod tests {
}
}
fn all_versions_expiration_lifecycle() -> PersistedLifecycleConfiguration {
PersistedLifecycleConfiguration {
fn all_versions_expiration_lifecycle() -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
expiration: Some(PersistedLifecycleExpiration {
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
expired_object_all_versions: Some(true),
..Default::default()
@@ -10429,29 +10405,29 @@ mod tests {
fn lock_enabled_without_default_retention() -> ObjectLockConfiguration {
ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: None,
}
}
fn lock_enabled_with_default_retention() -> ObjectLockConfiguration {
ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: Some(ObjectLockRule {
default_retention: Some(DefaultRetention {
days: Some(30),
mode: Some("COMPLIANCE".to_owned()),
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
years: None,
}),
}),
}
}
fn latest_transition_lifecycle() -> PersistedLifecycleConfiguration {
PersistedLifecycleConfiguration {
fn latest_transition_lifecycle() -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
@@ -10460,10 +10436,10 @@ mod tests {
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: Some(vec![PersistedTransition {
transitions: Some(vec![Transition {
days: Some(1),
date: None,
storage_class: Some("WARM".to_owned()),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]),
}],
}
@@ -13000,9 +12976,8 @@ mod tests {
for (name, object, lock_config, expected) in cases {
let object_opts = lifecycle::object_opts_from_object_info(&object);
let evaluator_lock_config = lock_config.clone().map(Arc::new);
let batch_event = Evaluator::new(Arc::new(lifecycle.clone()))
.with_lock_retention(evaluator_lock_config)
.with_lock_retention(lock_config.clone().map(Arc::new))
.eval(&[object_opts])
.await
.expect("batch lifecycle evaluation should succeed")
@@ -14259,7 +14234,10 @@ mod tests {
let object = current_object_with_metadata(
ReplicationStatusType::Completed,
HashMap::from([
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
]),
);
@@ -14271,7 +14249,7 @@ mod tests {
#[tokio::test]
async fn restored_copy_expiry_is_not_blocked_by_retention() {
let lifecycle = PersistedLifecycleConfiguration {
let lifecycle = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: Vec::new(),
};
@@ -14281,7 +14259,10 @@ mod tests {
let mut object = current_object_with_metadata(
ReplicationStatusType::Completed,
HashMap::from([
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
ObjectLockRetentionMode::COMPLIANCE.to_string(),
),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
]),
);
@@ -14498,7 +14479,7 @@ mod tests {
"removed" => {
metadata_sys::delete(&bucket, BUCKET_LIFECYCLE_CONFIG).await.unwrap();
}
"disabled" => changed.rules[0].status = "Disabled".to_owned(),
"disabled" => changed.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::DISABLED),
"longer-days" => {
changed.rules[0]
.noncurrent_version_expiration
@@ -14614,7 +14595,7 @@ mod tests {
);
}
async fn install_test_expiry_policy(ecstore: &Arc<ECStore>, bucket: &str) -> Arc<PersistedLifecycleConfiguration> {
async fn install_test_expiry_policy(ecstore: &Arc<ECStore>, bucket: &str) -> Arc<BucketLifecycleConfiguration> {
let xml = br#"<LifecycleConfiguration><Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Expiration><Days>0</Days></Expiration><NoncurrentVersionExpiration><NoncurrentDays>0</NoncurrentDays></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#;
metadata_sys::update_in(&ecstore.ctx, bucket, BUCKET_LIFECYCLE_CONFIG, xml.to_vec())
.await
@@ -15307,11 +15288,7 @@ mod tests {
#[tokio::test]
#[serial]
async fn tier_free_version_recovery_production_entrypoint_enqueues_seeded_item() {
let (disk_paths, ecstore) =
temp_env::async_with_vars([(super::ENV_TIER_FREE_VERSION_RECOVERY_ENABLED, Some("false"))], async {
setup_test_env().await
})
.await;
let (disk_paths, ecstore) = setup_test_env().await;
let tier = format!("RECOVERY{}", Uuid::new_v4().simple()).to_uppercase();
crate::services::tier::test_util::register_mock_tier(&ecstore.tier_config_mgr(), &tier).await;
let runtime_state = install_unconsumed_runtime_expiry_worker(&ecstore, 1).await;
+1 -1
View File
@@ -17,7 +17,7 @@ use crate::object_api::ObjectInfo;
pub use rustfs_lifecycle::{
Event, ExpirationOptions, IlmAction, LIFECYCLE_MALFORMED_XML_ERROR_KIND, Lifecycle, LifecycleCalculate, ObjectOpts,
RuleValidate, TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, abort_incomplete_multipart_upload_due,
expected_expiry_time, expiration_action_has_valid_target, format_lifecycle_date, parse_lifecycle_date,
expected_expiry_time, expiration_action_has_valid_target,
};
pub fn object_opts_from_object_info(oi: &ObjectInfo) -> ObjectOpts {
@@ -14,8 +14,7 @@
use std::sync::Arc;
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
use time::OffsetDateTime;
use uuid::Uuid;
@@ -25,7 +24,7 @@ use crate::error::{Error, Result};
#[derive(Debug)]
pub(crate) struct LifecycleExpiryConfigs {
pub(crate) lifecycle: Option<Arc<PersistedLifecycleConfiguration>>,
pub(crate) lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
pub(crate) object_lock: Option<Arc<ObjectLockConfiguration>>,
pub(crate) bucket_incarnation_id: Uuid,
pub(crate) table_bucket_enabled: bool,
@@ -101,7 +100,7 @@ pub(crate) async fn get_expiry_configs_for_options(
})
}
pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
pub(crate) async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
metadata_sys::get_lifecycle_config(bucket).await
}
@@ -111,15 +110,15 @@ mod tests {
use crate::bucket::metadata::BucketMetadata;
use crate::bucket::metadata_sys::{self, test_support::isolated_store_over_temp_disks};
use crate::storage_api_contracts::bucket::MakeBucketOptions;
use rustfs_gateway_types::persistence::{PersistedLifecycleExpiration, PersistedLifecycleRule};
use s3s::dto::{ExpirationStatus, LifecycleExpiration, LifecycleRule};
use serial_test::serial;
fn lifecycle_config() -> PersistedLifecycleConfiguration {
PersistedLifecycleConfiguration {
fn lifecycle_config() -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
expiration: Some(PersistedLifecycleExpiration {
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
@@ -22,7 +22,7 @@ pub(crate) fn is_object_locked_by_metadata(user_defined: &HashMap<String, String
}
pub(crate) fn check_object_lock_for_deletion_with_config(
config: Option<&rustfs_gateway_types::persistence::PersistedObjectLockConfiguration>,
config: Option<&s3s::dto::ObjectLockConfiguration>,
obj_info: &ObjectInfo,
bypass_governance: bool,
) -> crate::error::Result<Option<ObjectLockBlockReason>> {
@@ -82,13 +82,12 @@ mod tests {
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::storage_api_contracts::object::ObjectToDelete;
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
use rustfs_gateway_types::persistence::{
PersistedOptionalReplicationStatus, PersistedReplicationConfiguration, PersistedReplicationDestination,
PersistedReplicationRule, PersistedReplicationStatus,
};
use rustfs_scanner_metrics::metrics::IlmAction;
use s3s::dto::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
VersioningConfiguration,
};
use uuid::Uuid;
use super::*;
@@ -161,27 +160,27 @@ mod tests {
for purge_enabled in [false, true] {
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![PersistedReplicationRule {
delete_marker_replication: Some(PersistedOptionalReplicationStatus {
rules: vec![ReplicationRule {
delete_marker_replication: Some(DeleteMarkerReplication {
status: Some(if marker_enabled {
Status::ENABLED.to_string()
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)
} else {
Status::DISABLED.to_string()
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)
}),
}),
delete_replication: Some(PersistedReplicationStatus {
delete_replication: Some(DeleteReplication {
status: if purge_enabled {
Status::ENABLED.to_string()
DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED)
} else {
Status::DISABLED.to_string()
DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED)
},
}),
destination: PersistedReplicationDestination {
destination: Destination {
bucket: "arn:rustfs:replication:target".to_string(),
..Default::default()
},
@@ -191,7 +190,7 @@ mod tests {
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: Status::ENABLED.to_string(),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
);
@@ -14,7 +14,7 @@
use std::sync::Arc;
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
use s3s::dto::BucketLifecycleConfiguration;
use tokio::sync::RwLock;
use tokio_util::sync::CancellationToken;
@@ -52,7 +52,7 @@ pub(crate) fn deployment_id() -> Option<String> {
sources::deployment_id()
}
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<PersistedLifecycleConfiguration> {
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
sources::bucket_lifecycle_config(bucket).await
}
+44 -59
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time};
use super::object_lock::ObjectLockApi;
use super::object_lock::{ObjectLockApi, ObjectLockStatusExt};
use super::versioning::VersioningApi;
use super::{quota::BucketQuota, target::BucketTargets};
use crate::bucket::replication::invalid_replication_config_status_field;
@@ -25,17 +25,12 @@ use crate::object_api::WriteCompletion;
use crate::runtime::sources as runtime_sources;
use crate::store::ECStore;
use byteorder::{BigEndian, ByteOrder, LittleEndian};
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration as CORSConfiguration, PersistedTagging as Tagging};
use rustfs_gateway_types::persistence::{
PersistedAccelerateConfiguration as AccelerateConfiguration,
PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
PersistedBucketLoggingStatus as BucketLoggingStatus, PersistedLifecycleConfiguration,
PersistedNotificationConfiguration as NotificationConfiguration, PersistedObjectLockConfiguration as ObjectLockConfiguration,
PersistedPublicAccessBlockConfiguration as PublicAccessBlockConfiguration, PersistedReplicationConfiguration,
PersistedRequestPaymentConfiguration as RequestPaymentConfiguration,
PersistedVersioningConfiguration as VersioningConfiguration, PersistedWebsiteConfiguration as WebsiteConfiguration,
};
use rustfs_policy::policy::BucketPolicy;
use s3s::dto::{
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, BucketVersioningStatus, CORSConfiguration,
NotificationConfiguration, ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration,
RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
};
use serde::Serializer;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
@@ -45,11 +40,23 @@ use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset
use tracing::error;
use uuid::Uuid;
// The persisted Object Lock configuration answers the storage-level trait
// here because this module owns the persisted `ObjectLockConfiguration`.
// The serving-layer DTO impls for the storage-level Object Lock traits live
// here because this module owns the persisted `ObjectLockConfiguration`
// during the s3s ratchet migration (rustfs/backlog#1842).
impl ObjectLockApi for ObjectLockConfiguration {
fn enabled(&self) -> bool {
self.object_lock_enabled.as_deref() == Some("Enabled")
self.object_lock_enabled
.as_ref()
.is_some_and(|v| v.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)
}
}
impl ObjectLockStatusExt for s3s::dto::ObjectLockLegalHoldStatus {
fn valid(&self) -> bool {
matches!(
self.as_str(),
s3s::dto::ObjectLockLegalHoldStatus::ON | s3s::dto::ObjectLockLegalHoldStatus::OFF
)
}
}
@@ -250,14 +257,12 @@ pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
/// Validate tag entries supplied to the S3 API.
pub fn validate_tag_set_entries(
tag_set: &[rustfs_gateway_types::cors_tagging::PersistedTag],
) -> std::result::Result<(), &'static str> {
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
validate_tag_entries(tag_set, validate_s3_tag_entry)
}
fn validate_tag_entries(
tag_set: &[rustfs_gateway_types::cors_tagging::PersistedTag],
tag_set: &[s3s::dto::Tag],
mut validate_entry: impl FnMut(&str, &str) -> std::result::Result<(), &'static str>,
) -> std::result::Result<(), &'static str> {
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
@@ -318,9 +323,7 @@ pub fn is_swift_bucket_metadata_tag_key(key: &str) -> bool {
|| key.starts_with(SWIFT_CONTAINER_META_TAG_PREFIX)
}
fn validate_persisted_bucket_tags(
tags: &[rustfs_gateway_types::cors_tagging::PersistedTag],
) -> std::result::Result<(), &'static str> {
fn validate_persisted_bucket_tags(tags: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
const SWIFT_ACL_MAX_BYTES: usize = 8192;
const SWIFT_META_NAME_MAX_BYTES: usize = 128;
const MAX_PERSISTED_TAGS: usize = MAX_BUCKET_TAGS + 2 * MAX_SWIFT_METADATA_COUNT + 4;
@@ -558,8 +561,6 @@ pub struct BucketMetadata {
pub bucket_incarnation_id: Uuid,
pub(crate) bucket_incarnation_sidecar: bool,
pub(crate) bucket_creation_committed: bool,
// A legacy default is not proof of an interrupted creation.
pub(crate) bucket_creation_commit_record_present: bool,
pub policy_config_json: Vec<u8>,
pub notification_config_xml: Vec<u8>,
pub lifecycle_config_xml: Vec<u8>,
@@ -608,13 +609,13 @@ pub struct BucketMetadata {
pub policy_config: Option<BucketPolicy>,
pub notification_config: Option<NotificationConfiguration>,
pub lifecycle_config: Option<PersistedLifecycleConfiguration>,
pub lifecycle_config: Option<BucketLifecycleConfiguration>,
pub object_lock_config: Option<ObjectLockConfiguration>,
pub versioning_config: Option<VersioningConfiguration>,
pub sse_config: Option<ServerSideEncryptionConfiguration>,
pub tagging_config: Option<Tagging>,
pub quota_config: Option<BucketQuota>,
pub replication_config: Option<PersistedReplicationConfiguration>,
pub replication_config: Option<ReplicationConfiguration>,
pub bucket_target_config: Option<BucketTargets>,
pub bucket_target_config_meta: Option<HashMap<String, String>>,
pub cors_config: Option<CORSConfiguration>,
@@ -635,7 +636,6 @@ impl Default for BucketMetadata {
bucket_incarnation_id: Uuid::nil(),
bucket_incarnation_sidecar: false,
bucket_creation_committed: false,
bucket_creation_commit_record_present: false,
policy_config_json: Default::default(),
notification_config_xml: Default::default(),
lifecycle_config_xml: Default::default(),
@@ -874,10 +874,7 @@ impl BucketMetadata {
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
"OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
"BucketCreationCommitted" => {
self.bucket_creation_committed = read_msgp_bool(rd)?;
self.bucket_creation_commit_record_present = true;
}
"BucketCreationCommitted" => self.bucket_creation_committed = read_msgp_bool(rd)?,
other => {
tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field");
skip_msgp_value(rd)?;
@@ -890,10 +887,8 @@ impl BucketMetadata {
/// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
// Preserve the absence of a legacy commit record across config rewrites.
let commit_record_present = self.bucket_creation_commit_record_present || self.bucket_creation_committed;
// Map size: MinIO fields (25) + RustFS extensions (21) + optional commit record.
let map_len: u32 = 46 + u32::from(commit_record_present);
// Map size: MinIO fields (25) + RustFS extensions (22)
let map_len: u32 = 47;
rmp::encode::write_map_len(wr, map_len)?;
// MinIO field order (same as Go struct)
@@ -974,10 +969,8 @@ impl BucketMetadata {
write_msgp_time(wr, self.durability_config_updated_at)?;
rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
if commit_record_present {
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
}
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
Ok(())
}
@@ -1154,11 +1147,9 @@ impl BucketMetadata {
None
} else {
let config = deserialize::<VersioningConfiguration>(&data)?;
if config
.status
.as_ref()
.is_some_and(|status| !matches!(status.as_str(), "Enabled" | "Suspended"))
{
if config.status.as_ref().is_some_and(|status| {
!matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED)
}) {
return Err(Error::other("bucket versioning configuration has an invalid status"));
}
Some(config)
@@ -1171,7 +1162,7 @@ impl BucketMetadata {
let config = if data.is_empty() {
None
} else {
let config = deserialize::<PersistedReplicationConfiguration>(&data)?;
let config = deserialize::<ReplicationConfiguration>(&data)?;
if let Some(field) = invalid_replication_config_status_field(&config) {
return Err(Error::other(format!("replication field {field} has an invalid status")));
}
@@ -1365,8 +1356,8 @@ impl BucketMetadata {
}
self.lifecycle_config = None;
if !self.lifecycle_config_xml.is_empty()
&& let Err(e) = deserialize::<PersistedLifecycleConfiguration>(&self.lifecycle_config_xml)
.map(|c| self.lifecycle_config = Some(c))
&& let Err(e) =
deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c))
{
tracing::warn!(
event = "bucket_metadata_parse_failed",
@@ -1452,8 +1443,8 @@ impl BucketMetadata {
}
self.replication_config = None;
if !self.replication_config_xml.is_empty()
&& let Err(e) = deserialize::<PersistedReplicationConfiguration>(&self.replication_config_xml)
.map(|c| self.replication_config = Some(c))
&& let Err(e) =
deserialize::<ReplicationConfiguration>(&self.replication_config_xml).map(|c| self.replication_config = Some(c))
{
tracing::warn!(
event = "bucket_metadata_parse_failed",
@@ -1833,13 +1824,6 @@ mod test {
let mut bm = BucketMetadata::unmarshal(&body[4..]).expect("unmarshal recovered blob");
assert_eq!(bm.name, "interop");
assert!(!bm.bucket_creation_committed, "legacy metadata must default to uncommitted");
assert!(!bm.bucket_creation_commit_record_present, "legacy metadata has no creation-commit record");
let rewritten = BucketMetadata::unmarshal(&bm.marshal_msg().expect("rewrite legacy metadata"))
.expect("read rewritten legacy metadata");
assert!(
!rewritten.bucket_creation_commit_record_present,
"rewriting legacy metadata must not fabricate a creation intent"
);
bm.parse_all_configs().expect("parse recovered configs");
assert!(bm.lifecycle_config.is_some());
}
@@ -1857,7 +1841,6 @@ mod test {
assert_eq!(bm.name, new.name);
assert!(new.bucket_creation_committed);
assert!(new.bucket_creation_commit_record_present);
assert!(!bm.bucket_incarnation_id.is_nil());
assert_eq!(bm.bucket_incarnation_id, new.bucket_incarnation_id);
}
@@ -1929,7 +1912,7 @@ mod test {
#[test]
fn persisted_bucket_tags_keep_separate_s3_and_swift_limits() {
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
use s3s::dto::Tag;
let tag = |key: String, value: &str| Tag {
key: Some(key),
@@ -2021,15 +2004,17 @@ mod test {
#[test]
fn object_locking_requires_lock_metadata_not_plain_versioning() {
use s3s::dto::ObjectLockEnabled;
let mut bm = BucketMetadata::new("test-bucket");
bm.versioning_config = Some(VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(s3s::dto::BucketVersioningStatus::from_static("Enabled")),
..Default::default()
});
assert!(!bm.object_locking());
bm.object_lock_config = Some(ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
..Default::default()
});
assert!(bm.object_locking());
+48 -326
View File
@@ -31,18 +31,14 @@ use crate::storage_api_contracts::heal::HealOperations as _;
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
use crate::store::{ECStore, await_bucket_namespace_operation};
use futures::future::join_all;
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration as CORSConfiguration, PersistedTagging as Tagging};
use rustfs_gateway_types::persistence::{
PersistedAccelerateConfiguration as AccelerateConfiguration,
PersistedBucketEncryptionConfiguration as ServerSideEncryptionConfiguration,
PersistedBucketLoggingStatus as BucketLoggingStatus, PersistedLifecycleConfiguration,
PersistedNotificationConfiguration as NotificationConfiguration, PersistedObjectLockConfiguration as ObjectLockConfiguration,
PersistedPublicAccessBlockConfiguration as PublicAccessBlockConfiguration, PersistedReplicationConfiguration,
PersistedRequestPaymentConfiguration as RequestPaymentConfiguration,
PersistedVersioningConfiguration as VersioningConfiguration, PersistedWebsiteConfiguration as WebsiteConfiguration,
};
use rustfs_heal_contracts::heal_channel::HealOpts;
use rustfs_policy::policy::BucketPolicy;
use s3s::dto::ReplicationConfiguration;
use s3s::dto::{
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, PublicAccessBlockConfiguration,
RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
};
use std::collections::HashSet;
use std::time::Duration;
use std::{
@@ -173,7 +169,7 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe
if bm.lock_enabled {
return Ok(ObjectLockConfigState::Configured {
config: ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: None,
},
updated_at: bm.object_lock_config_updated_at,
@@ -217,10 +213,17 @@ pub(crate) fn configured_object_lock_state_for_tests(
) -> ObjectLockConfigState {
ObjectLockConfigState::Configured {
config: ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
rule: Some(rustfs_gateway_types::persistence::PersistedObjectLockRule {
default_retention: Some(rustfs_gateway_types::persistence::PersistedDefaultRetention {
mode: Some(mode.as_str().to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: Some(s3s::dto::ObjectLockRule {
default_retention: Some(s3s::dto::DefaultRetention {
mode: Some(s3s::dto::ObjectLockRetentionMode::from_static(match mode {
crate::bucket::object_lock::types::RetentionMode::Governance => {
s3s::dto::ObjectLockRetentionMode::GOVERNANCE
}
crate::bucket::object_lock::types::RetentionMode::Compliance => {
s3s::dto::ObjectLockRetentionMode::COMPLIANCE
}
})),
days: Some(days),
years: None,
}),
@@ -231,7 +234,7 @@ pub(crate) fn configured_object_lock_state_for_tests(
}
fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -> Result<()> {
if config.object_lock_enabled.as_deref() != Some("Enabled") {
if config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str) != Some(ObjectLockEnabled::ENABLED) {
return Err(Error::other("persisted bucket Object Lock enabled state is invalid"));
}
@@ -244,7 +247,7 @@ fn validate_authoritative_object_lock_config(config: &ObjectLockConfiguration) -
if !retention
.mode
.as_ref()
.is_some_and(|mode| matches!(mode.as_str(), "COMPLIANCE" | "GOVERNANCE"))
.is_some_and(|mode| matches!(mode.as_str(), ObjectLockRetentionMode::COMPLIANCE | ObjectLockRetentionMode::GOVERNANCE))
{
return Err(Error::other("persisted bucket Object Lock retention mode is invalid"));
}
@@ -1638,7 +1641,7 @@ pub async fn get_public_access_block_config(bucket: &str) -> Result<(PublicAcces
bucket_meta_sys.get_public_access_block_config(bucket).await
}
pub async fn get_lifecycle_config(bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1914,7 +1917,7 @@ pub(crate) async fn get_cached_quota_config_and_incarnation_in(
bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await
}
pub async fn get_replication_config(bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1924,7 +1927,7 @@ pub async fn get_replication_config(bucket: &str) -> Result<(PersistedReplicatio
pub(crate) async fn get_replication_config_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -2062,27 +2065,9 @@ pub(crate) fn current_local_meta_mutation_gen() -> u64 {
type LocalAuthorityCache = Arc<StdMutex<HashMap<String, (u64, Arc<BucketMetadata>)>>>;
#[derive(Debug, Clone)]
pub(crate) struct PendingBucketCreationProof {
pub(crate) generation: Uuid,
pub(crate) intent: Vec<u8>,
pub(crate) versioning_requested: bool,
pub(crate) witnesses: Vec<crate::disk::os::NativeBucketCreationWitness>,
}
impl PendingBucketCreationProof {
pub(crate) fn matches_metadata(&self, metadata: &BucketMetadata) -> Result<bool> {
Ok(!metadata.bucket_creation_committed
&& metadata.bucket_incarnation_sidecar
&& metadata.bucket_incarnation_id == self.generation
&& metadata.marshal_msg()? == self.intent)
}
}
#[derive(Debug, Clone)]
pub struct BucketMetadataSys {
metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
pending_bucket_creations: Arc<RwLock<HashMap<String, PendingBucketCreationProof>>>,
local_authority_cache: LocalAuthorityCache,
/// Serializes metadata-map commits and their derived cache updates for one
/// bucket. Namespace locks, when present, are acquired before this lock.
@@ -2127,7 +2112,6 @@ impl BucketMetadataSys {
pub fn new(api: Arc<ECStore>) -> Self {
Self {
metadata_map: Arc::new(RwLock::new(HashMap::new())),
pending_bucket_creations: Arc::new(RwLock::new(HashMap::new())),
local_authority_cache: Arc::new(StdMutex::new(HashMap::new())),
metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
locks: StdMutex::new(HashMap::new()),
@@ -2159,81 +2143,6 @@ impl BucketMetadataSys {
.expect("bucket metadata object store should still be live")
}
pub(crate) async fn pending_bucket_creation(&self, bucket: &str) -> Option<PendingBucketCreationProof> {
self.pending_bucket_creations.read().await.get(bucket).cloned()
}
pub(crate) async fn grant_pending_bucket_creation(&self, bucket: &str, proof: PendingBucketCreationProof) {
self.pending_bucket_creations.write().await.insert(bucket.to_owned(), proof);
}
pub(crate) async fn revoke_pending_bucket_creation(&self, bucket: &str) {
self.pending_bucket_creations.write().await.remove(bucket);
}
pub(crate) async fn consume_pending_bucket_creation(
&self,
bucket: &str,
metadata: &BucketMetadata,
witnesses: &[crate::disk::os::NativeBucketCreationWitness],
) -> Result<bool> {
let mut pending = self.pending_bucket_creations.write().await;
let Some(proof) = pending.get(bucket) else {
return Ok(false);
};
if !proof.matches_metadata(metadata)?
|| proof.witnesses.len() != witnesses.len()
|| !proof
.witnesses
.iter()
.zip(witnesses)
.all(|(old, current)| old.matches(current))
|| !crate::disk::os::consume_native_creation_witnesses(witnesses)
{
return Ok(false);
}
pending.remove(bucket);
Ok(true)
}
#[cfg(test)]
pub(crate) async fn refresh_bucket_for_test(&self, bucket: &str) -> Result<()> {
let expected = self.metadata_map.read().await.get(bucket).cloned();
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
self.load_bucket_under_namespace(bucket, MetadataLoadMode::Refresh, expected.as_ref(), &guard)
.await
}
#[cfg(test)]
pub(crate) async fn publish_persisted_metadata_for_test(&self, bucket: &str) -> Result<()> {
let expected = self.metadata_map.read().await.get(bucket).cloned();
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
let (metadata, persisted) = await_bucket_namespace_operation(
Some(&guard),
bucket,
"test cached metadata load",
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
)
.await?;
self.publish_if_unchanged(bucket, expected.as_ref(), metadata, persisted, &guard)
.await
}
#[cfg(test)]
pub(crate) async fn migrate_bucket_creation_commit_for_test(&self, bucket: &str) -> Result<()> {
self.migrate_bucket_creation_commit(bucket).await.map(|_| ())
}
#[cfg(test)]
pub(crate) async fn legacy_migration_reports_missing_bucket_for_test(&self, bucket: &str) -> Result<bool> {
Ok(matches!(
self.migrate_legacy_metadata(bucket).await?,
BucketMetadataAuthority::MissingBucket
))
}
fn object_store_if_live(&self) -> Option<Arc<ECStore>> {
self.api.upgrade()
}
@@ -2433,7 +2342,6 @@ impl BucketMetadataSys {
expected: Option<&Arc<BucketMetadata>>,
namespace_guard: &rustfs_lock::NamespaceLockGuard,
) -> Result<()> {
self.revoke_pending_bucket_creation(bucket).await;
if !await_bucket_namespace_operation(
Some(namespace_guard),
bucket,
@@ -2517,7 +2425,6 @@ impl BucketMetadataSys {
persisted: bool,
namespace_guard: &rustfs_lock::NamespaceLockGuard,
) -> Result<()> {
self.revoke_pending_bucket_creation(bucket).await;
if !persisted {
let _publish_guard = self
.lock_metadata_publish(bucket, namespace_guard, "refreshed bucket metadata absence publish")
@@ -2579,7 +2486,6 @@ impl BucketMetadataSys {
}
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
self.revoke_pending_bucket_creation(&bucket).await;
bump_local_meta_mutation_gen();
if !is_meta_bucketname(&bucket) {
let publish_lock = self.metadata_publish_lock(&bucket);
@@ -2603,7 +2509,6 @@ impl BucketMetadataSys {
if is_meta_bucketname(bucket) {
return false;
}
self.revoke_pending_bucket_creation(bucket).await;
let publish_lock = self.metadata_publish_lock(bucket);
let _publish_guard = publish_lock.lock().await;
let mut map = self.metadata_map.write().await;
@@ -2621,7 +2526,6 @@ impl BucketMetadataSys {
}
async fn _reset(&mut self) {
self.pending_bucket_creations.write().await.clear();
bump_local_meta_mutation_gen();
let mut map = self.metadata_map.write().await;
map.clear();
@@ -2761,7 +2665,6 @@ impl BucketMetadataSys {
/// server's metadata never leaks into the ambient (first) instance.
pub(crate) async fn persist_and_set(&self, bm: BucketMetadata) -> Result<()> {
let mut bm = bm;
self.revoke_pending_bucket_creation(&bm.name).await;
bm.save_with_store(self.object_store()).await?;
self.set(bm.name.clone(), Arc::new(bm)).await;
@@ -2770,7 +2673,6 @@ impl BucketMetadataSys {
}
async fn persist_new_and_set(&self, mut bm: BucketMetadata) -> Result<()> {
self.revoke_pending_bucket_creation(&bm.name).await;
bm.bucket_creation_committed = true;
bm.save_with_store_committed(self.object_store()).await?;
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
@@ -2780,9 +2682,7 @@ impl BucketMetadataSys {
}
async fn persist_new_bucket_metadata_intent(&self, mut bm: BucketMetadata) -> Result<()> {
self.revoke_pending_bucket_creation(&bm.name).await;
bm.bucket_creation_committed = false;
bm.bucket_creation_commit_record_present = true;
bm.save_with_store(self.object_store()).await?;
save_bucket_incarnation(self.object_store(), &bm.name, bm.bucket_incarnation_id).await?;
bm.bucket_incarnation_sidecar = true;
@@ -2792,7 +2692,6 @@ impl BucketMetadataSys {
}
async fn commit_bucket_metadata(&self, mut bm: BucketMetadata) -> Result<()> {
self.revoke_pending_bucket_creation(&bm.name).await;
bm.parse_all_configs()?;
bm.bucket_incarnation_sidecar = true;
// The caller holds the full bucket creation fence (lifecycle, metadata
@@ -2841,7 +2740,6 @@ impl BucketMetadataSys {
bucket: &str,
namespace_guard: &rustfs_lock::NamespaceLockGuard,
) -> Result<()> {
self.revoke_pending_bucket_creation(bucket).await;
let expected = self.metadata_map.read().await.get(bucket).cloned();
if !self
.bucket_exists(bucket, namespace_guard, "peer bucket metadata existence check")
@@ -2898,7 +2796,6 @@ impl BucketMetadataSys {
#[cfg(test)]
self.lazy_disk_loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
self.revoke_pending_bucket_creation(bucket).await;
let lock = self.object_store().new_ns_lock(bucket, bucket).await?;
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
#[cfg(test)]
@@ -3224,7 +3121,6 @@ impl BucketMetadataSys {
/// revalidated at write quorum, so a stale snapshot can neither revert an
/// acknowledged configuration update nor outlive a delete/recreate.
async fn migrate_bucket_creation_commit(&self, bucket: &str) -> Result<BucketMetadataAuthority> {
self.revoke_pending_bucket_creation(bucket).await;
let transaction_guard = acquire_metadata_transaction_guard(
self.object_store(),
bucket,
@@ -3302,7 +3198,6 @@ impl BucketMetadataSys {
&self,
bucket: &str,
) -> Result<BucketMetadataAuthority> {
self.revoke_pending_bucket_creation(bucket).await;
#[cfg(test)]
if self.object_lock_disk_read_errors.write().await.remove(bucket) {
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
@@ -3525,7 +3420,7 @@ impl BucketMetadataSys {
}
}
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(PersistedLifecycleConfiguration, OffsetDateTime)> {
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
match ConfigState::of(&bm.lifecycle_config_xml, &bm.lifecycle_config).require(bucket, BUCKET_LIFECYCLE_CONFIG)? {
@@ -3648,7 +3543,7 @@ impl BucketMetadataSys {
}))
}
pub async fn get_replication_config(&self, bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if !bm.replication_config_xml.is_empty() && bm.replication_config.is_none() {
@@ -3750,7 +3645,6 @@ mod tests {
};
use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
use crate::config::com::read_config;
use crate::disk::{DiskAPI as _, RUSTFS_META_TMP_BUCKET};
use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
use byteorder::{ByteOrder as _, LittleEndian};
use serial_test::serial;
@@ -3805,180 +3699,6 @@ mod tests {
),
];
#[tokio::test]
#[serial(storage_class_env)]
async fn persistence_entry_failures_revoke_pending_creation_before_cache_publish() {
let (dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
let sys = sys.read().await.clone();
let options = MakeBucketOptions {
lock_enabled: true,
..Default::default()
};
for entry in ["save", "persist", "persist-new", "commit"] {
let bucket = format!("revoke-{entry}-{}", Uuid::new_v4());
store.fail_next_bucket_creation_after_intent_for_test(&bucket);
store
.make_bucket(&bucket, &options)
.await
.expect_err("stop the real producer before physical publication");
sys.pending_bucket_creation(&bucket)
.await
.expect("the producer must leave a real live creation permission");
let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read the original producer intent");
assert!(persisted, "{entry}: the complete original intent must be persisted");
let original_intent = intent.marshal_msg().expect("encode the original intent");
// Keep the creation fence order used by the real producer.
let lifecycle_guard = store
.acquire_bucket_lifecycle_write_lock(&bucket)
.await
.expect("hold the bucket lifecycle fence");
let transaction_guard = acquire_bucket_metadata_transaction_lock_in(&store.ctx, &bucket)
.await
.expect("hold the metadata transaction fence");
let namespace_lock = store
.new_ns_lock(&bucket, &bucket)
.await
.expect("create the exact bucket namespace lock");
let namespace_guard = namespace_lock
.get_write_lock(crate::set_disk::get_lock_acquire_timeout())
.await
.expect("hold the exact bucket namespace fence");
// A file at each native staging root causes actual IO to fail before
// metadata can reach the cache-publishing set call. Preserve the
// original staging directories so no fixture data is discarded.
for dir in &dirs {
let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
std::fs::rename(&staging, &saved).expect("preserve the native staging directory");
std::fs::write(&staging, b"block metadata staging with a non-directory")
.expect("install an actual non-directory IO obstruction");
}
let result = match entry {
"save" => sys.save(intent).await,
"persist" => sys.persist_and_set(intent).await,
"persist-new" => sys.persist_new_and_set(intent).await,
"commit" => sys.commit_bucket_metadata(intent).await,
_ => unreachable!(),
};
for dir in &dirs {
let staging = dir.path().join(RUSTFS_META_TMP_BUCKET);
let saved = dir.path().join(".rustfs.sys/revocation-saved-tmp");
std::fs::remove_file(&staging).expect("remove only the fixture's staging obstruction");
std::fs::rename(&saved, &staging).expect("restore the complete original staging directory");
}
drop((namespace_guard, transaction_guard, lifecycle_guard));
result.expect_err("native persistence must fail before its downstream cache set");
assert!(
sys.pending_bucket_creation(&bucket).await.is_none(),
"{entry}: the entry revoker must retire live permission even when persistence fails"
);
let (after_failure, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read the intent after the failed persistence entry");
assert!(present, "{entry}: a failed persistence entry must retain the original metadata");
assert_eq!(
after_failure
.marshal_msg()
.expect("encode the original intent after persistence failure"),
original_intent,
"{entry}: native IO failure must not change the original generation or complete intent"
);
let error = store
.make_bucket(&bucket, &options)
.await
.expect_err("a failed ordinary persistence entry cannot preserve creation permission");
assert!(error.to_string().contains("administrator recovery"), "{entry}: {error}");
let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read the original intent after the rejected retry");
assert_eq!(
after_retry.marshal_msg().expect("encode the intent after the rejected retry"),
original_intent,
"{entry}: the rejected retry must preserve the complete original intent"
);
for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
assert!(
matches!(
disk.as_ref().expect("native disk").stat_volume(&bucket).await,
Err(crate::disk::error::DiskError::VolumeNotFound)
),
"{entry}: failed persistence and the rejected retry must not publish a physical volume"
);
}
}
}
#[tokio::test]
#[serial(storage_class_env)]
async fn metadata_reset_revokes_real_pending_creation_without_changing_persisted_intent() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = format!("revoke-reset-{}", Uuid::new_v4());
let options = MakeBucketOptions {
lock_enabled: true,
..Default::default()
};
store.fail_next_bucket_creation_after_intent_for_test(&bucket);
store
.make_bucket(&bucket, &options)
.await
.expect_err("stop the real producer before physical publication");
let sys = bucket_metadata_sys_of(&store.ctx).expect("instance metadata system");
let mut sys = sys.read().await.clone();
sys.pending_bucket_creation(&bucket)
.await
.expect("the real producer must leave live creation permission before reset");
let (intent, persisted) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read the original intent before cache reset");
assert!(persisted, "cache reset starts from a complete real persisted intent");
let original_intent = intent.marshal_msg().expect("encode the original reset intent");
sys._reset().await;
assert!(
sys.pending_bucket_creation(&bucket).await.is_none(),
"reset must revoke the producer-issued permission shared with the instance metadata system"
);
let (after_reset, present) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read the original intent after cache reset");
assert!(present, "cache reset must retain the persisted intent");
assert_eq!(
after_reset.marshal_msg().expect("encode the intent after cache reset"),
original_intent,
"cache reset must leave the original generation and complete persisted intent unchanged"
);
let error = store
.make_bucket(&bucket, &options)
.await
.expect_err("reset cannot regrant the producer's old creation permission");
assert!(error.to_string().contains("administrator recovery"), "{error}");
let (after_retry, _) = get_config_from_disk_with_presence_in(&store.ctx, &bucket)
.await
.expect("read the intent after the reset-induced retry rejection");
assert_eq!(
after_retry
.marshal_msg()
.expect("encode the intent after the rejected reset retry"),
original_intent,
"the reset-induced rejection must preserve the original complete intent"
);
for disk in store.pools[0].disk_set[0].disks.read().await.iter() {
assert!(
matches!(
disk.as_ref().expect("native disk").stat_volume(&bucket).await,
Err(crate::disk::error::DiskError::VolumeNotFound)
),
"cache reset and the rejected retry must not publish a physical volume"
);
}
}
#[tokio::test]
async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
let (dirs, store) = isolated_store_over_temp_disks().await;
@@ -4167,7 +3887,7 @@ mod tests {
let mut versioned = BucketMetadata::new("authoritative-versioned");
versioned.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
versioned.versioning_config = Some(VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
..Default::default()
});
sys.set("authoritative-versioned".to_string(), Arc::new(versioned)).await;
@@ -4290,24 +4010,25 @@ mod tests {
else {
panic!("legacy lock-enabled metadata must remain configured");
};
assert_eq!(config.object_lock_enabled.as_deref(), Some("Enabled"));
assert_eq!(
config.object_lock_enabled.as_ref().map(|value| value.as_str()),
Some(ObjectLockEnabled::ENABLED)
);
assert!(config.rule.is_none());
}
#[test]
fn authoritative_object_lock_state_rejects_invalid_retention_periods() {
use rustfs_gateway_types::persistence::{
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockRule as ObjectLockRule,
};
use s3s::dto::{DefaultRetention, ObjectLockRule};
for (days, years) in [(Some(0), None), (Some(-1), None), (Some(1), Some(1)), (None, None)] {
let mut metadata = BucketMetadata::new("invalid-object-lock");
metadata.object_lock_config = Some(ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: Some(ObjectLockRule {
default_retention: Some(DefaultRetention {
days,
mode: Some("COMPLIANCE".to_owned()),
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
years,
}),
}),
@@ -4322,11 +4043,9 @@ mod tests {
#[test]
fn authoritative_object_lock_state_rejects_invalid_enabled_mode_and_rule() {
use rustfs_gateway_types::persistence::{
PersistedDefaultRetention as DefaultRetention, PersistedObjectLockRule as ObjectLockRule,
};
use s3s::dto::{DefaultRetention, ObjectLockRule};
for enabled in [None, Some("Disabled".to_string())] {
for enabled in [None, Some(ObjectLockEnabled::from("Disabled".to_string()))] {
let mut metadata = BucketMetadata::new("invalid-object-lock-enabled");
metadata.lock_enabled = true;
metadata.object_lock_config = Some(ObjectLockConfiguration {
@@ -4338,11 +4057,11 @@ mod tests {
let mut invalid_mode = BucketMetadata::new("invalid-object-lock-mode");
invalid_mode.object_lock_config = Some(ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: Some(ObjectLockRule {
default_retention: Some(DefaultRetention {
days: Some(1),
mode: Some("invalid".to_string()),
mode: Some(ObjectLockRetentionMode::from("invalid".to_string())),
years: None,
}),
}),
@@ -4351,7 +4070,7 @@ mod tests {
let mut missing_retention = BucketMetadata::new("missing-object-lock-retention");
missing_retention.object_lock_config = Some(ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: Some(ObjectLockRule { default_retention: None }),
});
assert!(object_lock_config_state_from_authoritative_metadata(&missing_retention).is_err());
@@ -4361,7 +4080,7 @@ mod tests {
fn object_lock_ever_enabled_marker_survives_config_removal() {
let mut metadata = BucketMetadata::new("lock-config-removed");
let config = ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: None,
};
metadata
@@ -4379,7 +4098,10 @@ mod tests {
else {
panic!("an ever-enabled bucket must remain Object Lock configured");
};
assert_eq!(config.object_lock_enabled.as_deref(), Some("Enabled"));
assert_eq!(
config.object_lock_enabled.as_ref().map(ObjectLockEnabled::as_str),
Some(ObjectLockEnabled::ENABLED)
);
}
#[tokio::test]
@@ -4395,7 +4117,7 @@ mod tests {
let node = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore.clone())));
let config = ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: None,
};
update_with_sys(
@@ -5509,7 +5231,7 @@ mod tests {
#[tokio::test]
async fn update_config_with_persists_tagging_rewrite_across_disk_reload() {
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
use s3s::dto::Tag;
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
@@ -5673,7 +5395,7 @@ mod tests {
#[tokio::test]
async fn concurrent_update_config_with_calls_do_not_lose_writes() {
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
use s3s::dto::Tag;
let (dirs, ecstore) = isolated_store_over_temp_disks().await;
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(ecstore)));
@@ -5773,7 +5495,7 @@ mod tests {
let mut persisted = BucketMetadata::new("reload-bucket");
persisted.policy_config_json = b"persisted-marker".to_vec();
let lock_config = ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: None,
};
persisted
@@ -6107,7 +5829,7 @@ mod tests {
#[serial]
async fn bucket_delete_waits_for_config_mutation_fence() {
use crate::bucket::metadata::BUCKET_TAGGING_CONFIG;
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
use s3s::dto::Tag;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+38 -3
View File
@@ -17,7 +17,7 @@
use crate::bucket::metadata::BUCKET_METADATA_FILE;
use crate::bucket::replication::ReplicationMigrationBridge;
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found};
use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found, to_object_err};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions},
@@ -334,6 +334,14 @@ async fn migration_target_exists<S: EcstoreObjectOperations>(store: &S, path: &s
}
}
/// Names the target record in a failed migration write, as the existence
/// probe's error already does. The commit path returns bare disk errors, and on
/// Windows a blocked record is first seen here: reading through a regular-file
/// ancestor reports the record missing instead of refusing access to it.
fn migration_write_error(err: Error, path: &str) -> Error {
to_object_err(err, vec![RUSTFS_META_BUCKET, path])
}
async fn migrate_one_if_missing<S>(
store: Arc<S>,
opts: &ObjectOptions,
@@ -369,7 +377,10 @@ where
.unwrap_or(data);
let mut put_data = PutObjReader::from_vec(data);
store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await?;
store
.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts)
.await
.map_err(|err| migration_write_error(err, path))?;
info!("Migrated {label}");
Ok(())
}
@@ -481,7 +492,10 @@ where
Err(_) => return Err(MigrationMetadataError::Incompatible(path.to_owned()).into()),
};
let mut put_data = PutObjReader::from_vec(data);
store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await?;
store
.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts)
.await
.map_err(|err| migration_write_error(err, path))?;
info!("Migrated IAM config: {path}");
total_migrated += 1;
}
@@ -531,6 +545,27 @@ mod tests {
));
}
#[test]
fn migration_write_errors_name_the_target_record_without_fragmenting_groups() {
use super::{Error, migration_write_error};
let first = migration_write_error(Error::FileAccessDenied, "buckets/first/.metadata.bin");
let second = migration_write_error(Error::FileAccessDenied, "config/iam/format.json");
assert!(matches!(first, Error::PrefixAccessDenied(..)), "unexpected error: {first:?}");
assert!(
first.to_string().contains(".rustfs.sys/buckets/first/.metadata.bin"),
"the startup error must name the blocked record: {first}"
);
assert_eq!(first, second, "record paths must not fragment error grouping");
let unrelated = migration_write_error(Error::other("disk offline"), "buckets/first/.metadata.bin");
assert_eq!(
unrelated,
Error::other("disk offline"),
"errors without a record form pass through unchanged"
);
}
#[test]
fn migration_errors_group_by_cause_and_retain_typed_record_context() {
use super::{Error, MigrationMetadataError};
+9 -2
View File
@@ -17,8 +17,15 @@ pub mod objectlock_sys;
pub mod types;
/// Whether a bucket Object Lock configuration has locking enabled. The
/// persisted `ObjectLockConfiguration` implements this in the bucket-metadata
/// module, which owns the persisted configuration type.
/// serving-layer `ObjectLockConfiguration` DTO implements this in
/// the bucket-metadata module, which owns the persisted configuration type
/// during the s3s ratchet migration (rustfs/backlog#1842).
pub trait ObjectLockApi {
fn enabled(&self) -> bool;
}
/// Whether a legal-hold status value is one of the two valid wire values.
/// Implemented for the serving-layer DTO in the bucket-metadata module.
pub trait ObjectLockStatusExt {
fn valid(&self) -> bool;
}
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub use rustfs_replication::config_cache_serde;
pub use rustfs_replication::{
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
use s3s::dto::ReplicationConfiguration;
use super::replication_config_boundary::ReplicationConfigurationExt as _;
use super::replication_filemeta_boundary::{
@@ -42,7 +42,7 @@ impl ReplicationLifecycleBridge {
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn new_config(config: PersistedReplicationConfiguration) -> ReplicationLifecycleConfig {
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
ReplicationConfig::new(Some(config), None)
}
@@ -118,17 +118,16 @@ impl ReplicationLifecycleBridge {
#[cfg(test)]
mod tests {
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::{PersistedReplicationDestination, PersistedReplicationRule};
use s3s::dto::{Destination, ReplicationRule, ReplicationRuleStatus};
use super::super::replication_filemeta_boundary::{ReplicateTargetDecision, VersionPurgeStatusType};
use super::*;
fn replication_rule() -> PersistedReplicationRule {
PersistedReplicationRule {
fn replication_rule() -> ReplicationRule {
ReplicationRule {
delete_marker_replication: None,
delete_replication: None,
destination: PersistedReplicationDestination {
destination: Destination {
bucket: "arn:aws:s3:::target-bucket".to_string(),
..Default::default()
},
@@ -138,13 +137,13 @@ mod tests {
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: Status::ENABLED.to_string(),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}
}
#[test]
fn has_pending_version_purge_preserves_replication_active_rule_behavior() {
let config = ReplicationLifecycleBridge::new_config(PersistedReplicationConfiguration {
let config = ReplicationLifecycleBridge::new_config(ReplicationConfiguration {
role: String::new(),
rules: vec![replication_rule()],
});
@@ -17,8 +17,8 @@ use std::sync::Arc;
use crate::bucket::{metadata::BucketMetadata, metadata_sys};
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::runtime::instance::InstanceContext;
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
use rustfs_utils::path::path_join_buf;
use s3s::dto::ReplicationConfiguration;
use time::OffsetDateTime;
use super::replication_error_boundary::{Error, Result};
@@ -37,11 +37,11 @@ impl ReplicationMetadataStore {
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
pub(crate) async fn replication_config(bucket: &str) -> Result<(PersistedReplicationConfiguration, OffsetDateTime)> {
pub(crate) async fn replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
metadata_sys::get_replication_config(bucket).await
}
pub(crate) async fn optional_replication_config(bucket: &str) -> Result<Option<PersistedReplicationConfiguration>> {
pub(crate) async fn optional_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
let config = match Self::replication_config(bucket).await {
Ok((config, _)) => Some(config),
Err(err) => {
@@ -16,9 +16,7 @@ use std::{collections::HashMap, fmt, sync::Arc};
use super::replication_filemeta_boundary::metadata_keys;
use crate::bucket::metadata::BucketMetadata;
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use tracing::error;
@@ -41,7 +39,7 @@ use super::replication_target_boundary::{BucketTargets, ReplicationTargetStore};
use super::replication_versioning_boundary::ReplicationVersioningStore;
use super::runtime_boundary as runtime_sources;
pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<PersistedReplicationConfiguration>> {
pub(crate) async fn get_replication_config(bucket: &str) -> Result<Option<ReplicationConfiguration>> {
let config = ReplicationMetadataStore::optional_replication_config(bucket).await?;
validate_delete_replication_config(&VersioningConfiguration::default(), config.as_ref())?;
Ok(config)
@@ -66,7 +64,7 @@ impl DeleteReplicationConfigSnapshot {
#[cfg(test)]
pub(crate) fn from_configs_for_test(
versioning: VersioningConfiguration,
replication: Option<PersistedReplicationConfiguration>,
replication: Option<ReplicationConfiguration>,
) -> Self {
let metadata = replication.map(|config| {
let mut metadata = BucketMetadata::new("test-bucket");
@@ -80,7 +78,7 @@ impl DeleteReplicationConfigSnapshot {
&self.versioning
}
pub fn replication_config(&self) -> Option<&PersistedReplicationConfiguration> {
pub fn replication_config(&self) -> Option<&ReplicationConfiguration> {
self.metadata
.as_ref()
.and_then(|metadata| metadata.replication_config.as_ref())
@@ -103,7 +101,7 @@ impl DeleteReplicationConfigSnapshot {
pub(crate) fn active_delete_marker_rules_require_tags(&self, object: &str) -> bool {
self.replication_config().is_some_and(|config| {
config.rules.iter().any(|rule| {
if rule.status == Status::DISABLED.as_str() {
if rule.status == ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED) {
return false;
}
if !object.starts_with(rule.prefix()) {
@@ -124,12 +122,12 @@ impl DeleteReplicationConfigSnapshot {
fn validate_delete_replication_config(
versioning: &VersioningConfiguration,
config: Option<&PersistedReplicationConfiguration>,
config: Option<&ReplicationConfiguration>,
) -> Result<()> {
if versioning
.status
.as_ref()
.is_some_and(|status| !matches!(status.as_str(), "Enabled" | "Suspended"))
.is_some_and(|status| !matches!(status.as_str(), BucketVersioningStatus::ENABLED | BucketVersioningStatus::SUSPENDED))
{
return Err(super::replication_error_boundary::Error::other(
"bucket versioning configuration has an invalid status",
@@ -146,7 +144,7 @@ fn validate_delete_replication_config(
let role = config.role.trim();
let mut role_destination = None;
for rule in &config.rules {
if rule.status.as_str() == Status::ENABLED.as_str() {
if rule.status.as_str() == ReplicationRuleStatus::ENABLED {
let destination = rule.destination.bucket.trim();
if role.is_empty() && destination.is_empty() {
return Err(super::replication_error_boundary::Error::other(
@@ -171,7 +169,7 @@ fn validate_delete_replication_config(
Ok(())
}
fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&PersistedReplicationConfiguration>> {
fn replication_config_from_metadata(metadata: &BucketMetadata) -> Result<Option<&ReplicationConfiguration>> {
if !metadata.replication_config_xml.is_empty() && metadata.replication_config.is_none() {
return Err(super::replication_error_boundary::Error::other(
"persisted bucket replication configuration is invalid",
@@ -252,15 +250,12 @@ pub(crate) async fn load_delete_replication_config_in(
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ReplicationConfig {
/// Carried in the scanner's persisted data usage cache, so its serde form
/// is the one the s3s DTO wrote there (see `config_cache_serde` in the replication crate).
#[serde(with = "super::replication_config_boundary::config_cache_serde")]
pub config: Option<PersistedReplicationConfiguration>,
pub config: Option<ReplicationConfiguration>,
pub remotes: Option<BucketTargets>,
}
impl ReplicationConfig {
pub fn new(config: Option<PersistedReplicationConfiguration>, remotes: Option<BucketTargets>) -> Self {
pub fn new(config: Option<ReplicationConfiguration>, remotes: Option<BucketTargets>) -> Self {
Self { config, remotes }
}
@@ -473,7 +468,7 @@ fn check_replicate_delete_with_config(
oi: &ObjectInfo,
del_opts: &ObjectOptions,
source_error: bool,
config: Option<&PersistedReplicationConfiguration>,
config: Option<&ReplicationConfiguration>,
existing_delete_marker: bool,
trust_persisted_replica_status: bool,
) -> ReplicateDecision {
@@ -592,7 +587,7 @@ pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplic
}
fn metadata_target_should_replicate(
cfg: &PersistedReplicationConfiguration,
cfg: &ReplicationConfiguration,
opts: &ObjectOpts,
mopts: &MustReplicateOptions,
arn: &str,
@@ -602,21 +597,20 @@ fn metadata_target_should_replicate(
#[cfg(test)]
mod tests {
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::{
PersistedOptionalReplicationStatus, PersistedReplicationDestination, PersistedReplicationFilter,
PersistedReplicationRule, PersistedReplicationStatus, PersistedReplicationTag, PersistedSourceSelectionCriteria,
use s3s::dto::{
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
ReplicaModifications, ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, SourceSelectionCriteria, Tag,
};
use super::super::replication_filemeta_boundary::VersionPurgeStatusType;
use super::super::replication_target_boundary::BucketTarget;
use super::*;
fn replication_rule() -> PersistedReplicationRule {
PersistedReplicationRule {
fn replication_rule() -> ReplicationRule {
ReplicationRule {
delete_marker_replication: None,
delete_replication: None,
destination: PersistedReplicationDestination {
destination: Destination {
bucket: "arn:aws:s3:::target-bucket".to_string(),
..Default::default()
},
@@ -626,7 +620,7 @@ mod tests {
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: Status::ENABLED.to_string(),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}
}
@@ -635,11 +629,11 @@ mod tests {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut rule = replication_rule();
rule.destination.bucket = arn.to_string();
rule.filter = Some(PersistedReplicationFilter {
rule.filter = Some(ReplicationRuleFilter {
prefix: Some("admitted/".to_string()),
..Default::default()
});
let cfg = PersistedReplicationConfiguration {
let cfg = ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
};
@@ -677,7 +671,7 @@ mod tests {
assert!(!empty.replicate(&ObjectOpts::default()));
let config = ReplicationConfig::new(
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![replication_rule()],
}),
@@ -731,7 +725,7 @@ mod tests {
fn delete_snapshot_rejects_enabled_rules_without_a_destination() {
let mut rule = replication_rule();
rule.destination.bucket.clear();
let config = PersistedReplicationConfiguration {
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
};
@@ -747,7 +741,7 @@ mod tests {
let first = replication_rule();
let mut second = replication_rule();
second.destination.bucket = "arn:aws:s3:::other-target".to_string();
let config = PersistedReplicationConfiguration {
let config = ReplicationConfiguration {
role: "arn:aws:s3:::role-target".to_string(),
rules: vec![first, second],
};
@@ -758,47 +752,47 @@ mod tests {
#[test]
fn delete_snapshot_rejects_unknown_status_values() {
let invalid_versioning = VersioningConfiguration {
status: Some("Enabld".to_string()),
status: Some("Enabld".to_string().into()),
..Default::default()
};
assert!(validate_delete_replication_config(&invalid_versioning, None).is_err());
let mut invalid_rule = replication_rule();
invalid_rule.status = "Enabld".to_string();
let config = PersistedReplicationConfiguration {
invalid_rule.status = "Enabld".to_string().into();
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![invalid_rule],
};
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
let mut invalid_delete = replication_rule();
invalid_delete.delete_replication = Some(PersistedReplicationStatus {
status: "Enabld".to_string(),
invalid_delete.delete_replication = Some(DeleteReplication {
status: "Enabld".to_string().into(),
});
let config = PersistedReplicationConfiguration {
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![invalid_delete],
};
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
let mut invalid_delete_marker = replication_rule();
invalid_delete_marker.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
status: Some("Enabld".to_string()),
invalid_delete_marker.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some("Enabld".to_string().into()),
});
let config = PersistedReplicationConfiguration {
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![invalid_delete_marker],
};
assert!(validate_delete_replication_config(&VersioningConfiguration::default(), Some(&config)).is_err());
let mut invalid_replica_modifications = replication_rule();
invalid_replica_modifications.source_selection_criteria = Some(PersistedSourceSelectionCriteria {
replica_modifications: Some(PersistedReplicationStatus {
status: "Enabld".to_string(),
invalid_replica_modifications.source_selection_criteria = Some(SourceSelectionCriteria {
replica_modifications: Some(ReplicaModifications {
status: "Enabld".to_string().into(),
}),
sse_kms_encrypted_objects: None,
});
let config = PersistedReplicationConfiguration {
let config = ReplicationConfiguration {
role: String::new(),
rules: vec![invalid_replica_modifications],
};
@@ -810,11 +804,11 @@ mod tests {
let mut metadata = BucketMetadata::new("bucket");
metadata.versioning_config_xml = b"configured".to_vec();
metadata.versioning_config = Some(VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
});
metadata.replication_config_xml = b"configured".to_vec();
metadata.replication_config = Some(PersistedReplicationConfiguration {
metadata.replication_config = Some(ReplicationConfiguration {
role: String::new(),
rules: vec![replication_rule()],
});
@@ -854,7 +848,7 @@ mod tests {
);
let mut inconsistent = BucketMetadata::new("bucket");
inconsistent.replication_config = Some(PersistedReplicationConfiguration {
inconsistent.replication_config = Some(ReplicationConfiguration {
role: String::new(),
rules: vec![replication_rule()],
});
@@ -869,11 +863,11 @@ mod tests {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut rule = replication_rule();
rule.destination.bucket = arn.to_string();
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
status: Some(Status::ENABLED.to_string()),
rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
});
let mut metadata = BucketMetadata::new("bucket");
metadata.replication_config = Some(PersistedReplicationConfiguration {
metadata.replication_config = Some(ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
});
@@ -904,27 +898,27 @@ mod tests {
fn delete_marker_source_read_is_required_only_for_tag_filtered_rules() {
let mut prefix_rule = replication_rule();
prefix_rule.prefix = Some("logs/".to_string());
prefix_rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
status: Some(Status::ENABLED.to_string()),
prefix_rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
});
let prefix_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![prefix_rule],
}),
);
let mut tag_rule = replication_rule();
tag_rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
status: Some(Status::ENABLED.to_string()),
tag_rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
});
tag_rule.filter = Some(PersistedReplicationFilter {
tag: Some(PersistedReplicationTag {
tag_rule.filter = Some(ReplicationRuleFilter {
tag: Some(Tag {
key: Some("class".to_string()),
value: Some("audit".to_string()),
}),
@@ -932,10 +926,10 @@ mod tests {
});
let tag_snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![tag_rule],
}),
@@ -950,14 +944,14 @@ mod tests {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut rule = replication_rule();
rule.destination.bucket = arn.to_string();
rule.delete_replication = Some(PersistedReplicationStatus {
status: Status::ENABLED.to_string(),
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
status: Some(Status::DISABLED.to_string()),
rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
});
let config = ReplicationConfig::new(
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
}),
@@ -991,22 +985,22 @@ mod tests {
#[test]
fn live_delete_does_not_trust_persisted_replica_status() {
let mut rule = replication_rule();
rule.delete_replication = Some(PersistedReplicationStatus {
status: Status::ENABLED.to_string(),
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
rule.source_selection_criteria = Some(PersistedSourceSelectionCriteria {
replica_modifications: Some(PersistedReplicationStatus {
status: Status::DISABLED.to_string(),
rule.source_selection_criteria = Some(SourceSelectionCriteria {
replica_modifications: Some(ReplicaModifications {
status: s3s::dto::ReplicaModificationsStatus::from_static(s3s::dto::ReplicaModificationsStatus::DISABLED),
}),
sse_kms_encrypted_objects: None,
});
let replication = PersistedReplicationConfiguration {
let replication = ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
};
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(replication.clone()),
@@ -1042,14 +1036,14 @@ mod tests {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let mut rule = replication_rule();
rule.destination.bucket = arn.to_string();
rule.delete_replication = Some(PersistedReplicationStatus {
status: Status::ENABLED.to_string(),
rule.delete_replication = Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
});
rule.delete_marker_replication = Some(PersistedOptionalReplicationStatus {
status: Some(Status::DISABLED.to_string()),
rule.delete_marker_replication = Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)),
});
let config = ReplicationConfig::new(
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![rule],
}),
@@ -5074,10 +5074,9 @@ mod tests {
#[tokio::test]
async fn heal_queue_marks_missing_versioning_state_as_missed() {
use super::super::replication_target_boundary::BucketTargets;
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::{
PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
PersistedReplicationStatus,
use s3s::dto::{
DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule,
ReplicationRuleStatus,
};
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
@@ -5092,14 +5091,14 @@ mod tests {
..Default::default()
},
ReplicationConfig::new(
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![PersistedReplicationRule {
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: Some(PersistedReplicationStatus {
status: Status::ENABLED.to_string(),
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
}),
destination: PersistedReplicationDestination {
destination: Destination {
bucket: arn.to_string(),
..Default::default()
},
@@ -5109,7 +5108,7 @@ mod tests {
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: Status::ENABLED.to_string(),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
Some(BucketTargets::default()),
@@ -85,8 +85,6 @@ use http_body_util::StreamBody;
use metrics::counter;
#[cfg(test)]
use rmp_serde;
#[cfg(test)]
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
use rustfs_s3_types::EventName;
use rustfs_utils::http::{
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_TAGGING_DIRECTIVE,
@@ -94,6 +92,8 @@ use rustfs_utils::http::{
insert_str, replication_target_versions,
};
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
#[cfg(test)]
use s3s::dto::ReplicationConfiguration;
use std::collections::{HashMap, HashSet};
use std::fmt::Display;
use std::pin::Pin;
@@ -6579,10 +6579,9 @@ mod tests {
use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
use super::*;
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
use rustfs_gateway_types::persistence::{
PersistedReplicationDestination, PersistedReplicationRule, PersistedReplicationStatus,
use s3s::dto::{
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ExcludedPrefix, ReplicationRule,
ReplicationRuleStatus, VersioningConfiguration,
};
use std::collections::HashMap;
use time::OffsetDateTime;
@@ -8694,7 +8693,7 @@ mod tests {
..Default::default()
};
let rcfg = ReplicationConfig::new(
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: role.to_string(),
rules: vec![],
}),
@@ -8720,7 +8719,7 @@ mod tests {
ReplicationVersioningStore::install_prefix_state_test_config(
&bucket,
VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
);
@@ -8753,20 +8752,22 @@ mod tests {
ReplicationVersioningStore::install_prefix_state_test_config(
&bucket,
VersioningConfiguration {
status: Some("Enabled".to_owned()),
excluded_prefixes: Some(vec![Some("archive/".to_string())]),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
excluded_prefixes: Some(vec![ExcludedPrefix {
prefix: Some("archive/".to_string()),
}]),
..Default::default()
},
);
let rcfg = ReplicationConfig::new(
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![PersistedReplicationRule {
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: Some(PersistedReplicationStatus {
status: Status::ENABLED.to_string(),
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
}),
destination: PersistedReplicationDestination {
destination: Destination {
bucket: arn.to_string(),
..Default::default()
},
@@ -8776,7 +8777,7 @@ mod tests {
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: Status::ENABLED.to_string(),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
Some(BucketTargets {
@@ -8812,14 +8813,14 @@ mod tests {
let admitted_arn = "arn:rustfs:replication:us-east-1:target:admitted";
let current_role = "arn:rustfs:replication:us-east-1:target:current";
let rcfg = ReplicationConfig::new(
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: current_role.to_string(),
rules: vec![PersistedReplicationRule {
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: Some(PersistedReplicationStatus {
status: Status::DISABLED.to_string(),
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
}),
destination: PersistedReplicationDestination {
destination: Destination {
bucket: current_role.to_string(),
..Default::default()
},
@@ -8829,7 +8830,7 @@ mod tests {
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: Status::ENABLED.to_string(),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
Some(BucketTargets {
@@ -8877,12 +8878,12 @@ mod tests {
#[tokio::test]
async fn heal_rejects_semantically_invalid_replication_config() {
let rcfg = ReplicationConfig::new(
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![PersistedReplicationRule {
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: None,
destination: PersistedReplicationDestination {
destination: Destination {
bucket: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
..Default::default()
},
@@ -8892,7 +8893,7 @@ mod tests {
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: "Enabld".to_owned(),
status: ReplicationRuleStatus::from_static("Enabld"),
}],
}),
Some(BucketTargets::default()),
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
use s3s::dto::ReplicationConfiguration;
use super::replication_config_boundary::{ObjectOpts, ReplicationConfigurationExt};
use super::replication_filemeta_boundary::ReplicationType;
@@ -20,7 +20,7 @@ use super::replication_filemeta_boundary::ReplicationType;
pub(crate) struct ReplicationTargetConfigBridge;
impl ReplicationTargetConfigBridge {
pub(crate) fn target_is_used_by_rules(config: &PersistedReplicationConfiguration, arn: &str) -> bool {
pub(crate) fn target_is_used_by_rules(config: &ReplicationConfiguration, arn: &str) -> bool {
config
.filter_target_arns(&ObjectOpts {
op_type: ReplicationType::All,
@@ -37,7 +37,7 @@ mod tests {
#[test]
fn target_config_bridge_matches_role_target() {
let config = PersistedReplicationConfiguration {
let config = ReplicationConfiguration {
role: " arn:target ".to_string(),
rules: Vec::new(),
};
@@ -15,7 +15,7 @@
use super::replication_error_boundary::Result;
use crate::bucket::{versioning::VersioningApi as _, versioning_sys::BucketVersioningSys};
#[cfg(test)]
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
use s3s::dto::VersioningConfiguration;
#[cfg(test)]
use std::{collections::HashMap, sync::LazyLock, sync::Mutex};
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_gateway_types::cors_tagging::PersistedTag as Tag;
use s3s::dto::Tag;
use std::cmp::Ordering;
use std::collections::HashMap;
use url::form_urlencoded;
+27 -211
View File
@@ -15,23 +15,9 @@
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result, StorageError};
use regex::Regex;
use rustfs_gateway_types::cors_tagging::{
CorsTaggingCodecError, PersistedCorsConfiguration, PersistedTagging, parse_cors, parse_tagging, serialize_cors,
serialize_tagging,
};
use rustfs_gateway_types::persistence::{
PersistedAccelerateConfiguration, PersistedBucketEncryptionConfiguration, PersistedBucketLoggingStatus,
PersistedLifecycleConfiguration, PersistedNotificationConfiguration, PersistedObjectLockConfiguration,
PersistedPublicAccessBlockConfiguration, PersistedReplicationConfiguration, PersistedRequestPaymentConfiguration,
PersistedVersioningConfiguration, PersistedWebsiteConfiguration, PersistenceCodecError, parse_accelerate,
parse_bucket_encryption, parse_bucket_logging, parse_lifecycle_legacy_dates, parse_notification, parse_object_lock,
parse_public_access_block, parse_replication, parse_request_payment, parse_versioning, parse_website, serialize_accelerate,
serialize_bucket_encryption, serialize_bucket_logging, serialize_lifecycle_legacy_dates, serialize_notification,
serialize_object_lock, serialize_public_access_block, serialize_replication, serialize_request_payment, serialize_versioning,
serialize_website,
};
use rustfs_utils::WindowsFileNameError;
use rustfs_utils::path::SLASH_SEPARATOR;
use s3s::xml;
use tracing::instrument;
pub fn is_meta_bucketname(name: &str) -> bool {
@@ -88,173 +74,37 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
check_bucket_name_common(bucket_name, true)
}
/// A bucket configuration persisted as XML in the bucket metadata, together
/// with the codec that reads and writes its stored bytes.
///
/// Every family reads and writes its stored bytes through the gateway
/// persistence codecs (`rustfs_gateway_types::persistence` and
/// `rustfs_gateway_types::cors_tagging`, rustfs/backlog#2745). The codecs keep
/// the persisted bytes the `bucket_config_goldens` pin, so a document the old
/// s3s codec wrote reads back and re-encodes unchanged.
pub trait BucketConfigXml: Sized {
/// Decodes the stored bytes of one configuration document.
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError>;
/// Encodes the value as RustFS stores it.
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError>;
// RUSTFS_COMPAT_TODO(s3gate-metadata-xml): the s3s codec reads persisted XML during migration. Remove after every supported writer uses the gateway codec and every retained metadata object and backup archive is verified or rewritten.
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
where
T: for<'xml> xml::Deserialize<'xml>,
{
let mut d = xml::Deserializer::new(input);
let ans = T::deserialize(&mut d)?;
d.expect_eof()?;
Ok(ans)
}
/// Why a persisted bucket configuration could not be decoded or encoded.
#[derive(Debug, thiserror::Error)]
pub enum BucketConfigXmlError {
/// A gateway persistence codec refused the bytes or the value.
#[error(transparent)]
Persisted(#[from] PersistenceCodecError),
/// The gateway CORS or Tagging codec refused the bytes or the value.
#[error(transparent)]
CorsTagging(#[from] CorsTaggingCodecError),
#[allow(
dead_code,
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
)]
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
let mut buf = Vec::with_capacity(256);
{
let mut ser = xml::Serializer::new(&mut buf);
val.serialize_content(&mut ser)?;
}
Ok(String::from_utf8(buf).unwrap())
}
/// Decodes one persisted bucket configuration document.
pub fn deserialize<T: BucketConfigXml>(input: &[u8]) -> std::result::Result<T, BucketConfigXmlError> {
T::decode_config_xml(input)
}
/// Encodes one bucket configuration as RustFS persists it.
pub fn serialize<T: BucketConfigXml>(val: &T) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
val.encode_config_xml()
}
impl BucketConfigXml for PersistedReplicationConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_replication(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_replication(self)?)
}
}
impl BucketConfigXml for PersistedLifecycleConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_lifecycle_legacy_dates(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_lifecycle_legacy_dates(self)?)
}
}
// The gateway writers of these families are infallible: a value that exists is
// always encodable, so only the readers can refuse bytes.
impl BucketConfigXml for PersistedAccelerateConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_accelerate(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_accelerate(self))
}
}
impl BucketConfigXml for PersistedBucketLoggingStatus {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_bucket_logging(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_bucket_logging(self))
}
}
impl BucketConfigXml for PersistedCorsConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_cors(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_cors(self))
}
}
impl BucketConfigXml for PersistedNotificationConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_notification(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_notification(self))
}
}
impl BucketConfigXml for PersistedObjectLockConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_object_lock(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_object_lock(self))
}
}
impl BucketConfigXml for PersistedPublicAccessBlockConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_public_access_block(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_public_access_block(self))
}
}
impl BucketConfigXml for PersistedRequestPaymentConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_request_payment(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_request_payment(self))
}
}
impl BucketConfigXml for PersistedBucketEncryptionConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_bucket_encryption(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_bucket_encryption(self))
}
}
impl BucketConfigXml for PersistedTagging {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_tagging(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_tagging(self))
}
}
impl BucketConfigXml for PersistedVersioningConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_versioning(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_versioning(self))
}
}
impl BucketConfigXml for PersistedWebsiteConfiguration {
fn decode_config_xml(input: &[u8]) -> std::result::Result<Self, BucketConfigXmlError> {
Ok(parse_website(input)?)
}
fn encode_config_xml(&self) -> std::result::Result<Vec<u8>, BucketConfigXmlError> {
Ok(serialize_website(self))
pub fn serialize<T: xml::Serialize>(val: &T) -> xml::SerResult<Vec<u8>> {
let mut buf = Vec::with_capacity(256);
{
let mut ser = xml::Serializer::new(&mut buf);
val.serialize(&mut ser)?;
}
Ok(buf)
}
pub fn has_bad_path_component(path: &str) -> bool {
@@ -596,40 +446,6 @@ mod tests {
use super::*;
use proptest::prelude::*;
/// Every bucket-configuration family reads and writes through its own
/// gateway codec (rustfs/backlog#2745): the family's golden document
/// decodes and re-encodes to the same bytes, and a document of another
/// family is refused instead of decoding into an empty configuration.
#[test]
fn every_bucket_config_family_uses_its_own_gateway_codec() {
fn assert_family<T: BucketConfigXml>(family: &str, own: &[u8], foreign: &[u8]) {
let value = deserialize::<T>(own).unwrap_or_else(|err| panic!("{family}: own document must decode: {err}"));
let encoded = serialize(&value).unwrap_or_else(|err| panic!("{family}: value must encode: {err}"));
assert_eq!(encoded, own, "{family}: re-encoding must keep the persisted bytes");
assert!(deserialize::<T>(foreign).is_err(), "{family}: a foreign root element must be refused");
}
macro_rules! golden {
($family:literal) => {
include_bytes!(concat!("../../tests/fixtures/bucket-config-goldens/", $family, "/minio.out.xml"))
};
}
let tagging = golden!("tagging");
let versioning = golden!("versioning");
assert_family::<PersistedAccelerateConfiguration>("accelerate", golden!("accelerate"), tagging);
assert_family::<PersistedBucketLoggingStatus>("logging", golden!("logging"), tagging);
assert_family::<PersistedCorsConfiguration>("cors", golden!("cors"), tagging);
assert_family::<PersistedNotificationConfiguration>("notification", golden!("notification"), tagging);
assert_family::<PersistedObjectLockConfiguration>("object_lock", golden!("object_lock"), tagging);
assert_family::<PersistedPublicAccessBlockConfiguration>("public_access_block", golden!("public_access_block"), tagging);
assert_family::<PersistedRequestPaymentConfiguration>("request_payment", golden!("request_payment"), tagging);
assert_family::<PersistedBucketEncryptionConfiguration>("encryption", golden!("encryption"), tagging);
assert_family::<PersistedTagging>("tagging", tagging, versioning);
assert_family::<PersistedVersioningConfiguration>("versioning", versioning, tagging);
assert_family::<PersistedWebsiteConfiguration>("website", golden!("website"), tagging);
}
/// rustfs#5785: keys whose path segments exceed the on-disk NAME_MAX
/// budget must be rejected up front as ObjectNameInvalid (4xx), not leak
/// ENAMETOOLONG as InternalError 500 from the disk layer.
+25 -17
View File
@@ -12,8 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
use rustfs_utils::string::match_simple;
use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
pub trait VersioningApi {
fn enabled(&self) -> bool;
@@ -28,10 +28,10 @@ pub trait VersioningApi {
impl VersioningApi for VersioningConfiguration {
fn enabled(&self) -> bool {
self.status.as_deref() == Some("Enabled")
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED))
}
fn prefix_enabled(&self, prefix: &str) -> bool {
if self.status.as_deref() != Some("Enabled") {
if self.status != Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
return false;
}
@@ -44,10 +44,12 @@ impl VersioningApi for VersioningConfiguration {
}
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
for sprefix in excluded_prefixes.iter().flatten() {
let pattern = format!("{sprefix}*");
if match_simple(&pattern, prefix) {
return false;
for p in excluded_prefixes.iter() {
if let Some(ref sprefix) = p.prefix {
let pattern = format!("{sprefix}*");
if match_simple(&pattern, prefix) {
return false;
}
}
}
}
@@ -56,11 +58,11 @@ impl VersioningApi for VersioningConfiguration {
}
fn prefix_suspended(&self, prefix: &str) -> bool {
if self.status.as_deref() == Some("Suspended") {
if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)) {
return true;
}
if self.status.as_deref() == Some("Enabled") {
if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) {
if prefix.is_empty() {
return false;
}
@@ -73,10 +75,12 @@ impl VersioningApi for VersioningConfiguration {
}
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
for sprefix in excluded_prefixes.iter().flatten() {
let pattern = format!("{sprefix}*");
if match_simple(&pattern, prefix) {
return true;
for p in excluded_prefixes.iter() {
if let Some(ref sprefix) = p.prefix {
let pattern = format!("{sprefix}*");
if match_simple(&pattern, prefix) {
return true;
}
}
}
}
@@ -88,12 +92,14 @@ impl VersioningApi for VersioningConfiguration {
self.prefix_enabled(prefix) || self.prefix_suspended(prefix)
}
fn suspended(&self) -> bool {
self.status.as_deref() == Some("Suspended")
self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED))
}
}
#[cfg(test)]
mod tests {
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix};
use super::*;
struct LegacyVersioning;
@@ -128,8 +134,10 @@ mod tests {
#[test]
fn delete_state_treats_excluded_prefixes_as_unversioned() {
let config = VersioningConfiguration {
status: Some("Enabled".to_owned()),
excluded_prefixes: Some(vec![Some("archive/".to_string())]),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
excluded_prefixes: Some(vec![ExcludedPrefix {
prefix: Some("archive/".to_string()),
}]),
..Default::default()
};
@@ -138,7 +146,7 @@ mod tests {
assert_eq!(config.delete_state("live/object"), (true, false));
let suspended = VersioningConfiguration {
status: Some("Suspended".to_owned()),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
..Default::default()
};
assert_eq!(suspended.delete_state("archive/object"), (false, true));
+2 -2
View File
@@ -17,7 +17,7 @@ use super::metadata::unreadable_config_refusal;
use super::{metadata_sys::get_bucket_metadata_sys, versioning::VersioningApi};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::Result;
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
use s3s::dto::VersioningConfiguration;
use tracing::{error, warn};
pub struct BucketVersioningSys {}
@@ -163,7 +163,7 @@ mod tests {
fn enabled() -> VersioningConfiguration {
VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
..Default::default()
}
}
@@ -217,8 +217,6 @@ impl Drop for NotifyOnDrop {
#[derive(Default)]
pub struct ListPathRawOptions {
/// Retain native listing admission through producer cancellation/draining.
pub read_budget: Option<Arc<tokio::sync::OwnedSemaphorePermit>>,
pub disks: Vec<Option<DiskStore>>,
pub fallback_disks: Vec<Option<DiskStore>>,
pub bucket: String,
@@ -229,7 +227,6 @@ pub struct ListPathRawOptions {
pub hidden_entry_budget: usize,
pub pending_prefix: Option<String>,
pub filter_prefix: Option<String>,
pub identifier_tree_leaf: Option<String>,
pub forward_to: Option<String>,
pub min_disks: usize,
/// Deliver every replica to the partial callback, including matching headers.
@@ -258,7 +255,6 @@ pub struct ListPathRawOptions {
impl Clone for ListPathRawOptions {
fn clone(&self) -> Self {
Self {
read_budget: self.read_budget.clone(),
disks: self.disks.clone(),
fallback_disks: self.fallback_disks.clone(),
bucket: self.bucket.clone(),
@@ -269,7 +265,6 @@ impl Clone for ListPathRawOptions {
hidden_entry_budget: self.hidden_entry_budget,
pending_prefix: self.pending_prefix.clone(),
filter_prefix: self.filter_prefix.clone(),
identifier_tree_leaf: self.identifier_tree_leaf.clone(),
forward_to: self.forward_to.clone(),
min_disks: self.min_disks,
preserve_replica_metadata: self.preserve_replica_metadata,
@@ -302,7 +297,6 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
pending_prefix: opts.pending_prefix.clone(),
report_notfound: opts.report_not_found,
filter_prefix: opts.filter_prefix.clone(),
identifier_tree_leaf: opts.identifier_tree_leaf.clone(),
forward_to: opts.forward_to.clone(),
limit: opts.per_disk_limit,
skip_total_timeout: opts.skip_walkdir_total_timeout,
@@ -395,9 +389,7 @@ async fn list_path_raw_inner(
let disk_io_context_clone = disk_io_context.clone();
let (rd, wr) = tokio::io::duplex(64 * 1024);
readers.push(MetacacheReader::new(rd));
let read_budget = opts.read_budget.clone();
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
let _read_budget = read_budget;
let mut wr = PublishedBytesWriter::new(wr);
#[cfg(test)]
if let Some(observer) = &test_disk_io_context_observed {
@@ -1236,21 +1228,6 @@ mod tests {
assert!(!options.skip_total_timeout);
}
#[test]
fn identifier_tree_selection_survives_job_clone_and_disk_request() {
let request = ListPathRawOptions {
path: "catalog/namespaces/".to_string(),
identifier_tree_leaf: Some("table-entry.json".to_string()),
recursive: true,
..Default::default()
};
let cloned = request.clone();
let walk = walk_dir_options(&cloned);
assert_eq!(walk.identifier_tree_leaf, request.identifier_tree_leaf);
assert_eq!(walk.base_dir, request.path);
assert!(walk.recursive);
}
#[test]
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
let request = ListPathRawOptions {
@@ -1920,42 +1897,6 @@ mod tests {
assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &["bucket/object".to_string()]);
}
#[tokio::test]
async fn list_read_budget_is_retained_by_raw_producer_until_cancellation() {
let semaphore = Arc::new(tokio::sync::Semaphore::new(4));
let grant = Arc::new(semaphore.clone().acquire_many_owned(4).await.expect("listing grant"));
let weak_grant = Arc::downgrade(&grant);
let entered = Arc::new(tokio::sync::Notify::new());
let stopped = Arc::new(tokio::sync::Notify::new());
let worker = tokio::spawn(list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
read_budget: Some(grant),
disks: vec![None],
min_disks: 1,
test_reader_behaviors: vec![TestReaderBehavior::StallWithSignals {
entered: entered.clone(),
stopped: stopped.clone(),
}],
..Default::default()
},
));
timeout(Duration::from_secs(5), entered.notified())
.await
.expect("producer entered");
assert!(weak_grant.strong_count() >= 2, "producer must retain a grant independently of the parent");
assert_eq!(semaphore.available_permits(), 0);
worker.abort();
assert!(worker.await.expect_err("parent cancelled").is_cancelled());
timeout(Duration::from_secs(5), stopped.notified())
.await
.expect("producer stopped");
let _next = timeout(Duration::from_secs(5), semaphore.acquire_many(4))
.await
.expect("cancelled listing releases all permits")
.expect("budget stays open");
}
#[tokio::test]
async fn list_path_raw_aborts_unresponsive_producer_after_quorum_eof() {
let result = timeout(
File diff suppressed because it is too large Load Diff
+11 -11
View File
@@ -35,17 +35,17 @@ pub use client::{
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
#[allow(unused_imports)]
pub use http_auth::{
RPC_BACKGROUND_ADMISSION_HEADER, RpcReplayCacheAdmissionError, TONIC_RPC_PREFIX, build_auth_headers,
build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers,
gen_tonic_signature_headers, normalize_tonic_rpc_audience, rpc_auth_is_resource_exhausted, rpc_background_admission,
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest,
set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
RPC_BACKGROUND_ADMISSION_HEADER, TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer,
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
normalize_tonic_rpc_audience, rpc_background_admission, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest,
set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability,
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
};
#[cfg(test)]
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
@@ -1965,15 +1965,6 @@ impl PeerRestClient {
Ok((self.topology_member.clone(), epoch))
}
pub async fn probe_restore_copyback_capability(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
let probe = rustfs_protos::restore_copyback_capability_probe(Uuid::new_v4().as_bytes());
let result = self
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
.await?;
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
Ok((self.topology_member.clone(), epoch))
}
pub async fn probe_cross_pool_fence(&self, topology_fingerprint: String) -> Result<(String, u32, Uuid)> {
let mut probe = rustfs_protos::CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX.to_vec();
probe.extend_from_slice(Uuid::new_v4().as_bytes());
@@ -18,7 +18,7 @@ use crate::cluster::rpc::client::{
node_service_time_out_client,
};
use crate::cluster::rpc::set_tonic_mutation_body_digest;
use crate::core::pools::{PoolMeta, PoolMetaSaveGate, PoolMetaSaveGateWriteGuard};
use crate::core::pools::{PoolMeta, PoolMetaWriteState};
use crate::disk::error::DiskError;
use crate::disk::error::{Error, Result};
use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
@@ -798,12 +798,12 @@ fn try_acquire_bucket_heal_movement_guard<'a>(
}
async fn acquire_bucket_heal_write_guard<'a>(
gate: Option<&'a PoolMetaSaveGate>,
) -> Result<Option<PoolMetaSaveGateWriteGuard<'a>>> {
gate: Option<&'a tokio::sync::Mutex<PoolMetaWriteState>>,
) -> Result<Option<tokio::sync::MutexGuard<'a, PoolMetaWriteState>>> {
let Some(gate) = gate else {
return Ok(None);
};
let guard = gate.write().await;
let guard = gate.lock().await;
guard
.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")
.map_err(|e| e.narrow_to_disk().unwrap_or_else(Error::other))?;
@@ -2388,8 +2388,8 @@ mod tests {
#[tokio::test]
async fn local_bucket_heal_refuses_receiver_with_unsafe_pool_metadata() {
let gate = PoolMetaSaveGate::default();
gate.write().await.observe_replicas(PoolMetaReplicaState {
let gate = tokio::sync::Mutex::new(PoolMetaWriteState::default());
gate.lock().await.observe_replicas(PoolMetaReplicaState {
needs_repair: true,
repair_write_safe: false,
});
+1 -54
View File
@@ -376,21 +376,6 @@ struct WalkCopyError {
source: io::Error,
}
impl WalkCopyError {
fn into_disk_error(self) -> DiskError {
// Only a local output write/flush proves the metacache consumer closed.
// An identical I/O kind from the remote input remains a transport failure.
if matches!(self.reason, RemoteWalkFailureReason::Write | RemoteWalkFailureReason::Flush)
&& self.source.kind() == io::ErrorKind::BrokenPipe
&& self.source.get_ref().is_none()
{
DiskError::metacache_output_stream_closed()
} else {
DiskError::Io(self.source)
}
}
}
async fn copy_stream_with_buffer<R, W>(
reader: &mut R,
writer: &mut W,
@@ -2111,7 +2096,6 @@ impl RemoteDisk {
failure_health_action: FailureHealthAction,
) {
if let Err(err) = operation_result
&& !err.is_metacache_output_stream_closed()
&& is_network_like_disk_error(err)
{
counter!(
@@ -4072,7 +4056,7 @@ impl DiskAPI for RemoteDisk {
Ok(_) => return Ok(()),
Err(error) => {
failure_reason = Some(error.reason);
return Err(error.into_disk_error());
return Err(DiskError::Io(error.source));
}
}
}
@@ -9703,43 +9687,6 @@ mod tests {
}
}
#[tokio::test]
async fn remote_walk_preserves_local_metacache_consumer_close() {
let transport =
RetryingWalkDirInternodeDataTransport::with_steps(vec![WalkDirTestStep::Data(b"metacache payload".to_vec())]);
let disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let (mut writer, reader) = tokio::io::duplex(1);
drop(reader);
let error = disk
.walk_dir(WalkDirOptions::default(), &mut writer)
.await
.expect_err("a dropped metacache consumer must stop the producer");
assert!(
error.is_metacache_output_stream_closed(),
"preserve the benign completion marker: {error:?}"
);
assert_eq!(transport.calls().len(), 1, "local consumer closure is not a remote open retry");
assert!(!disk.health.is_faulty());
}
#[tokio::test]
async fn remote_walk_keeps_remote_read_broken_pipe_as_transport_failure() {
let transport = RetryingWalkDirInternodeDataTransport::with_steps(vec![WalkDirTestStep::PartialDataThenError {
data: b"partial metacache".to_vec(),
error: io::ErrorKind::BrokenPipe.into(),
}]);
let disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let mut writer = Vec::new();
let error = disk
.walk_dir(WalkDirOptions::default(), &mut writer)
.await
.expect_err("an upstream close after partial output is a failed walk");
assert!(!error.is_metacache_output_stream_closed());
assert!(error.contains_io_error_kind(io::ErrorKind::BrokenPipe));
assert_eq!(writer, b"partial metacache");
assert_eq!(transport.calls().len(), 1, "partial output must not be replayed");
}
#[tokio::test]
async fn walk_stream_backpressure_limits_read_ahead_and_is_cancellable() {
let mut reader = Cursor::new(vec![5; 4 * DEFAULT_READ_BUFFER_SIZE]);
@@ -423,15 +423,11 @@ impl RemoteClient {
/// peer toward the offline threshold. See issue #4567.
fn is_transport_failure(status: &tonic::Status) -> bool {
use tonic::Code;
// A valid stale-epoch request is rejected before execution. Its signed
// response refreshes the peer epoch on the existing healthy channel.
let epoch_rejection = status.code() == Code::Unavailable && status.message() == "RPC boot epoch changed";
std::error::Error::source(status).is_some()
|| (!epoch_rejection
&& matches!(
status.code(),
Code::Unavailable | Code::DeadlineExceeded | Code::Unknown | Code::Cancelled
))
|| matches!(
status.code(),
Code::Unavailable | Code::DeadlineExceeded | Code::Unknown | Code::Cancelled
)
}
async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str) {
@@ -2831,37 +2827,6 @@ mod tests {
}
}
#[tokio::test]
async fn stale_boot_epoch_rejections_do_not_open_lock_request_breaker() {
let addr = "http://stale-boot-epoch-lock-breaker:9000";
let client = RemoteClient::new(addr.to_string());
reset_lock_peer_health_for_test(addr);
for _ in 0..LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
let error = client
.execute_rpc::<(), _>(
"lock",
"bucket/new-object",
Duration::from_secs(1),
async { Err(tonic::Status::unavailable("RPC boot epoch changed")) },
RpcCleanup::new(None, None, None),
)
.await
.expect_err("a stale epoch still rejects the operation before execution");
assert!(error.to_string().contains("RPC boot epoch changed"));
}
let health = lock_peer_health_for_test(addr);
assert_eq!(health.request_consecutive_timeouts, 0);
assert!(health.request_backoff_until.is_none());
assert!(health.last_eviction.is_none());
let admission = client
.admit_lock_request("lock")
.expect("authenticated epoch refresh may proceed immediately");
client.release_lock_request("lock", Some(admission));
reset_lock_peer_health_for_test(addr);
}
#[test]
#[serial_test::serial]
fn test_remote_client_rpc_timeout_honors_configured_deadline() {
@@ -628,7 +628,7 @@ async fn pool_meta_native_physical_runtime_fork_after_apply_latches_original_gat
let fork = identity_epoch_fork(&fixture).await;
let since = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixed original failure timestamp should be valid");
{
let mut state = fixture.store.pool_meta_save_gate.write().await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
state.write_blocked = false;
state.block_context = None;
@@ -685,7 +685,7 @@ async fn pool_meta_native_physical_runtime_fork_after_apply_latches_original_gat
);
assert_committed(&fixture).await;
{
let state = fixture.store.pool_meta_save_gate.write().await;
let state = fixture.store.pool_meta_save_gate.lock().await;
assert!(
state.write_blocked,
"candidate validation must propagate its permanent latch to the original state"
@@ -706,7 +706,7 @@ async fn pool_meta_native_physical_runtime_fork_after_apply_latches_original_gat
.expect("permanently blocked recovery should return without mutation"),
"ordinary-readable repaired heads must not clear the original known-fork latch"
);
let state = fixture.store.pool_meta_save_gate.write().await;
let state = fixture.store.pool_meta_save_gate.lock().await;
assert!(state.write_blocked);
assert_eq!(
state
@@ -979,7 +979,7 @@ async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_ra
async {
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.write().await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
@@ -1036,7 +1036,7 @@ async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.write().await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
@@ -1110,7 +1110,7 @@ async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.write().await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
tokio::time::timeout(std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
.await.expect("mixed legacy heal must terminate")
@@ -1199,7 +1199,7 @@ async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order()
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.write().await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture
@@ -1301,7 +1301,7 @@ async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes()
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.write().await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
@@ -1410,7 +1410,7 @@ async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes()
let before = snapshots(&fixture.store).await;
let identities = owner_repair_identity_heads(&fixture.store).await;
let guard = owner(&fixture.store).await;
let mut state = fixture.store.pool_meta_save_gate.write().await;
let mut state = fixture.store.pool_meta_save_gate.lock().await;
let result = tokio::time::timeout(
std::time::Duration::from_secs(30),
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
File diff suppressed because it is too large Load Diff
+6 -37
View File
@@ -527,8 +527,8 @@ mod decommission_lock_order_tests {
rebalance_meta: tokio::sync::RwLock::new(other_store.rebalance_meta.read().await.clone()),
decommission_cancelers: tokio::sync::RwLock::new(vec![None; other_store.pools.len()]),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(
other_store.pool_meta_save_gate.write().await.independent_clone_for_test(),
pool_meta_save_gate: tokio::sync::Mutex::new(
other_store.pool_meta_save_gate.lock().await.independent_clone_for_test(),
),
ctx,
bucket_fence_registry: Arc::default(),
@@ -1616,7 +1616,7 @@ mod decommission_lock_order_tests {
run_routine_to_a_deferral_pause(
&store,
1,
"target pool 1 no longer fits the selected mutation: target pool 1 reservation requires",
"target pool 1 no longer fits the selected mutation: source pool 0 reservation estimate is exhausted",
)
.await;
});
@@ -1647,7 +1647,7 @@ mod decommission_lock_order_tests {
run_routine_to_a_deferral_pause(
&store,
1,
"target pool 1 no longer fits the selected mutation: target pool 1 reservation requires",
"target pool 1 no longer fits the selected mutation: source pool 0 reservation estimate is exhausted",
)
.await;
});
@@ -6062,12 +6062,6 @@ mod decommission_lock_order_tests {
)
.await
.expect("read the next source object for the multipart capacity oracle");
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_result = tokio::time::timeout(
Duration::from_secs(30),
data_movement::migrate_decommission_object(
@@ -7833,12 +7827,6 @@ mod decommission_lock_order_tests {
.expect("same-mutation retry should publish a readable target");
assert_eq!(retry_info.size, i64::try_from(body.len()).expect("test body size fits i64"));
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_object = "data-movement-put-after-retry.bin";
let next_version_id = uuid::Uuid::new_v4().to_string();
let mut next_opts = ObjectOptions {
@@ -7853,11 +7841,7 @@ mod decommission_lock_order_tests {
let next_result = other_store
.put_object_for_data_movement(RUSTFS_META_BUCKET, next_object, &mut next_data, &next_opts, None)
.await;
let next_error = match next_result {
Err(error) | Ok((_, Err(error))) => error,
Ok((_, Ok(_))) => panic!("a different mutation must not reuse A's recovered capacity"),
};
assert!(next_error.to_string().contains("decommission_capacity_blocked"), "{next_error}");
assert!(next_result.is_err(), "a different mutation must not reuse A's recovered capacity");
let next_target_err = other_store.pools[2]
.get_object_info(
RUSTFS_META_BUCKET,
@@ -9945,12 +9929,6 @@ mod decommission_lock_order_tests {
)
.await
.expect("read the next source object for the capacity oracle");
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_result = tokio::time::timeout(
Duration::from_secs(30),
data_movement::migrate_decommission_object(
@@ -10212,20 +10190,11 @@ mod decommission_lock_order_tests {
assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0);
assert_eq!(reconciled_reservation.consumed_target_physical_bytes, 1);
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 1, 2, 1);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_after_reconcile = other_store
.delete_object(&bucket, next_object, next_opts)
.await
.expect_err("the consumed exact-fit capacity must reject the next marker");
assert!(
next_after_reconcile.to_string().contains("decommission_capacity_blocked"),
"{next_after_reconcile}"
);
assert!(next_after_reconcile.to_string().contains("decommission_capacity_blocked"));
let next_target_err = other_store.pools[2]
.get_object_info(
&bucket,
-40
View File
@@ -65,38 +65,6 @@ use uuid::Uuid;
const LIST_MULTIPART_SETS_CONCURRENCY: usize = 4;
#[cfg(test)]
type TestSetLockers = HashMap<usize, Vec<Arc<dyn rustfs_lock::LockClient>>>;
#[cfg(test)]
static TEST_SET_LOCKERS: std::sync::LazyLock<std::sync::Mutex<TestSetLockers>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
/// Keep a fixture's real locker domain scoped to its owning instance.
#[cfg(test)]
pub(crate) struct ScopedSetLockers(Arc<InstanceContext>);
#[cfg(test)]
impl ScopedSetLockers {
pub(crate) fn install(ctx: Arc<InstanceContext>, lockers: Vec<Arc<dyn rustfs_lock::LockClient>>) -> Self {
let key = Arc::as_ptr(&ctx) as usize;
let mut domains = TEST_SET_LOCKERS.lock().expect("test set lockers mutex");
assert!(!domains.contains_key(&key), "the instance already owns a test locker domain");
domains.insert(key, lockers);
Self(ctx)
}
}
#[cfg(test)]
impl Drop for ScopedSetLockers {
fn drop(&mut self) {
TEST_SET_LOCKERS
.lock()
.expect("test set lockers mutex")
.remove(&(Arc::as_ptr(&self.0) as usize));
}
}
fn is_idempotent_delete_prefix_error(err: &Error) -> bool {
is_err_object_not_found(err) || is_err_strict_volume_not_found(err)
}
@@ -298,14 +266,6 @@ impl Sets {
}
}
#[cfg(test)]
let pool_lockers = TEST_SET_LOCKERS
.lock()
.expect("test set lockers mutex")
.get(&(Arc::as_ptr(&instance_ctx) as usize))
.cloned()
.unwrap_or_else(|| pool_lockers.clone());
let set_disks = SetDisks::new_with_instance_ctx(
runtime_sources::local_node_name().await,
Arc::new(RwLock::new(set_drive)),
+1 -1
View File
@@ -3436,7 +3436,7 @@ mod tests {
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: TokioMutex::new(()),
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
pool_meta_save_gate: TokioMutex::default(),
ctx,
bucket_fence_registry: Arc::default(),
})
@@ -682,11 +682,15 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
return Err(Error::other("ServerNotInitialized"));
};
collect_pools_info(all_disks, |pool_index, set_index| {
let store = store.clone();
async move {
let mut pools_info: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
for d in all_disks {
let pool_info = pools_info.entry(d.pool_index).or_default();
let erasure_set = pool_info.entry(d.set_index).or_default();
if erasure_set.id == 0 {
erasure_set.id = d.set_index;
let load = load_data_usage_cache(
store.pools[pool_index as usize].disk_set[set_index as usize].as_ref(),
store.pools[d.pool_index as usize].disk_set[d.set_index as usize].as_ref(),
DATA_USAGE_CACHE_NAME,
);
#[cfg(feature = "diagnostic-read-version-caller")]
@@ -694,34 +698,11 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ServerInfoPools,
load,
);
load.await
}
})
.await
}
async fn collect_pools_info<F, Fut>(all_disks: &[Disk], mut load_usage: F) -> Result<HashMap<i32, HashMap<i32, ErasureSetInfo>>>
where
F: FnMut(i32, i32) -> Fut,
Fut: std::future::Future<Output = Result<rustfs_data_usage::DataUsageCache>>,
{
let mut pools_info: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
for d in all_disks {
let pool_info = pools_info.entry(d.pool_index).or_default();
let erasure_set = match pool_info.entry(d.set_index) {
std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
std::collections::hash_map::Entry::Vacant(entry) => {
let erasure_set = entry.insert(ErasureSetInfo {
id: d.set_index,
..Default::default()
});
match load_usage(d.pool_index, d.set_index).await {
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
}
erasure_set
match load.await {
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
}
};
}
erasure_set.raw_capacity += d.total_space;
erasure_set.raw_usage += d.used_space;
@@ -1260,59 +1241,6 @@ mod tests {
assert_eq!(info.deployment_id, expected_deployment_id);
}
#[tokio::test]
async fn pool_usage_is_loaded_once_per_set_including_set_zero() {
let mut disks = Vec::new();
for (pool, set, count) in [(0, 0, 16), (1, 0, 16), (1, 1, 4)] {
disks.extend((0..count).map(|_| Disk {
pool_index: pool,
set_index: set,
used_space: 10,
total_space: 100,
..Default::default()
}));
}
let mut loads = Vec::new();
let info = super::collect_pools_info(&disks, |pool, set| {
loads.push((pool, set));
async move {
if pool == 1 && set == 0 {
return Err(crate::error::Error::other("usage unavailable"));
}
let mut cache = rustfs_data_usage::DataUsageCache::default();
cache.info.name = DATA_USAGE_ROOT.to_owned();
cache.info.last_update = Some(std::time::SystemTime::UNIX_EPOCH);
cache.info.snapshot_complete = true;
cache.replace(
DATA_USAGE_ROOT,
"",
rustfs_data_usage::DataUsageEntry {
size: 25,
objects: 2,
..Default::default()
},
);
Ok(cache)
}
})
.await
.expect("collect mixed successful and unavailable usage");
assert_eq!(loads, [(0, 0), (1, 0), (1, 1)]);
for (pool, set, count) in [(0, 0, 16), (1, 0, 16), (1, 1, 4)] {
let set_info = &info[&pool][&set];
assert_eq!(set_info.id, set);
assert_eq!(set_info.raw_usage, count * 10);
assert_eq!(set_info.raw_capacity, count * 100);
if pool == 1 && set == 0 {
assert_eq!(set_info.usage_error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
} else {
assert_eq!(set_info.usage, 25, "logical usage must not multiply by drive count");
assert_eq!(set_info.objects_count, 2);
assert!(set_info.usage_error.is_none());
}
}
}
#[serial]
#[tokio::test]
async fn local_server_property_includes_runtime_stats_without_endpoint_pools() {
-9
View File
@@ -3053,15 +3053,6 @@ impl DiskAPI for LocalDiskWrapper {
})
}
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<super::os::NativeBucketCreationWitness>> {
self.track_disk_health_with_op(
"bucket_creation_witness",
|| async { self.disk.bucket_creation_witness(volume).await },
Duration::ZERO,
)
.await
}
async fn make_volume(&self, volume: &str) -> Result<()> {
// Scoped heal must drain directory creation before releasing its lifecycle owner.
let timeout = if crate::store::bucket_heal_scope(volume).is_some() {
-7
View File
@@ -92,9 +92,6 @@ pub const O_APPEND: FileMode = 0x00400;
// create_new: bool,
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
if mode & O_CREATE != 0 {
super::os::record_native_publication(path.as_ref());
}
let path = path.as_ref().to_path_buf();
native_io(move || {
let mut options = std::fs::OpenOptions::new();
@@ -147,7 +144,6 @@ pub fn create_dir_all_std(path: &Path) -> io::Result<()> {
}
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
super::os::record_native_publication(path.as_ref());
let path = path.as_ref().to_path_buf();
native_io(move || create_dir_all_std(&path)).await
}
@@ -220,13 +216,11 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
#[cfg(test)]
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
super::os::record_native_publication(path.as_ref());
let path = path.as_ref().to_path_buf();
native_io(move || std::fs::create_dir(path)).await
}
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
super::os::record_native_publication(to.as_ref());
let from = from.as_ref().to_path_buf();
let to = to.as_ref().to_path_buf();
native_io(move || std::fs::rename(from, to)).await
@@ -234,7 +228,6 @@ pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<
#[cfg(any(not(windows), test))]
pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
super::os::record_native_publication(to.as_ref());
std::fs::rename(from, to)
}
File diff suppressed because it is too large Load Diff
+48 -157
View File
@@ -15,6 +15,8 @@
//! Single-disk object rename publication and rollback. The shared execution core
//! retains instrumentation, mutation leases, and commit guards through the syscall.
#[cfg(target_os = "linux")]
use super::has_rename_data_test_failpoints;
#[cfg(all(test, windows))]
use super::run_destination_commit_directory_preparation;
#[cfg(any(not(windows), test))]
@@ -22,8 +24,8 @@ use super::should_fail_local_inline_rollback_hardlink;
use super::{
EVENT_DISK_LOCAL_ACCESS_FAILED, EVENT_DISK_LOCAL_HEAL_PURGE_FAILED, EVENT_DISK_LOCAL_RENAME_REJECTED, LOG_COMPONENT_ECSTORE,
LOG_SUBSYSTEM_DISK_LOCAL, LocalDisk, SyncMode, effective_durability, has_inline_commit_test_failpoints,
has_rename_data_test_failpoints, inline_metadata_rollback_dir, observe_old_current, remove_dir_all_if_exists,
remove_dst_base_before_commit, remove_file_if_exists, rename_data_versions_signature, run_inline_preparation_before_backup,
inline_metadata_rollback_dir, observe_old_current, remove_dir_all_if_exists, remove_dst_base_before_commit,
remove_file_if_exists, rename_data_versions_signature, run_inline_preparation_before_backup,
should_fail_after_metadata_commit, should_fail_before_old_metadata_backup, should_fail_commit_rename,
should_remove_staged_meta_before_commit, skip_access_checks, tmp_staging_mutation_path,
};
@@ -170,44 +172,6 @@ impl RenameCommitContext<'_> {
}
}
#[cfg(any(not(windows), test))]
fn inline_rollback_backup_path(staging_file_path: &Path) -> std::io::Result<PathBuf> {
let Some(staging_parent) = staging_file_path.parent() else {
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
};
let backup_path = if staging_file_path.file_name().and_then(|n| n.to_str()) == Some(STORAGE_FORMAT_FILE) {
staging_parent.join(STORAGE_FORMAT_FILE_BACKUP)
} else {
staging_file_path.with_extension("meta.bkp")
};
remove_file_if_exists(&backup_path)?;
Ok(backup_path)
}
#[cfg(any(not(windows), test))]
fn write_local_inline_rollback_backup_from_bytes(
staging_file_path: &Path,
old_metadata: &[u8],
sync: bool,
) -> std::io::Result<PathBuf> {
let backup_path = inline_rollback_backup_path(staging_file_path)?;
let write_res = (|| -> std::io::Result<()> {
#[cfg(test)]
run_owned_file_before_open(&backup_path);
let mut file = std::fs::File::create(&backup_path)?;
std::io::Write::write_all(&mut file, old_metadata)?;
if sync && !old_metadata.is_empty() {
file.sync_data()?;
}
Ok(())
})();
if let Err(err) = write_res {
let _ = remove_file_if_exists(&backup_path);
return Err(err);
}
Ok(backup_path)
}
#[cfg(any(not(windows), test))]
pub(super) fn create_local_inline_rollback_backup(
dst_file_path: &Path,
@@ -215,12 +179,27 @@ pub(super) fn create_local_inline_rollback_backup(
old_metadata: &[u8],
sync: bool,
) -> std::io::Result<PathBuf> {
let backup_path = inline_rollback_backup_path(staging_file_path)?;
let Some(staging_parent) = staging_file_path.parent() else {
return Err(std::io::Error::new(ErrorKind::InvalidInput, "missing staging metadata parent"));
};
let backup_path = staging_parent.join(STORAGE_FORMAT_FILE_BACKUP);
remove_file_if_exists(&backup_path)?;
if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
return write_local_inline_rollback_backup_from_bytes(staging_file_path, old_metadata, sync);
} else if cfg!(test) && sync && !old_metadata.is_empty() {
// Skip redundant fdatasync on hardlinked rollback backups in production;
// keep a Windows-safe sync open in tests.
let write_res = (|| -> std::io::Result<()> {
#[cfg(test)]
run_owned_file_before_open(&backup_path);
let mut file = std::fs::File::create(&backup_path)?;
std::io::Write::write_all(&mut file, old_metadata)?;
if sync && !old_metadata.is_empty() {
file.sync_data()?;
}
Ok(())
})();
if let Err(err) = write_res {
let _ = remove_file_if_exists(&backup_path);
return Err(err);
}
} else if sync && !old_metadata.is_empty() {
os::open_file_for_sync_std(&backup_path)?.sync_data()?;
}
Ok(backup_path)
@@ -679,14 +658,7 @@ impl LocalDisk {
})
.collect()
};
let src_volume_dir = if cfg!(target_os = "linux")
&& src_volume == super::super::RUSTFS_META_TMP_BUCKET
&& skip_access_checks(src_volume)
{
self.io_root().join(src_volume)
} else {
self.io_get_bucket_path(src_volume)?
};
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
if !skip_access_checks(src_volume)
&& let Err(e) = super::super::fs::access_std(&src_volume_dir)
{
@@ -730,53 +702,26 @@ impl LocalDisk {
} else {
STORAGE_FORMAT_FILE
};
let flat_inline_tmp = cfg!(target_os = "linux")
&& !no_inline
&& src_volume == super::super::RUSTFS_META_TMP_BUCKET
&& !preserve_upload_metadata
&& multipart_publication.is_none()
&& pool_meta_condition.is_none()
&& !fi_healing
&& !os::dst_dir_fsync_group_commit_enabled()
&& !os::file_fdatasync_group_commit_enabled()
&& !has_rename_data_test_failpoints(dst_path);
let src_file_path = match multipart_publication.as_ref() {
Some(publication) => publication.canonical_head(),
None if flat_inline_tmp => {
self.io_get_object_open_path(src_volume, format!("{src_path}.{source_metadata_name}").as_str())?
}
None if cfg!(target_os = "linux") => {
self.io_get_object_open_path(src_volume, format!("{src_path}/{source_metadata_name}").as_str())?
}
None => self.io_get_object_path(src_volume, format!("{src_path}/{source_metadata_name}").as_str())?,
};
let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
// Multipart restore completion has validated staged local parts,
// while its public status stays pending until finalization.
let pending_multipart_restore = src_volume == crate::disk::RUSTFS_META_MULTIPART_BUCKET
&& fi
.metadata
.get(rustfs_filemeta::metadata_keys::RESTORE)
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
.is_some_and(|status| status.is_restore_in_progress == Some(true));
if pending_multipart_restore && fi.data_dir.is_none_or(|dir| dir.is_nil()) {
return Err(DiskError::FileCorrupt);
}
// data_dir path (only used by the non-inline commit branch)
let has_data_dir_path = if no_inline && (!fi.is_remote() || pending_multipart_restore) {
let has_data_dir_path = if no_inline && !fi.is_remote() {
if let Some(data_dir) = fi
.data_dir
.map(|dir| rustfs_utils::path::retain_slash(dir.to_string().as_str()))
{
let src_rel = rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str());
let src_data_path = if cfg!(target_os = "linux") && multipart_publication.is_none() && !fi_healing {
self.io_get_object_open_path(src_volume, src_rel.as_str())?
} else {
self.io_get_object_path(src_volume, src_rel.as_str())?
};
let dst_data_path = destination_object_path.join(data_dir.trim_matches('/'));
let src_data_path = self.io_get_object_path(
src_volume,
rustfs_utils::path::retain_slash(format!("{}/{}", src_path, data_dir).as_str()).as_str(),
)?;
let dst_data_path = self.io_get_object_path(
dst_volume,
rustfs_utils::path::retain_slash(format!("{}/{}", dst_path, data_dir).as_str()).as_str(),
)?;
Some((src_data_path, dst_data_path))
} else {
@@ -859,7 +804,7 @@ impl LocalDisk {
let fused_attempt = move |is_cancelled: &dyn Fn() -> bool| -> std::io::Result<FusedCommitAttempt> {
let (guard, has_dst_buf) = prepare_rename_commit_and_destination_metadata_std(
!no_inline && !flat_inline_tmp,
!no_inline,
&src_parent_owned,
&dst_parent_owned,
&dst_file_path_owned,
@@ -931,16 +876,7 @@ impl LocalDisk {
run_owned_file_before_open(&src_file_path_owned);
let mut prepared_metadata_source =
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
let cleanup_flat_tmp = || {
if flat_inline_tmp {
let _ = remove_file_if_exists(&src_file_path_owned);
}
};
if let Err(err) = prepared_metadata_source.write_all(&new_dst_buf, false) {
drop(prepared_metadata_source);
cleanup_flat_tmp();
return Err(err);
}
prepared_metadata_source.write_all(&new_dst_buf, false)?;
if !no_inline {
run_inline_preparation_before_backup(&dst_path_str);
}
@@ -948,21 +884,14 @@ impl LocalDisk {
let mut staged_rollback_path = None;
if (!no_inline || rollback_data_dir.is_none())
&& let Some(ref old_metadata) = has_dst_buf
&& (rollback_data_dir.is_some() || cfg!(test))
&& (rollback_data_dir.is_some() || sync_commit_metadata || cfg!(test))
{
let backup_path = match create_local_inline_rollback_backup(
let backup_path = create_local_inline_rollback_backup(
&dst_file_path_owned,
&src_file_path_owned,
old_metadata,
sync_commit_metadata,
) {
Ok(path) => path,
Err(err) => {
drop(prepared_metadata_source);
cleanup_flat_tmp();
return Err(err);
}
};
)?;
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
}
@@ -991,8 +920,6 @@ impl LocalDisk {
return Err(err);
}
} else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
drop(prepared_metadata_source);
cleanup_flat_tmp();
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
let _ = remove_file_if_exists(backup_path);
}
@@ -1001,7 +928,6 @@ impl LocalDisk {
if is_cancelled() {
drop(prepared_metadata_source);
cleanup_flat_tmp();
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
let _ = remove_file_if_exists(backup_path);
}
@@ -1018,8 +944,6 @@ impl LocalDisk {
) {
Ok(guard) => guard,
Err(err) => {
drop(prepared_metadata_source);
cleanup_flat_tmp();
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
let _ = remove_file_if_exists(backup_path);
}
@@ -1103,8 +1027,6 @@ impl LocalDisk {
None,
sync_commit_metadata,
) {
drop(prepared_metadata_source);
cleanup_flat_tmp();
let _ = remove_file_if_exists(staged_backup);
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
}
@@ -1125,7 +1047,6 @@ impl LocalDisk {
)
};
if let Err(err) = rename_res {
cleanup_flat_tmp();
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
let _ = remove_file_if_exists(backup_path);
}
@@ -1154,38 +1075,8 @@ impl LocalDisk {
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
}
if sync_commit_metadata {
if staged_rollback_path.is_none()
&& rollback_data_dir.is_none()
&& let Some(ref old_metadata) = has_dst_buf
&& (!no_inline || rollback_data_dir.is_none())
{
let fsync_res = (|| -> std::io::Result<()> {
if let Some(dst_parent) = dst_file_path_owned.parent() {
os::fsync_commit_directory_limited_std(dst_parent, &guard)?;
}
Ok(())
})();
if let Err(err) = fsync_res {
let rollback_ok =
write_local_inline_rollback_backup_from_bytes(&src_file_path_owned, old_metadata, true)
.and_then(|backup_path| {
let backup = StagedRollbackBackup(backup_path);
let res =
rollback_inline_metadata_std(&dst_file_path_owned, rollback_data_dir, Some(&backup));
let _ = remove_file_if_exists(backup.path());
res
})
.is_ok();
if no_inline {
if rollback_ok && let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
let _ = remove_dir_all_if_exists(new_data_path);
}
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
}
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
}
} else if let Err(err) = fsync_inline_commit_std(
if sync_commit_metadata
&& let Err(err) = fsync_inline_commit_std(
&dst_file_path_owned,
&dst_volume_dir_owned,
destination_was_absent,
@@ -1196,19 +1087,19 @@ impl LocalDisk {
.filter(|_| no_inline)
.map(|(_, new_data_path)| new_data_path.as_path()),
&guard,
) {
if no_inline {
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
}
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
)
{
if no_inline {
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
}
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
}
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
let _ = remove_file_if_exists(backup_path);
}
if !flat_inline_tmp && let Some(parent) = src_file_path_owned.parent() {
if let Some(parent) = src_file_path_owned.parent() {
let _ = std::fs::remove_dir(parent);
}
@@ -1473,7 +1364,7 @@ impl LocalDisk {
}
}
}
match publication_owned.finish_on_thread(sync_commit_metadata, !cfg!(test), staging_cleanup) {
match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
multipart_source: Some(Box::new(result)),
old_data_dir: has_old_data_dir,
+58 -395
View File
@@ -856,12 +856,6 @@ pub(super) fn inspection_with_markers_bounded(
let prepared_digest = prepared.as_deref().map(digest);
let target_digest = target.as_deref().map(digest);
let receipt_digest = receipt_bytes.as_deref().map(digest);
let finished = finished
|| (committed
&& prepared.is_none()
&& before.is_none()
&& target.is_none()
&& current_digest == journal.as_ref().and_then(|j| j.target_digest));
if let Some(journal) = &journal {
if (!finished
&& journal.target_digest != target_digest
@@ -1119,34 +1113,13 @@ pub(super) fn inspection_with_markers_bounded(
Ok((inspection, has_physical_data_dirs_or_markers))
}
#[derive(Clone)]
struct PreparedDeleteReceiptEntry {
object_path: PathBuf,
bucket_path: PathBuf,
file_path: PathBuf,
staging_root: PathBuf,
journal_raw: Vec<u8>,
journal: DeleteRecoveryJournal,
sole_transaction: bool,
}
#[derive(Clone)]
struct PublishedDeleteReceiptEntry {
object_path: PathBuf,
bucket_path: PathBuf,
file_path: PathBuf,
staging_root: PathBuf,
journal_raw: Vec<u8>,
}
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
static PUBLISHED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PublishedDeleteReceiptEntry>> =
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
}
@@ -1155,27 +1128,6 @@ fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceipt
PREPARED_DELETE_RECEIPTS.remove(rollback)
}
fn put_published_delete_receipt(rollback: impl AsRef<Path>, entry: PublishedDeleteReceiptEntry) {
PUBLISHED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
}
fn take_published_delete_receipt(rollback: &Path) -> Option<PublishedDeleteReceiptEntry> {
PUBLISHED_DELETE_RECEIPTS.remove(rollback)
}
fn format_commit_receipt_bytes(journal_raw: &[u8], finished: bool) -> Vec<u8> {
let suffix: &[u8] = if finished {
b",\"finished\":true}"
} else {
b",\"finished\":false}"
};
let mut out = Vec::with_capacity(11 + journal_raw.len() + suffix.len());
out.extend_from_slice(b"{\"journal\":");
out.extend_from_slice(journal_raw);
out.extend_from_slice(suffix);
out
}
impl LocalDisk {
// Native MPU source closure has its own conditional protocol. Generic
// object recovery must never restore its head or acquire SourceRW under
@@ -2424,7 +2376,6 @@ impl LocalDisk {
barrier(volume, object, transaction, stage, &self.root).await
}
#[cfg(test)]
pub(super) async fn try_fused_recoverable_delete_version(
&self,
volume: &str,
@@ -2433,21 +2384,6 @@ impl LocalDisk {
opts: &DeleteOptions,
transaction: Uuid,
owner: Option<Arc<dyn Send + Sync>>,
) -> Result<Option<()>> {
self.try_fused_recoverable_delete_version_with_paths(volume, object, fi, opts, transaction, owner, None)
.await
}
#[expect(clippy::too_many_arguments)]
pub(super) async fn try_fused_recoverable_delete_version_with_paths(
&self,
volume: &str,
object: &str,
fi: &FileInfo,
opts: &DeleteOptions,
transaction: Uuid,
owner: Option<Arc<dyn Send + Sync>>,
prevalidated_paths: Option<(PathBuf, PathBuf, PathBuf)>,
) -> Result<Option<()>> {
if !opts.recoverable_delete
|| opts.undo_write
@@ -2460,22 +2396,11 @@ impl LocalDisk {
self.reject_multipart_delete_recovery_mutation(volume, object)?;
self.wait_for_startup_cleanup().await;
let (object_path, bucket_path, file_path) = match prevalidated_paths {
Some(paths) => paths,
None => {
let obj = self.get_object_path(volume, object)?;
let (b, f) = self.io_get_bucket_and_object_path(volume, object)?;
check_path_length(f.to_string_lossy().as_ref())?;
(obj, b, f)
}
};
let (_, file_path) = self.io_get_bucket_and_object_path(volume, object)?;
check_path_length(file_path.to_string_lossy().as_ref())?;
let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]);
let destination = file_path.join(transaction.to_string());
let staging_root = if cfg!(target_os = "linux") {
self.io_root().join(RUSTFS_META_TMP_BUCKET)
} else {
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?
};
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
let sync = effective_durability(volume).syncs_commit_metadata();
let incarnation = opts.delete_recovery_bucket_incarnation;
let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
@@ -2653,7 +2578,6 @@ impl LocalDisk {
Err(err) => return Err(err),
}
let staged_dir = staged.keep();
os::record_native_publication(&destination);
if let Err(err) = std::fs::rename(&staged_dir, &destination) {
let _ = std::fs::remove_dir_all(&staged_dir);
return Err(err);
@@ -2663,8 +2587,6 @@ impl LocalDisk {
let _ = std::fs::remove_dir_all(&destination);
return Err(err);
}
#[cfg(all(test, unix))]
fused_prepare_checkpoint::run(&file_path);
let mut reserved_data_path = None;
if let Some(old_path) = old_data_path {
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
@@ -2684,11 +2606,9 @@ impl LocalDisk {
reserved_data_path = Some((old_path, reserved_marker));
}
if let Some(ref target_bytes) = target {
// A provisional head must stay outside the published recovery transaction.
let target_in_rollback = destination.join(TARGET);
let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
let _ = std::fs::remove_file(&linked_xl);
os::record_native_publication(&xl_path);
let write_res = if std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok() {
let rename_res = std::fs::rename(&linked_xl, &xl_path);
if rename_res.is_err() {
@@ -2733,18 +2653,7 @@ impl LocalDisk {
Err(err) => return Err(err),
}
}
put_prepared_delete_receipt(
destination,
PreparedDeleteReceiptEntry {
object_path,
bucket_path,
file_path,
staging_root,
journal_raw,
journal,
sole_transaction,
},
);
put_prepared_delete_receipt(destination, (journal, sole_transaction));
Ok(Ok(Some(old_dir)))
},
)
@@ -2781,45 +2690,19 @@ impl LocalDisk {
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
if transaction.is_nil() {
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if !safe_path(object, false) || transaction.is_nil() {
return Err(DiskError::InvalidPath);
}
let rollback_open_path = if volume != RUSTFS_META_MULTIPART_BUCKET {
self.io_get_object_open_path(volume, object)
.ok()
.map(|p| p.join(transaction.to_string()))
} else {
None
};
let cached_prepared = rollback_open_path.as_deref().and_then(take_prepared_delete_receipt);
let (object_path, bucket_path, root, staging_root, cached_receipt) = match cached_prepared {
Some(entry) => (
entry.object_path,
Some(entry.bucket_path),
entry.file_path,
entry.staging_root,
Some((entry.journal_raw, entry.journal, entry.sole_transaction)),
),
None => {
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if !safe_path(object, false) {
return Err(DiskError::InvalidPath);
}
let obj_path = self.get_object_path(volume, object)?;
(
obj_path,
None,
self.io_get_object_path(volume, object)?,
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?,
None,
)
}
};
let object_path = self.get_object_path(volume, object)?;
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
let root = self.io_get_object_path(volume, object)?;
let rollback = root.join(transaction.to_string());
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
let sync = effective_durability(volume).syncs_commit_metadata();
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
let volume_str = volume.to_owned();
@@ -2829,39 +2712,23 @@ impl LocalDisk {
lease,
sync,
move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
if let Some((journal_raw, journal, sole_transaction)) = cached_receipt
if let Some((journal, sole_transaction)) = take_prepared_delete_receipt(&rollback)
&& rollback.join(PREPARED).is_file()
{
let tmp_receipt = staging_root.join(format!("receipt-{transaction}.tmp"));
let receipt_bytes = format_commit_receipt_bytes(&journal_raw, false);
let write_res = (|| -> std::io::Result<()> {
let mut file = std::fs::File::create(&tmp_receipt)?;
std::io::Write::write_all(&mut file, &receipt_bytes)?;
if sync && !receipt_bytes.is_empty() {
file.sync_data()?;
}
os::record_native_publication(&receipt_path);
std::fs::rename(&tmp_receipt, &receipt_path)?;
if sync {
os::fsync_dir_std(&rollback)?;
}
Ok(())
})();
if let Err(err) = write_res {
let _ = std::fs::remove_file(&tmp_receipt);
return Err(err);
let receipt = CommitReceipt {
journal: journal.clone(),
finished: false,
};
let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
std::fs::create_dir_all(&staging_root)?;
let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
if sync && !receipt_bytes.is_empty() {
staged.as_file().sync_data()?;
}
if let Some(bucket_path) = bucket_path {
put_published_delete_receipt(
rollback,
PublishedDeleteReceiptEntry {
object_path,
bucket_path,
file_path: root,
staging_root,
journal_raw,
},
);
std::fs::rename(staged.path(), &receipt_path)?;
if sync {
os::fsync_dir_std(&rollback)?;
}
return Ok(Ok(Some((journal, sole_transaction, true))));
}
@@ -2937,7 +2804,6 @@ impl LocalDisk {
if sync && !receipt_bytes.is_empty() {
staged.as_file().sync_data()?;
}
os::record_native_publication(&receipt_path);
std::fs::rename(staged.path(), &receipt_path)?;
if sync {
os::fsync_dir_std(&rollback)?;
@@ -2985,46 +2851,14 @@ impl LocalDisk {
Err(err) => Err(err),
}
}
if transaction.is_nil() {
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if !safe_path(object, false) || transaction.is_nil() {
return Err(DiskError::InvalidPath);
}
let rollback_open_path = if volume != RUSTFS_META_MULTIPART_BUCKET {
self.io_get_object_open_path(volume, object)
.ok()
.map(|p| p.join(transaction.to_string()))
} else {
None
};
let cached_published = rollback_open_path.as_deref().and_then(take_published_delete_receipt);
let (object_path, bucket_path, staging_root, root, cached_journal_raw) = match cached_published {
Some(entry) => (
entry.object_path,
entry.bucket_path,
entry.staging_root,
entry.file_path,
Some(entry.journal_raw),
),
None => {
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if !safe_path(object, false) {
return Err(DiskError::InvalidPath);
}
let obj_path = self.get_object_path(volume, object)?;
(
obj_path,
self.io_get_bucket_path(volume)?,
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?,
self.io_get_object_path(volume, object)?,
None,
)
}
};
let object_path = self.get_object_path(volume, object)?;
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
let rollback = root.join(transaction.to_string());
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
let sync = effective_durability(volume).syncs_commit_metadata();
let mut claims: Vec<SnapshotLeaseDeleteClaim> = Vec::with_capacity(journal.data_dirs.len());
for data_dir in &journal.data_dirs {
@@ -3046,9 +2880,15 @@ impl LocalDisk {
claims.push(claim);
}
let bucket_path = self.io_get_bucket_path(volume)?;
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
let root = self.io_get_object_path(volume, object)?;
let rollback = root.join(transaction.to_string());
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
let sync = effective_durability(volume).syncs_commit_metadata();
let data_dirs = journal.data_dirs.clone();
let has_target = journal.target_digest.is_some();
let journal_for_receipt = cached_journal_raw.is_none().then(|| journal.clone());
let journal_for_receipt = journal.clone();
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
let prune_paths: Vec<_> = rollback
.ancestors()
@@ -3057,32 +2897,6 @@ impl LocalDisk {
.collect();
let res = os::run_blocking_namespace_operation_with_sync(lease, sync, move || -> std::io::Result<()> {
let persist_receipt = |finished: bool| -> std::io::Result<()> {
let receipt_bytes = match &cached_journal_raw {
Some(raw) => format_commit_receipt_bytes(raw, finished),
None => {
let receipt = CommitReceipt {
journal: journal_for_receipt
.clone()
.ok_or_else(|| std::io::Error::other("missing journal for receipt"))?,
finished,
};
serde_json::to_vec(&receipt).map_err(std::io::Error::other)?
}
};
std::fs::create_dir_all(&staging_root)?;
let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
std::io::Write::write_all(staged_receipt.as_file_mut(), &receipt_bytes)?;
if sync && !receipt_bytes.is_empty() {
staged_receipt.as_file().sync_data()?;
}
os::record_native_publication(&receipt_path);
std::fs::rename(staged_receipt.path(), &receipt_path)?;
if sync {
let _ = os::fsync_dir_std(&rollback);
}
Ok(())
};
let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
@@ -3137,38 +2951,38 @@ impl LocalDisk {
os::fsync_dir_std(&root)?;
}
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
if let Err(err) = ignore_not_found(std::fs::remove_file(rollback.join(name))) {
if has_target || had_physical_data_dirs {
let _ = persist_receipt(true);
}
return Err(err);
ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
}
if has_target || had_physical_data_dirs {
let finished_receipt = CommitReceipt {
journal: journal_for_receipt,
finished: true,
};
let receipt_bytes = serde_json::to_vec(&finished_receipt).map_err(std::io::Error::other)?;
std::fs::create_dir_all(&staging_root)?;
let mut staged_receipt = tempfile::NamedTempFile::new_in(&staging_root)?;
std::io::Write::write_all(staged_receipt.as_file_mut(), &receipt_bytes)?;
if sync && !receipt_bytes.is_empty() {
staged_receipt.as_file().sync_data()?;
}
std::fs::rename(staged_receipt.path(), &receipt_path)?;
let _ = std::fs::File::create(rollback.join(FINISHED))?;
if sync {
os::fsync_dir_std(&rollback)?;
}
}
for name in [PREPARED, TARGET, FINISHED] {
for name in [TARGET, PREPARED, FINISHED] {
let path = rollback.join(name);
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
if let Err(err) = ignore_not_found(std::fs::remove_file(path)) {
if has_target || had_physical_data_dirs {
let _ = persist_receipt(true);
}
return Err(err);
}
ignore_not_found(std::fs::remove_file(path))?;
}
if sync && let Err(err) = os::fsync_dir_std(&rollback) {
if has_target || had_physical_data_dirs {
let _ = persist_receipt(true);
}
return Err(err);
if sync {
os::fsync_dir_std(&rollback)?;
}
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
if let Err(err) = ignore_not_found(std::fs::remove_file(&receipt_path)) {
if has_target || had_physical_data_dirs {
let _ = persist_receipt(true);
}
return Err(err);
}
ignore_not_found(std::fs::remove_file(&receipt_path))?;
for dir_path in &prune_paths {
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, dir_path);
@@ -3291,42 +3105,6 @@ pub(super) async fn restore_barrier(volume: &str, object_dir: &Path, transaction
Ok(())
}
#[cfg(all(test, unix))]
mod fused_prepare_checkpoint {
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{LazyLock, Mutex};
type Hook = Box<dyn FnOnce() + Send>;
static HOOKS: LazyLock<Mutex<HashMap<PathBuf, Hook>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
pub(super) struct Guard(PathBuf);
impl Drop for Guard {
fn drop(&mut self) {
HOOKS.lock().expect("checkpoint hooks").remove(&self.0);
}
}
pub(super) fn install(root: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
assert!(
HOOKS
.lock()
.expect("checkpoint hooks")
.insert(root.to_path_buf(), Box::new(hook))
.is_none()
);
Guard(root.to_path_buf())
}
pub(super) fn run(root: &Path) {
let hook = HOOKS.lock().expect("checkpoint hooks").remove(root);
if let Some(hook) = hook {
hook();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -7242,121 +7020,6 @@ mod tests {
marker.deleted = false;
}
#[cfg(unix)]
#[tokio::test]
async fn delete_recovery_fused_prepare_checkpoint_reopens_before_head_publication() {
use std::os::unix::fs::MetadataExt;
let _mode = durability_mode_override::set(DurabilityMode::Strict);
let (_temp, disk, mut first, _) = fixture().await;
first.version_id = Some(Uuid::new_v4());
let mut second = first.clone();
second.version_id = Some(Uuid::new_v4());
let mut meta = FileMeta::new();
meta.add_version(first).expect("add surviving version");
meta.add_version(second.clone()).expect("add deleted version");
let original = meta.marshal_msg().expect("encode original versions");
let (_, root) = disk.io_get_bucket_and_object_path("recovery", "object").expect("object path");
fs::write(root.join(STORAGE_FORMAT_FILE), &original)
.await
.expect("write versions");
let (checkpoint_temp, checkpoint_disk, _, _) = fixture().await;
let checkpoint_root = checkpoint_temp.path().join("recovery/object");
drop(checkpoint_disk);
let transaction = Uuid::new_v4();
let source_root = root.clone();
let copy_root = checkpoint_root.clone();
let _capture = fused_prepare_checkpoint::install(&root, move || {
let source = source_root.join(transaction.to_string());
let destination = copy_root.join(transaction.to_string());
std::fs::create_dir(&destination).expect("create checkpoint transaction");
let source_head = source_root.join(STORAGE_FORMAT_FILE);
let source_head_meta = std::fs::metadata(&source_head).expect("stat source head");
let copy_head = copy_root.join(STORAGE_FORMAT_FILE);
std::fs::copy(&source_head, &copy_head).expect("capture head before publication");
for entry in std::fs::read_dir(source).expect("read published preparation") {
let entry = entry.expect("prepared file");
assert!(entry.file_type().expect("prepared file type").is_file());
let dst = destination.join(entry.file_name());
if entry.file_name() == STORAGE_FORMAT_FILE_BACKUP {
let backup_meta = entry.metadata().expect("stat prepared backup");
assert_eq!(
(backup_meta.dev(), backup_meta.ino()),
(source_head_meta.dev(), source_head_meta.ino()),
"fused preparation must hardlink backup to live head"
);
std::fs::hard_link(&copy_head, &dst).expect("preserve hardlinked backup");
} else {
std::fs::copy(entry.path(), dst).expect("capture prepared file");
}
}
});
let opts = DeleteOptions {
recoverable_delete: true,
..Default::default()
};
assert_eq!(
disk.try_fused_recoverable_delete_version("recovery", "object", &second, &opts, transaction, None)
.await
.expect("finish original preparation"),
Some(())
);
assert!(checkpoint_root.join(transaction.to_string()).join(PREPARED).is_file());
let endpoint =
Endpoint::try_from(checkpoint_temp.path().to_str().expect("checkpoint path")).expect("checkpoint endpoint");
let reopened = LocalDisk::new(&endpoint, false)
.await
.expect("reopen checkpoint as a fresh disk");
let inspection = reopened
.delete_recovery_inspect("recovery", "object", transaction)
.await
.expect("actual preparation checkpoint must remain recoverable");
assert_eq!(inspection.state, DeleteRecoveryState::NewPrepared);
let reopened_tx = reopened
.io_get_object_path("recovery", &format!("object/{transaction}"))
.expect("reopened transaction path");
os::fsync_dir_recorder::set_failure(&reopened_tx, ErrorKind::Other);
assert!(
reopened
.settle_delete_transaction(
"recovery",
"object",
transaction,
inspection.fingerprint,
DeleteRecoveryAction::Rollback
)
.await
.is_err(),
"failed transaction directory sync after backup removal must retain journal"
);
assert!(reopened_tx.join(PREPARED).is_file());
assert!(!reopened_tx.join(STORAGE_FORMAT_FILE_BACKUP).exists());
let retry_inspection = reopened
.delete_recovery_inspect("recovery", "object", transaction)
.await
.expect("inspect interrupted rollback");
assert_eq!(retry_inspection.state, DeleteRecoveryState::NewPrepared);
let settled = reopened
.settle_delete_transaction(
"recovery",
"object",
transaction,
retry_inspection.fingerprint,
DeleteRecoveryAction::Rollback,
)
.await
.expect("roll back interrupted preparation");
assert_eq!(settled.state, DeleteRecoveryState::Absent, "same-inode backup must not survive rollback");
assert!(!checkpoint_root.join(transaction.to_string()).exists());
assert_eq!(
fs::read(checkpoint_root.join(STORAGE_FORMAT_FILE))
.await
.expect("read rolled-back versions"),
original
);
}
#[tokio::test]
async fn delete_recovery_fused_versioned_delete_with_surviving_version_and_interrupted_target_cleanup() {
let (_temp, disk, mut v1, _) = fixture().await;
@@ -372,32 +372,29 @@ impl IndexOwner {
#[cfg(unix)]
{
use rustix::fs::{Mode, OFlags, mkdirat, openat};
let open_dir = |parent: rustix::fd::BorrowedFd<'_>, component: &str| -> io::Result<rustix::fd::OwnedFd> {
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
match openat(parent, component, flags, Mode::empty()) {
Ok(dir) => Ok(dir),
Err(rustix::io::Errno::NOENT) => {
match mkdirat(parent, component, Mode::RWXU | Mode::RWXG | Mode::RWXO) {
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
Err(error) => return Err(error.into()),
}
openat(parent, component, flags, Mode::empty()).map_err(Into::into)
}
Err(error) => Err(error.into()),
let mut directories = vec![self._identity_pin.try_clone()?];
for component in [".rustfs.sys", "multipart-index-v1", &sha(bucket)] {
let parent = directories.last().ok_or_else(|| io::Error::other("index parent missing"))?;
match mkdirat(parent, component, Mode::RWXU | Mode::RWXG | Mode::RWXO) {
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
Err(error) => return Err(error.into()),
}
};
use std::os::fd::AsFd;
let bucket_sha = sha(bucket);
let sys_dir = open_dir(self._identity_pin.as_fd(), ".rustfs.sys")?;
let index_dir = open_dir(sys_dir.as_fd(), "multipart-index-v1")?;
let parent = open_dir(index_dir.as_fd(), &bucket_sha)?;
let directory = openat(
parent,
component,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)?;
directories.push(std::fs::File::from(directory));
}
let parent = directories.last().ok_or_else(|| io::Error::other("index parent missing"))?;
// The index is boot-local: every start rescans the staging area
// before serving it, so a marker lost to a crash is recreated before
// it can be consulted. Creation is enough; no fsync of the marker or
// its ancestors. An existing marker is touched so pruning sees the
// new upload's age, not the previous one's.
match openat(
&parent,
parent,
source_root,
OFlags::CREATE | OFlags::EXCL | OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::RUSR | Mode::WUSR,
@@ -412,7 +409,7 @@ impl IndexOwner {
// the previous upload's age, which only shortens the prune
// grace for this key; index completeness does not depend on it.
if let Ok(marker) =
openat(&parent, source_root, OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::empty())
openat(parent, source_root, OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::empty())
{
let _ = std::fs::File::from(marker).set_modified(std::time::SystemTime::now());
}
@@ -433,18 +430,6 @@ impl IndexOwner {
}
fn ensure_current_root(&self) -> io::Result<()> {
#[cfg(unix)]
let current = {
use std::os::unix::fs::MetadataExt;
let metadata = std::fs::symlink_metadata(&*self.root_path)?;
if !metadata.is_dir() {
return Err(io::Error::other("multipart index physical root is not a directory"));
}
let mut inode = [0; 16];
inode[..8].copy_from_slice(&metadata.ino().to_le_bytes());
(metadata.dev(), inode)
};
#[cfg(not(unix))]
let current = os::PublicationRoot::new(&self.root_path)?.physical_identity()?;
if current != self.physical {
self.disabled.store(true, Ordering::Release);
@@ -1979,28 +1979,25 @@ impl NativeMultipartPublication {
let canonical = self.canonical_head();
let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
if let Some((ref original_bytes, _, _)) = deferred_initial {
if cfg!(test) || !can_mark_presynced {
let current =
load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
let snapshot = current
.snapshot
.as_ref()
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
let current_head = current
.head
.as_ref()
.and_then(SourceHead::raw)
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
if !current.head_matches_state
|| snapshot.phase != MultipartSourcePhase::Active
|| snapshot.capability_sha256.is_some()
|| snapshot.lineage != expected_lineage
|| snapshot.generation != expected_generation
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|| current_head != original_bytes.as_slice()
{
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
}
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
let snapshot = current
.snapshot
.as_ref()
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
let current_head = current
.head
.as_ref()
.and_then(SourceHead::raw)
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
if !current.head_matches_state
|| snapshot.phase != MultipartSourcePhase::Active
|| snapshot.capability_sha256.is_some()
|| snapshot.lineage != expected_lineage
|| snapshot.generation != expected_generation
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|| current_head != original_bytes.as_slice()
{
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
}
} else {
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
@@ -2053,7 +2050,7 @@ impl NativeMultipartPublication {
let _ = std::fs::remove_file(prepared_path);
return Err(err);
}
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root, Some(directory_guard))
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
})()
.map_err(DiskError::from)?;
*self.state.lock() = state;
@@ -2163,7 +2160,7 @@ impl NativeMultipartPublication {
let mut prepared_metadata_source =
os::create_prepared_rename_source_with_commit_guard(&prepared_path, &dst_file_path, &owned_directory_guard)?;
prepared_metadata_source.write_all(&new_bytes, true)?;
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root, Some(&owned_directory_guard))
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, &base, &root)
})
.await
.map_err(DiskError::from)?;
@@ -2210,7 +2207,7 @@ impl NativeMultipartPublication {
let base = self.store.base.clone();
let root = self.store.publication_root.clone();
os::run_blocking_namespace_operation(self.store.lease.clone(), move || {
os::install_multipart_prepared_head(prepared, &canonical, &base, &root, None)
os::install_multipart_prepared_head(prepared, &canonical, &base, &root)
})
.await
.map_err(DiskError::from)
@@ -2267,7 +2264,6 @@ impl NativeMultipartPublication {
pub(super) fn finish_on_thread(
&self,
target_already_synced: bool,
fused_in_place: bool,
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
) -> Result<MultipartPublicationResult> {
let mut state = self.state.lock().clone();
@@ -2297,64 +2293,62 @@ impl NativeMultipartPublication {
let disk_ids = &self.store.disk_ids;
let terminal = terminal_record_path(source, &state);
(|| -> std::io::Result<()> {
if !fused_in_place {
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
let current_state = current
.state
.as_ref()
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
let current_raw = encode(current_state).map_err(IoError::other)?;
if current.head.is_some() || current_raw != expected_state_raw {
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
let current_state = current
.state
.as_ref()
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
let current_raw = encode(current_state).map_err(IoError::other)?;
if current.head.is_some() || current_raw != expected_state_raw {
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
}
if target_already_synced {
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
IoError::other(DiskError::ConditionalDeleteConflict)
} else {
error
}
})?;
let target = os::read_multipart_source_metadata_in_pinned(
&pinned,
&target_dir,
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
base,
root,
)
.map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
IoError::other(DiskError::ConditionalDeleteConflict)
} else {
error
}
})?;
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
}
if target_already_synced {
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
IoError::other(DiskError::ConditionalDeleteConflict)
} else {
error
} else {
os::read_and_sync_multipart_target_metadata(
&target_dir,
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
base,
root,
|target| {
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
}
})?;
let target = os::read_multipart_source_metadata_in_pinned(
&pinned,
&target_dir,
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
base,
root,
)
.map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
IoError::other(DiskError::ConditionalDeleteConflict)
} else {
error
}
})?;
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
Ok(())
},
)
.map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
IoError::other(DiskError::ConditionalDeleteConflict)
} else {
error
}
} else {
os::read_and_sync_multipart_target_metadata(
&target_dir,
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
base,
root,
|target| {
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
}
Ok(())
},
)
.map_err(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
IoError::other(DiskError::ConditionalDeleteConflict)
} else {
error
}
})?;
}
})?;
}
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), !fused_in_place)?;
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
if let Some((io_path, cleaned)) = staging_cleanup {
match std::fs::remove_dir(&io_path) {
Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
@@ -2387,7 +2381,7 @@ impl NativeMultipartPublication {
) -> Result<MultipartPublicationResult> {
let this = Arc::clone(self);
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
this.finish_on_thread(target_already_synced, false, staging_cleanup)
this.finish_on_thread(target_already_synced, staging_cleanup)
.map_err(IoError::other)
})
.await
+2 -170
View File
@@ -732,13 +732,6 @@ impl DiskAPI for Disk {
}
}
async fn bucket_creation_witness(&self, volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
match self {
Disk::Local(local_disk) => local_disk.bucket_creation_witness(volume).await,
Disk::Remote(_) => Ok(None),
}
}
#[tracing::instrument(level = "trace", skip_all)]
async fn make_volume(&self, volume: &str) -> Result<()> {
match self {
@@ -1605,11 +1598,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
// NSScanner
// Volume operations.
/// Unknown or remote backends cannot attest process-local publication history.
async fn bucket_creation_witness(&self, _volume: &str) -> Result<Option<os::NativeBucketCreationWitness>> {
Ok(None)
}
async fn make_volume(&self, volume: &str) -> Result<()>;
async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>;
async fn list_volumes(&self) -> Result<Vec<VolumeInfo>>;
@@ -2234,36 +2222,6 @@ pub struct WalkDirOptions {
// Override the remote stream stall timeout for long background walks.
#[serde(default)]
pub stall_timeout_ms: Option<u64>,
/// Select this leaf below identifier-like directory components. The root is
/// `base_dir`; unrelated object metadata must not be read or returned.
/// Older peers ignore this additive field, so consumers must also filter.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub identifier_tree_leaf: Option<String>,
}
pub(crate) fn is_identifier_tree_component(value: &str) -> bool {
!value.is_empty()
&& value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-'))
}
pub(crate) fn is_identifier_tree_leaf(value: &str) -> bool {
value.len() <= 255
&& value.contains('.')
&& !matches!(value, "." | "..")
&& value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.'))
}
pub(crate) fn identifier_tree_leaf_matches(prefix: &str, leaf: &str, object: &str) -> bool {
let Some(relative) = object.strip_prefix(prefix) else {
return false;
};
let mut components = relative.split('/');
components.next_back() == Some(leaf) && components.all(is_identifier_tree_component)
}
impl WalkDirOptions {
@@ -2465,7 +2423,7 @@ impl DeleteRecoveryJournal {
}
let id = request.version_id.unwrap_or_default();
let Some(original) = originals.get(&id) else {
return !id.is_nil() && !locals.contains_key(&id);
return false;
};
match locals.get(&id) {
Some(local) => local == original,
@@ -2515,10 +2473,7 @@ impl DeleteRecoveryJournal {
};
match (local, remote) {
(Some(local), Some(remote)) => local == remote,
(None, None) => {
(!(request.deleted && request.mark_deleted) && request.version_id.is_some_and(|id| !id.is_nil()))
|| (self.original_digest.is_none() && authority.original_digest.is_none())
}
(None, None) => self.original_digest.is_none() && authority.original_digest.is_none(),
_ => false,
}
})
@@ -3202,7 +3157,6 @@ mod tests {
skip_total_timeout: false,
timeout_ms: Some(10_000),
stall_timeout_ms: Some(20_000),
identifier_tree_leaf: None,
};
assert_eq!(opts.bucket, "test-bucket");
@@ -3237,69 +3191,6 @@ mod tests {
assert!(!decoded.skip_hidden_prefix_check);
}
#[test]
fn identifier_tree_leaf_selection_preserves_nested_identifier_directories() {
let prefix = "catalog/namespaces/";
let leaf = "table-entry.json";
for relative in [
"sales/tables/orders/table-entry.json",
"sales/tables/orders/maintenance/tables/owner/table-entry.json",
"maintenance/tables/tables/table-entry.json",
] {
assert!(identifier_tree_leaf_matches(prefix, leaf, &format!("{prefix}{relative}")));
}
for relative in [
"sales/tables/orders/not-table-entry.json",
"sales/tables/orders/table-entry.json/child",
"sales/report.json/tables/owner/table-entry.json",
"sales//tables/owner/table-entry.json",
"sales/../tables/owner/table-entry.json",
] {
assert!(!identifier_tree_leaf_matches(prefix, leaf, &format!("{prefix}{relative}")));
}
assert!(!identifier_tree_leaf_matches(prefix, leaf, "other/sales/tables/orders/table-entry.json"));
assert!(is_identifier_tree_leaf(leaf));
for invalid in ["", ".", "..", "table-entry", "table/entry.json", "table\\entry.json"] {
assert!(!is_identifier_tree_leaf(invalid), "invalid selector {invalid}");
}
}
#[test]
fn identifier_tree_walk_options_are_additive_on_json_and_messagepack_maps() {
#[derive(serde::Deserialize)]
struct LegacyView {
bucket: String,
base_dir: String,
recursive: bool,
}
let options = WalkDirOptions {
bucket: "catalog".to_string(),
base_dir: "namespaces/".to_string(),
recursive: true,
identifier_tree_leaf: Some("table-entry.json".to_string()),
..Default::default()
};
let json = serde_json::to_vec(&options).expect("encode current HTTP options");
let messagepack = rmp_serde::to_vec_named(&options).expect("encode legacy RPC map");
for old in [
serde_json::from_slice::<LegacyView>(&json).expect("old JSON readers ignore the additive field"),
rmp_serde::from_slice::<LegacyView>(&messagepack).expect("old MessagePack readers ignore the additive field"),
] {
assert_eq!(old.bucket, options.bucket);
assert_eq!(old.base_dir, options.base_dir);
assert!(old.recursive);
}
let mut old = serde_json::to_value(&options).expect("encode legacy options");
old.as_object_mut().expect("options are a map").remove("identifier_tree_leaf");
let from_json: WalkDirOptions = serde_json::from_value(old.clone()).expect("decode old HTTP options");
let from_messagepack: WalkDirOptions =
rmp_serde::from_slice(&rmp_serde::to_vec_named(&old).expect("encode old RPC map")).expect("decode old RPC map");
assert!(from_json.identifier_tree_leaf.is_none());
assert!(from_messagepack.identifier_tree_leaf.is_none());
assert_eq!(from_json.base_dir, options.base_dir);
assert_eq!(from_messagepack.base_dir, options.base_dir);
}
/// Test DeleteOptions structure
#[test]
fn test_delete_options() {
@@ -3996,46 +3887,6 @@ mod delete_recovery_adoption_tests {
};
assert!(journal.matches_adoption_source(&[]), "certified non-null removal can already be absent");
assert!(journal.matches_adoption_source(std::slice::from_ref(&source)));
let missing = Uuid::new_v4();
journal.requests.push(FileInfo {
version_id: Some(missing),
..Default::default()
});
journal.original_digest = Some([3; 32]);
assert!(
journal.matches_prepared_authority(&journal),
"an explicit UUID absent from both prepared sources remains a no-op with original metadata"
);
assert!(journal.matches_adoption_source(&[]), "both non-null versions can remain absent");
assert!(
journal.matches_adoption_source(std::slice::from_ref(&source)),
"a missing non-null version in the batch remains a no-op"
);
let mut unexpected = source.clone();
unexpected.version_id = Some(missing);
let mut foreign_preparation = journal.clone();
foreign_preparation.original_versions.push(unexpected.clone());
assert!(
!journal.matches_prepared_authority(&foreign_preparation)
&& !foreign_preparation.matches_prepared_authority(&journal),
"absence from only one prepared source cannot establish a generation"
);
assert!(
!journal.matches_adoption_source(&[source.clone(), unexpected]),
"a version absent from the authority cannot be deleted locally"
);
for version_id in [None, Some(Uuid::nil())] {
journal.requests.last_mut().expect("missing-version request").version_id = version_id;
assert!(
!journal.matches_prepared_authority(&journal),
"a missing null version with original metadata cannot establish its generation"
);
assert!(
!journal.matches_adoption_source(std::slice::from_ref(&source)),
"a missing null version cannot establish its generation"
);
}
journal.requests.pop();
let mut foreign = source.clone();
foreign.metadata_digest = [2; 32];
assert!(!journal.matches_adoption_source(std::slice::from_ref(&foreign)));
@@ -4051,25 +3902,6 @@ mod delete_recovery_adoption_tests {
assert!(journal.matches_adoption_source(std::slice::from_ref(&journal.original_versions[0])));
journal.requests.clear();
assert!(!journal.matches_adoption_source(&journal.original_versions));
let mut marker = journal.clone();
marker.requests = vec![FileInfo {
version_id: Some(id),
deleted: true,
mark_deleted: true,
..Default::default()
}];
marker.original_versions.clear();
marker.original_digest = Some([3; 32]);
assert!(
!marker.matches_prepared_authority(&marker),
"a marker with original metadata requires a source generation"
);
marker.original_digest = None;
assert!(
marker.matches_prepared_authority(&marker),
"marker preparation without original metadata retains the existing allowance"
);
}
}
+5 -683
View File
@@ -21,7 +21,7 @@ use futures::TryStreamExt;
use parking_lot::Mutex;
use rustfs_utils::path::SLASH_SEPARATOR;
use std::{
collections::{HashMap, HashSet, VecDeque},
collections::{HashMap, VecDeque},
io,
path::{Component, Path, PathBuf},
sync::{Arc, LazyLock, Weak},
@@ -350,14 +350,6 @@ pub(crate) mod prepared_publication_test_hooks {
Rollback,
#[cfg(test)]
DirFsync,
#[cfg(test)]
CreateDirectory,
#[cfg(test)]
NativeRootIdentity,
#[cfg(all(test, target_os = "linux"))]
DirectWriteOpen,
#[cfg(all(test, target_os = "linux"))]
BeforeDataUsageLayout,
}
type Hook = Box<dyn FnOnce() + Send>;
@@ -2169,226 +2161,6 @@ fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'stati
})
}
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<ShardedPathLockRegistry<RwLock<()>>> = LazyLock::new(ShardedPathLockRegistry::new);
#[cfg(windows)]
type NativeRootIdentity = (u64, [u8; 16]);
#[cfg(not(windows))]
type NativeRootIdentity = (u64, u64);
// Publication history outlives disk handles, cleanup, and metadata snapshots.
// Reusing a pathname or descriptor for another physical root is not new proof.
#[derive(Debug, Default)]
struct NativePublicationHistory {
roots: HashMap<NativeRootIdentity, NativeRootHistory>,
aliases: HashMap<PathBuf, HashSet<NativeRootIdentity>>,
}
#[derive(Debug, Default)]
struct NativeRootHistory {
unknown: bool,
started: HashSet<String>,
}
static NATIVE_PUBLICATION_HISTORY: LazyLock<Mutex<NativePublicationHistory>> =
LazyLock::new(|| Mutex::new(NativePublicationHistory::default()));
#[derive(Clone, Debug)]
pub(crate) struct NativePublicationRegistration {
identity: NativeRootIdentity,
root: PathBuf,
io_root: PathBuf,
}
/// Opaque, process-local observation. Unsupported disks cannot construct it.
#[derive(Clone, Debug)]
pub struct NativeBucketCreationWitness {
registration: NativePublicationRegistration,
backend: Arc<dyn super::local::LocalIoBackend>,
bucket: String,
disk_id: uuid::Uuid,
}
fn native_root_identity(path: &Path) -> Option<NativeRootIdentity> {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
#[cfg(test)]
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::NativeRootIdentity, path);
let metadata = std::fs::metadata(path).ok()?;
metadata.is_dir().then(|| (metadata.dev(), metadata.ino()))
}
#[cfg(windows)]
{
// Resolve and validate the same directory anchor used for publication;
// keep its volume and complete 128-bit file ID in the history key.
let (_, directory) = open_windows_publication_root(path).ok()?;
windows_file_identity(directory.handle.as_ref()).ok()
}
#[cfg(all(not(unix), not(windows)))]
{
let _ = path;
None
}
}
pub(crate) fn register_native_publication_root(root: &Path, io_root: &Path) -> Option<NativePublicationRegistration> {
let identity = native_root_identity(root)?;
if native_root_identity(io_root) != Some(identity) {
return None;
}
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
history.roots.entry(identity).or_default();
for alias in [root, io_root] {
let alias: PathBuf = alias.components().collect();
let previous = history.aliases.entry(alias).or_default();
let collisions: Vec<_> = previous.iter().copied().filter(|old| *old != identity).collect();
previous.insert(identity);
if !collisions.is_empty() {
history.roots.get_mut(&identity)?.unknown = true;
for old in collisions {
history.roots.get_mut(&old)?.unknown = true;
}
}
}
Some(NativePublicationRegistration {
identity,
root: root.to_path_buf(),
io_root: io_root.to_path_buf(),
})
}
/// Record before queueing filesystem work. Cancellation never removes Started.
pub(crate) fn record_native_publication(path: &Path) {
let aliases: Vec<_> = {
let history = NATIVE_PUBLICATION_HISTORY.lock();
path.ancestors()
.filter(|alias| history.aliases.contains_key(*alias))
.map(Path::to_path_buf)
.collect()
};
// Physical lookups must never hold the process-wide publication mutex.
let observations: Vec<_> = aliases
.into_iter()
.filter_map(|alias| {
let relative = path.strip_prefix(&alias).ok()?;
let mut components = relative.components().filter(|part| !matches!(part, Component::CurDir));
let first = components.next()?;
let bucket = match first {
Component::Normal(bucket) if components.all(|part| matches!(part, Component::Normal(_))) => {
bucket.to_str().map(str::to_owned)
}
_ => None,
};
let physical = native_root_identity(&alias);
Some((alias, physical, bucket))
})
.collect();
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
for (alias, physical, bucket) in observations {
// Aliases only accumulate identities. Include registration concurrent
// with the stat, rather than using a stale captured identity list.
let identities = history.aliases.get(&alias).cloned().unwrap_or_default();
if physical.is_none() || identities.iter().any(|identity| Some(*identity) != physical) {
for identity in identities {
history.roots.entry(identity).or_default().unknown = true;
}
if let Some(identity) = physical {
history.roots.entry(identity).or_default().unknown = true;
}
} else if let Some(identity) = physical {
let root = history.roots.entry(identity).or_default();
match bucket {
Some(bucket) => {
root.started.insert(bucket);
}
None => root.unknown = true,
}
}
}
}
impl NativePublicationRegistration {
pub(crate) fn invalidate(&self) {
if let Some(root) = NATIVE_PUBLICATION_HISTORY.lock().roots.get_mut(&self.identity) {
root.unknown = true;
}
}
pub(crate) fn witness(
&self,
bucket: &str,
disk_id: uuid::Uuid,
backend: Arc<dyn super::local::LocalIoBackend>,
) -> Option<NativeBucketCreationWitness> {
if disk_id.is_nil()
|| !matches!(Path::new(bucket).components().next(), Some(Component::Normal(_)))
|| Path::new(bucket).components().count() != 1
{
return None;
}
let witness = NativeBucketCreationWitness {
registration: self.clone(),
backend,
bucket: bucket.to_owned(),
disk_id,
};
validate_native_creation_witnesses(std::slice::from_ref(&witness)).then_some(witness)
}
}
impl NativeBucketCreationWitness {
pub(crate) fn matches(&self, other: &Self) -> bool {
self.registration.identity == other.registration.identity
&& Arc::ptr_eq(&self.backend, &other.backend)
&& self.disk_id == other.disk_id
&& self.bucket == other.bucket
}
fn physical_root_is_current(&self) -> bool {
let root = native_root_identity(&self.registration.root);
let io_root = native_root_identity(&self.registration.io_root);
if root == Some(self.registration.identity) && io_root == Some(self.registration.identity) {
return true;
}
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
history.roots.entry(self.registration.identity).or_default().unknown = true;
for identity in [root, io_root].into_iter().flatten() {
history.roots.entry(identity).or_default().unknown = true;
}
false
}
fn valid_in(&self, history: &NativePublicationHistory) -> bool {
history
.roots
.get(&self.registration.identity)
.is_some_and(|root| !root.unknown && !root.started.contains(&self.bucket))
}
}
pub(crate) fn validate_native_creation_witnesses(witnesses: &[NativeBucketCreationWitness]) -> bool {
if witnesses.is_empty() || !witnesses.iter().all(NativeBucketCreationWitness::physical_root_is_current) {
return false;
}
let history = NATIVE_PUBLICATION_HISTORY.lock();
witnesses.iter().all(|witness| witness.valid_in(&history))
}
pub(crate) fn consume_native_creation_witnesses(witnesses: &[NativeBucketCreationWitness]) -> bool {
if witnesses.is_empty() || !witnesses.iter().all(NativeBucketCreationWitness::physical_root_is_current) {
return false;
}
let mut history = NATIVE_PUBLICATION_HISTORY.lock();
if !witnesses.iter().all(|witness| witness.valid_in(&history)) {
return false;
}
for witness in witnesses {
if let Some(root) = history.roots.get_mut(&witness.registration.identity) {
root.started.insert(witness.bucket.clone());
}
}
true
}
type NamespaceMutationLock = AsyncMutex<()>;
static DISK_NAMESPACE_MUTATION_LOCKS: LazyLock<ShardedPathLockRegistry<NamespaceMutationLock>> =
LazyLock::new(ShardedPathLockRegistry::new);
@@ -2428,7 +2200,6 @@ pub(crate) const MAX_FSYNC_WORKERS_PER_DISK: usize = 8;
// Return busy workers to the shared pool so other disks can acquire a slot.
const MAX_BLOCKING_JOBS_PER_WORKER: usize = 64;
const WORKER_IDLE_SPIN_ITERS: usize = 256;
const FSYNC_WORKER_IDLE_SPIN_ITERS: usize = 1024;
const FALLBACK_NAMESPACE_QUEUE_SHARDS: usize = 8;
#[derive(Clone, Copy, PartialEq, Eq)]
@@ -2637,7 +2408,7 @@ impl DiskBlockingQueue {
.ready
.store(!state.jobs.is_empty(), std::sync::atomic::Ordering::Relaxed);
if job.is_none() {
if lane != DiskBlockingLane::Fsync && state.idle_workers > 0 {
if state.idle_workers > 0 {
state.active_workers = state.active_workers.saturating_sub(1);
guard.state = DiskBlockingWorkerState::Retired;
return;
@@ -2655,12 +2426,7 @@ impl DiskBlockingQueue {
}
idle_polled = false;
} else {
let spin_iters = if lane == DiskBlockingLane::Fsync {
FSYNC_WORKER_IDLE_SPIN_ITERS
} else {
WORKER_IDLE_SPIN_ITERS
};
for _ in 0..spin_iters {
for _ in 0..WORKER_IDLE_SPIN_ITERS {
if queue.lane(lane).ready.load(std::sync::atomic::Ordering::Relaxed) {
break;
}
@@ -3923,7 +3689,6 @@ pub(in crate::disk) async fn rename_with_namespace_owner(
dst: &Path,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> io::Result<()> {
record_native_publication(dst);
if namespace_owner.is_none() {
return tokio::fs::rename(src, dst).await;
}
@@ -3945,19 +3710,12 @@ pub(in crate::disk) async fn create_dir_all_with_namespace_owner(
path: &Path,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> io::Result<()> {
record_native_publication(path);
if namespace_owner.is_none() {
return tokio::fs::create_dir_all(path).await;
}
let path = path.to_path_buf();
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
run_blocking_namespace_operation(lease, move || {
#[cfg(all(test, not(windows)))]
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::CreateDirectory, &path);
record_native_publication(&path);
std::fs::create_dir_all(path)
})
.await
run_blocking_namespace_operation(lease, move || std::fs::create_dir_all(path)).await
}
#[tracing::instrument(name = "rename_all", level = "debug", skip_all)]
@@ -3968,7 +3726,6 @@ pub(crate) async fn rename_all_with_owner(
publication_root: &PublicationRoot,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
record_native_publication(dst_file_path.as_ref());
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
rename_all_with_lease(src_file_path, dst_file_path, base_dir, publication_root, lease).await
}
@@ -3980,7 +3737,6 @@ pub(crate) async fn rename_all_with_lease(
publication_root: &PublicationRoot,
lease: Arc<NamespaceMutationLease>,
) -> Result<()> {
record_native_publication(dst_file_path.as_ref());
reliable_rename_inner_with_lease(
src_file_path.as_ref().to_path_buf(),
dst_file_path.as_ref().to_path_buf(),
@@ -4081,7 +3837,6 @@ pub(crate) async fn rename_reconciled_metadata(
owner: Option<Arc<dyn Send + Sync>>,
authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
) -> Result<()> {
record_native_publication(&destination);
let lease = acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
run_blocking_namespace_operation_with_sync(lease, true, move || {
let preparation = prepare_rename_with_retry(&source, &destination, &base_dir, &publication_root)?;
@@ -4737,7 +4492,6 @@ pub(crate) async fn rename_all_with_prepared_source(
commit_guard: &RenameCommitGuard,
lease: Arc<NamespaceMutationLease>,
) -> Result<()> {
record_native_publication(dst_file_path.as_ref());
let src_file_path = src_file_path.as_ref().to_path_buf();
let dst_file_path = dst_file_path.as_ref().to_path_buf();
let base_dir = base_dir.as_ref().to_path_buf();
@@ -4784,7 +4538,6 @@ pub(crate) async fn rename_all_with_commit_guard(
commit_guard: &RenameCommitGuard,
lease: Arc<NamespaceMutationLease>,
) -> Result<()> {
record_native_publication(dst_file_path.as_ref());
#[cfg(target_os = "linux")]
{
let source = src_file_path.as_ref().to_path_buf();
@@ -4838,7 +4591,6 @@ pub(crate) async fn rename_all_ignore_missing_source_with_owner(
publication_root: &PublicationRoot,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
record_native_publication(dst_file_path.as_ref());
let src_file_path = src_file_path.as_ref();
let lease = acquire_namespace_mutation_lease_with_owner(dst_file_path.as_ref(), namespace_owner).await;
match reliable_rename_inner_with_lease(
@@ -4893,7 +4645,6 @@ async fn reliable_rename_inner(
publication_root: &PublicationRoot,
warn_on_missing_source: bool,
) -> io::Result<()> {
record_native_publication(dst_file_path.as_ref());
let src_file_path = src_file_path.as_ref().to_path_buf();
let dst_file_path = dst_file_path.as_ref().to_path_buf();
let base_dir = base_dir.as_ref().to_path_buf();
@@ -4917,7 +4668,6 @@ async fn reliable_rename_inner_with_lease(
warn_on_missing_source: bool,
lease: Arc<NamespaceMutationLease>,
) -> io::Result<()> {
record_native_publication(&dst_file_path);
let operation = {
let src_file_path = src_file_path.clone();
let dst_file_path = dst_file_path.clone();
@@ -5410,7 +5160,6 @@ fn rename_into_existing_parent(
parent_guard: Option<&ExistingBaseDirectoryGuard>,
#[cfg(target_os = "linux")] source_parent: &std::fs::File,
) -> io::Result<()> {
record_native_publication(dst_file_path);
use rustix::fs::renameat;
let Some(parent_guard) = parent_guard else {
@@ -5464,7 +5213,6 @@ fn rename_into_existing_parent(
parent_guard: Option<&ExistingBaseDirectoryGuard>,
source: &winapi_util::Handle,
) -> io::Result<()> {
record_native_publication(dst_file_path);
use std::{
mem::size_of,
os::windows::{ffi::OsStrExt, io::AsRawHandle},
@@ -5754,7 +5502,6 @@ impl PublicationRoot {
}
pub(crate) fn create_truncate(&self, path: &Path) -> io::Result<std::fs::File> {
record_native_publication(path);
#[cfg(target_os = "linux")]
{
use rustix::fs::{Mode, OFlags};
@@ -5765,7 +5512,6 @@ impl PublicationRoot {
}
pub(crate) fn create_truncate_with_parent(&self, path: &Path) -> io::Result<(std::fs::File, Option<std::fs::File>)> {
record_native_publication(path);
#[cfg(target_os = "linux")]
{
use rustix::fs::{Mode, OFlags};
@@ -6038,7 +5784,6 @@ impl LinuxCommitDirectories {
}
fn rename(&self, source: &Path, destination: &Path, prepared: Option<&PreparedRenameSource>) -> io::Result<()> {
record_native_publication(destination);
let source_name = self.file_name(source, true)?;
let destination_name = self.file_name(destination, false)?;
let destination_parent = self.prepare_destination()?;
@@ -7698,7 +7443,6 @@ pub(crate) fn install_multipart_prepared_head(
canonical_head: &Path,
base: &Path,
publication_root: &PublicationRoot,
directory_guard: Option<&RenameCommitGuard>,
) -> io::Result<PreparedRenameSource> {
let original_path = prepared.path.clone();
let parent = canonical_head
@@ -7707,37 +7451,9 @@ pub(crate) fn install_multipart_prepared_head(
if original_path.parent() != Some(parent) || original_path == canonical_head {
return Err(io::Error::new(io::ErrorKind::InvalidInput, "MPU proxy must be a private sibling"));
}
#[cfg(target_os = "linux")]
let guard = if let Some(directory_guard) = directory_guard.filter(|g| g.directories.source_path == parent) {
let dir = &directory_guard.directories.source;
RenameCommitGuard {
directories: Arc::new(LinuxCommitDirectories {
root: directory_guard.directories.root.clone(),
source_path: parent.to_path_buf(),
destination_path: parent.to_path_buf(),
source: LinuxCommitDirectory {
file: Arc::clone(&dir.file),
device: dir.device,
inode: dir.inode,
},
destination: std::sync::OnceLock::from(LinuxCommitDirectory {
file: Arc::clone(&dir.file),
device: dir.device,
inode: dir.inode,
}),
destination_base: base.to_path_buf(),
}),
}
} else {
prepare_rename_commit_guard(parent, parent, base, publication_root)?
};
#[cfg(not(target_os = "linux"))]
let _pinned = pin_multipart_source_directory(parent, base, publication_root)?;
#[cfg(not(target_os = "linux"))]
let guard = {
let _ = directory_guard;
prepare_rename_commit_guard(parent, parent, base, publication_root)?
};
let guard = prepare_rename_commit_guard(parent, parent, base, publication_root)?;
rename_prepared_source_std(&prepared, &original_path, canonical_head, base, publication_root, &guard)?;
prepared.path = canonical_head.to_path_buf();
fsync_commit_directory_limited_std(parent, &guard)?;
@@ -8479,7 +8195,6 @@ pub(crate) fn mkdir_all_below_existing_base_std(
base_dir: &Path,
publication_root: &PublicationRoot,
) -> io::Result<ExistingBaseDirectoryGuard> {
record_native_publication(dir_path);
let relative = existing_base_directory_suffix(dir_path, base_dir)?;
#[cfg(unix)]
@@ -10270,77 +9985,6 @@ mod tests {
assert_eq!(err.raw_os_error(), Some(disk_full));
}
#[cfg(windows)]
#[test]
fn windows_native_root_identity_matches_directory_handles() {
let temp = tempdir().expect("create native identity fixture");
let first = temp.path().join("first");
let second = temp.path().join("second");
std::fs::create_dir(&first).expect("create first root");
std::fs::create_dir(&second).expect("create second root");
let (_, first_anchor) = open_windows_publication_root(&first).expect("open first directory anchor");
let (_, second_anchor) = open_windows_publication_root(&second).expect("open second directory anchor");
let first_identity = windows_file_identity(first_anchor.handle.as_ref()).expect("query full first identity");
let second_identity = windows_file_identity(second_anchor.handle.as_ref()).expect("query full second identity");
assert_eq!(native_root_identity(&first), Some(first_identity));
assert_eq!(native_root_identity(&first.join(".")), Some(first_identity));
assert_eq!(native_root_identity(&second), Some(second_identity));
assert_ne!(first_identity, second_identity, "distinct physical roots must have distinct identities");
}
#[cfg(windows)]
#[tokio::test]
async fn windows_native_creation_witness_retains_started_history() {
let temp = tempdir().expect("create native witness fixture");
let root = std::fs::canonicalize(temp.path()).expect("canonical native root");
let _publication_root = PublicationRoot::new(&root).expect("retain native directory anchor");
let registration = register_native_publication_root(&root, &root.join(".")).expect("register same physical root");
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root.clone()));
let disk_id = uuid::Uuid::new_v4();
let witness = registration
.witness("published", disk_id, Arc::clone(&backend))
.expect("fresh native witness");
let bucket = root.join("published");
super::super::fs::mkdir(&bucket)
.await
.expect("publish through the actual native entry");
assert!(bucket.is_dir(), "the production entry must create the directory");
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&witness)));
std::fs::remove_dir(&bucket).expect("remove published directory");
assert!(registration.witness("published", disk_id, Arc::clone(&backend)).is_none());
let reconnected = register_native_publication_root(&root, &root).expect("reconnect same physical root");
assert!(reconnected.witness("published", disk_id, Arc::clone(&backend)).is_none());
let unstarted = reconnected
.witness("unstarted", disk_id, Arc::clone(&backend))
.expect("untouched bucket keeps its witness");
assert!(consume_native_creation_witnesses(std::slice::from_ref(&unstarted)));
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&unstarted)));
assert!(reconnected.witness("unstarted", disk_id, backend).is_none());
}
#[cfg(windows)]
#[test]
fn windows_native_creation_witness_rejects_unknown_roots() {
let temp = tempdir().expect("create unknown identity fixture");
let root = temp.path().join("root");
let other = temp.path().join("other");
let file = temp.path().join("file");
std::fs::create_dir(&root).expect("create native root");
std::fs::create_dir(&other).expect("create distinct native root");
std::fs::write(&file, b"not a directory").expect("create regular file");
assert!(native_root_identity(&temp.path().join("missing")).is_none());
assert!(native_root_identity(&file).is_none());
assert!(register_native_publication_root(&root, &other).is_none());
let registration = register_native_publication_root(&root, &root).expect("register actual native root");
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root));
registration.invalidate();
assert!(registration.witness("unstarted", uuid::Uuid::new_v4(), backend).is_none());
}
#[cfg(windows)]
#[test]
fn windows_rename_all_legacy_file_identity_distinguishes_sources() {
@@ -13390,328 +13034,6 @@ mod tests {
}
}
#[cfg(all(test, unix))]
mod native_creation_history_tests {
use super::*;
struct NativeFixture {
_temp: tempfile::TempDir,
root: PathBuf,
registration: NativePublicationRegistration,
backend: Arc<dyn super::super::local::LocalIoBackend>,
disk_id: uuid::Uuid,
}
impl NativeFixture {
fn new() -> Self {
let temp = tempfile::tempdir().expect("native root");
let root = std::fs::canonicalize(temp.path()).expect("canonical root");
let registration = register_native_publication_root(&root, &root).expect("native registration");
let backend = Arc::new(super::super::local::StdBackend::new(root.clone()));
Self {
_temp: temp,
root,
registration,
backend,
disk_id: uuid::Uuid::new_v4(),
}
}
fn witness(&self, bucket: &str) -> Option<NativeBucketCreationWitness> {
self.registration.witness(bucket, self.disk_id, Arc::clone(&self.backend))
}
}
#[tokio::test]
async fn every_native_directory_publication_entry_invalidates_retry() {
let fixture = NativeFixture::new();
for operation in [
"recursive",
"single",
"open",
"rename",
"rename_std",
"fallback",
"owned_mkdir",
"owned_rename",
"handle_relative",
"root_create",
"root_create_with_parent",
"guarded_rename",
"guarded_prepared_rename",
] {
let bucket = format!("native-{operation}-{}", uuid::Uuid::new_v4());
let path = fixture.root.join(&bucket);
let witness = fixture.witness(&bucket).expect("fresh native witness");
match operation {
"recursive" => super::super::fs::make_dir_all(path.join("object/child"))
.await
.expect("recursive mkdir"),
"single" => super::super::fs::mkdir(&path).await.expect("single mkdir"),
"open" => {
let file = super::super::fs::open_file(&path, super::super::fs::O_CREATE | super::super::fs::O_WRONLY)
.await
.expect("create file");
drop(file);
}
"rename" | "rename_std" | "owned_rename" => {
let source = fixture.root.join(format!("source-{}", uuid::Uuid::new_v4()));
std::fs::create_dir(&source).expect("rename source");
match operation {
"rename" => super::super::fs::rename(&source, &path).await.expect("async rename"),
"rename_std" => super::super::fs::rename_std(&source, &path).expect("sync rename"),
_ => rename_with_namespace_owner(&source, &path, Some(Arc::new(())))
.await
.expect("owned rename"),
}
}
"fallback" => make_dir_all(path.join("object/child"), &path)
.await
.expect("implicit volume fallback"),
"owned_mkdir" => create_dir_all_with_namespace_owner(&path.join("child"), Some(Arc::new(())))
.await
.expect("owned recursive mkdir"),
"handle_relative" => {
let publication_root = PublicationRoot::new(&fixture.root).expect("publication root");
let guard = mkdir_all_below_existing_base_std(&path.join("child"), &fixture.root, &publication_root)
.expect("first-level handle-relative mkdir");
drop(guard);
}
"root_create" | "root_create_with_parent" => {
let publication_root = PublicationRoot::new(&fixture.root).expect("publication root");
if operation == "root_create" {
drop(publication_root.create_truncate(&path).expect("root-relative creation"));
} else {
drop(
publication_root
.create_truncate_with_parent(&path)
.expect("root-relative creation with parent"),
);
}
}
"guarded_rename" | "guarded_prepared_rename" => {
let publication_root = PublicationRoot::new(&fixture.root).expect("publication root");
let source_parent = fixture.root.join(format!("source-{}", uuid::Uuid::new_v4()));
std::fs::create_dir(&source_parent).expect("source directory");
std::fs::create_dir(&path).expect("existing destination directory");
let guard = prepare_rename_commit_guard(&source_parent, &path, &fixture.root, &publication_root)
.expect("existing commit directories");
let source = source_parent.join("xl.meta");
let destination = path.join("xl.meta");
assert!(
validate_native_creation_witnesses(std::slice::from_ref(&witness)),
"preparing existing directories must not mask the publication entry"
);
let lease = acquire_namespace_mutation_lease(&destination).await;
if operation == "guarded_rename" {
std::fs::write(&source, b"metadata").expect("source metadata");
rename_all_with_commit_guard(&source, &destination, &fixture.root, &publication_root, &guard, lease)
.await
.expect("guarded publication");
} else {
let mut staged = create_prepared_rename_source_with_commit_guard(&source, &destination, &guard)
.expect("prepared source");
staged.write_all(b"metadata", false).expect("source metadata");
rename_all_with_prepared_source(
staged,
&source,
&destination,
&fixture.root,
&publication_root,
&guard,
lease,
)
.await
.expect("prepared guarded publication");
}
assert_eq!(std::fs::read(&destination).expect("published metadata"), b"metadata");
}
_ => unreachable!(),
}
assert!(
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
"{operation}: old proof remains live"
);
if path.is_dir() {
std::fs::remove_dir_all(&path).expect("remove published directory");
} else {
std::fs::remove_file(&path).expect("remove published file");
}
assert!(fixture.witness(&bucket).is_none(), "{operation}: removal must not regrant proof");
}
}
#[tokio::test]
async fn cancelled_namespace_waiter_cannot_restore_native_proof() {
let fixture = NativeFixture::new();
let bucket = format!("native-cancel-{}", uuid::Uuid::new_v4());
let path = fixture.root.join(&bucket);
let witness = fixture.witness(&bucket).expect("fresh witness");
let lease = acquire_namespace_mutation_lease(&path).await;
let mut attempt = Box::pin(create_dir_all_with_namespace_owner(&path, Some(Arc::new(()))));
assert!(matches!(futures::poll!(&mut attempt), std::task::Poll::Pending));
assert!(
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
"Started must precede admission wait"
);
assert!(!path.exists(), "the namespace wait must still precede the syscall");
drop(attempt);
drop(lease);
assert!(fixture.witness(&bucket).is_none(), "cancellation cannot undo an admitted attempt");
}
#[tokio::test]
async fn queued_mkdir_cancellation_retains_lease_and_consumed_history() {
use futures::FutureExt;
use std::sync::mpsc;
let fixture = NativeFixture::new();
let bucket = format!("native-queued-cancel-{}", uuid::Uuid::new_v4());
let path = fixture.root.join(&bucket);
let witness = fixture.witness(&bucket).expect("fresh witness");
let (entered_tx, entered_rx) = mpsc::channel();
let (release_tx, release_rx) = mpsc::channel();
let _hook = prepared_publication_test_hooks::install_at(
prepared_publication_test_hooks::Stage::CreateDirectory,
&path,
move || {
entered_tx.send(()).expect("signal real executor entry");
let _ = release_rx.recv();
},
);
let operation_path = path.clone();
let operation =
tokio::spawn(async move { create_dir_all_with_namespace_owner(&operation_path, Some(Arc::new(()))).await });
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(std::time::Duration::from_secs(30)))
.await
.expect("entry waiter")
.expect("the real blocking executor must enter");
operation.abort();
assert!(operation.await.expect_err("cancel the async waiter").is_cancelled());
assert!(
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
"queued publication is permanently Started"
);
assert!(
acquire_namespace_mutation_lease(&path).now_or_never().is_none(),
"the actual blocked executor must still own the namespace lease"
);
release_tx.send(()).expect("release the syscall");
let drained = tokio::time::timeout(std::time::Duration::from_secs(30), acquire_namespace_mutation_lease(&path))
.await
.expect("the lease must drain with the syscall");
drop(drained);
assert!(path.is_dir(), "the cancelled waiter cannot discard the already queued syscall");
std::fs::remove_dir_all(&path).expect("remove the published directory");
assert!(fixture.witness(&bucket).is_none(), "drain and removal cannot restore proof");
}
#[tokio::test]
async fn publication_detects_retargeted_physical_alias_without_registration() {
let source = NativeFixture::new();
let destination = NativeFixture::new();
let bucket = format!("native-retarget-{}", uuid::Uuid::new_v4());
let witness = destination.witness(&bucket).expect("destination witness");
let retired = source.root.with_extension("retired");
std::fs::rename(&source.root, &retired).expect("move the original physical root");
std::os::unix::fs::symlink(&destination.root, &source.root).expect("retarget the registered alias");
super::super::fs::mkdir(source.root.join(&bucket))
.await
.expect("publish through the changed alias");
assert!(
!validate_native_creation_witnesses(std::slice::from_ref(&witness)),
"publication-time alias drift must invalidate the actual destination root"
);
std::fs::remove_dir(destination.root.join(&bucket)).expect("remove the published bucket");
assert!(destination.witness(&bucket).is_none(), "removal must not undo alias uncertainty");
std::fs::remove_file(&source.root).expect("remove the symlink");
std::fs::rename(retired, &source.root).expect("restore the test root");
}
#[test]
fn blocked_physical_identity_lookup_does_not_block_other_publication() {
use std::sync::mpsc;
let stalled = NativeFixture::new();
let healthy = NativeFixture::new();
let witness = stalled.witness("unpublished").expect("stalled root witness");
let (entered_tx, entered_rx) = mpsc::channel();
let (release_tx, release_rx) = mpsc::channel();
let _hook = prepared_publication_test_hooks::install_at(
prepared_publication_test_hooks::Stage::NativeRootIdentity,
&stalled.root,
move || {
entered_tx.send(()).expect("signal physical lookup");
let _ = release_rx.recv();
},
);
let validator = std::thread::spawn(move || validate_native_creation_witnesses(&[witness]));
entered_rx
.recv_timeout(std::time::Duration::from_secs(30))
.expect("enter the actual identity lookup");
let (published_tx, published_rx) = mpsc::channel();
let path = healthy.root.join("published");
let publisher = std::thread::spawn(move || {
record_native_publication(&path);
published_tx.send(()).expect("signal independent publication");
});
let published = published_rx.recv_timeout(std::time::Duration::from_secs(5));
release_tx.send(()).expect("release the identity lookup");
validator.join().expect("identity lookup thread");
publisher.join().expect("publication thread");
assert!(
published.is_ok(),
"a slow physical lookup must not retain the process-wide publication mutex"
);
}
#[test]
fn native_consume_is_atomic_and_history_survives_reconnect() {
let fixture = NativeFixture::new();
let first = format!("native-first-{}", uuid::Uuid::new_v4());
let second = format!("native-second-{}", uuid::Uuid::new_v4());
let first_witness = fixture.witness(&first).expect("first witness");
let second_witness = fixture.witness(&second).expect("second witness");
record_native_publication(&fixture.root.join(&second));
assert!(!consume_native_creation_witnesses(&[first_witness.clone(), second_witness]));
assert!(
validate_native_creation_witnesses(std::slice::from_ref(&first_witness)),
"failed cohort consume must be atomic"
);
assert!(consume_native_creation_witnesses(std::slice::from_ref(&first_witness)));
assert!(!validate_native_creation_witnesses(std::slice::from_ref(&first_witness)));
let reconnected = register_native_publication_root(&fixture.root, &fixture.root).expect("reconnected registration");
let backend: Arc<dyn super::super::local::LocalIoBackend> =
Arc::new(super::super::local::StdBackend::new(fixture.root.clone()));
assert!(
reconnected.witness(&first, fixture.disk_id, backend).is_none(),
"reconnect must retain Started"
);
}
#[test]
fn physical_alias_reuse_is_unknown_and_backend_identity_is_real() {
let temp = tempfile::tempdir().expect("alias parent");
let root = temp.path().join("root");
std::fs::create_dir(&root).expect("first physical root");
let old = register_native_publication_root(&root, &root).expect("first registration");
let backend: Arc<dyn super::super::local::LocalIoBackend> = Arc::new(super::super::local::StdBackend::new(root.clone()));
let id = uuid::Uuid::new_v4();
let witness = old.witness("same-name", id, Arc::clone(&backend)).expect("old witness");
let other_backend: Arc<dyn super::super::local::LocalIoBackend> =
Arc::new(super::super::local::StdBackend::new(root.clone()));
let other = old.witness("same-name", id, other_backend).expect("new backend witness");
assert!(!witness.matches(&other), "a different actual backend cannot reuse a pending observation");
std::fs::rename(&root, temp.path().join("retired-root")).expect("retire physical root");
std::fs::create_dir(&root).expect("replacement root at the same alias");
let replacement = register_native_publication_root(&root, &root).expect("replacement registration");
assert!(replacement.witness("same-name", id, backend).is_none(), "alias reuse must remain Unknown");
assert!(
!validate_native_creation_witnesses(&[witness]),
"old physical identity must not validate at a new root"
);
}
}
#[cfg(all(test, target_os = "linux"))]
mod fd_relative_tests {
use super::*;
+9 -3
View File
@@ -17,8 +17,8 @@
use crate::bucket::error::BucketMetadataError;
use crate::disk::error::DiskError;
use crate::storage_api_contracts::{error::StorageErrorCode, range::HTTPRangeError};
use rustfs_s3_types::S3ErrorCode;
use rustfs_utils::path::decode_dir_object;
use s3s::S3ErrorCode;
pub type Error = StorageError;
pub type Result<T> = core::result::Result<T, Error>;
@@ -1096,8 +1096,14 @@ impl From<rmp_serde::decode::Error> for Error {
}
}
impl From<crate::bucket::utils::BucketConfigXmlError> for Error {
fn from(e: crate::bucket::utils::BucketConfigXmlError) -> Self {
impl From<s3s::xml::SerError> for Error {
fn from(e: s3s::xml::SerError) -> Self {
Error::other(e)
}
}
impl From<s3s::xml::DeError> for Error {
fn from(e: s3s::xml::DeError) -> Self {
Error::other(e)
}
}
+24
View File
@@ -2149,6 +2149,15 @@ mod test {
}
}
/// Runs endpoint parsing cases whose local paths do not exist. Unix leaves
/// missing paths out of the physical-disk independence check, but Windows
/// resolves them to their volume, so paths such as `/d1` and `/d2` count as
/// one shared device there. These cases cover parsing, not disk
/// independence, which has its own tests below.
async fn with_disk_independence_check_bypassed<F: std::future::Future<Output = ()>>(cases: F) {
async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], cases).await;
}
#[test]
fn endpoint_parse_errors_do_not_echo_url_credentials() {
let err = Endpoints::try_from(["http://:topsecret@server/path"].as_slice()).unwrap_err();
@@ -2157,8 +2166,13 @@ mod test {
assert!(!err.to_string().contains("topsecret"));
}
#[serial]
#[tokio::test]
async fn explicit_local_endpoint_host_selects_only_exact_host_and_server_port() {
with_disk_independence_check_bypassed(explicit_local_endpoint_host_selects_only_exact_host_and_server_port_cases()).await;
}
async fn explicit_local_endpoint_host_selects_only_exact_host_and_server_port_cases() {
let args = vec![
"http://rustfs-0.rustfs-headless.ns.svc.cluster.local./data0",
"http://rustfs-0.rustfs-headless.ns.svc.cluster.local.:9443/data1",
@@ -2921,8 +2935,13 @@ mod test {
}
}
#[serial]
#[tokio::test]
async fn test_create_pool_endpoints() {
with_disk_independence_check_bypassed(test_create_pool_endpoints_cases()).await;
}
async fn test_create_pool_endpoints_cases() {
#[derive(Default)]
struct TestCase<'a> {
num: usize,
@@ -3541,8 +3560,13 @@ mod test {
(urls, local_flags)
}
#[serial]
#[tokio::test]
async fn test_create_server_endpoints() {
with_disk_independence_check_bypassed(test_create_server_endpoints_cases()).await;
}
async fn test_create_server_endpoints_cases() {
let test_cases = [
// Invalid input.
("", vec![], false),
@@ -20,6 +20,7 @@
#![allow(clippy::all)]
use http::HeaderMap;
use s3s::dto::ETag;
use std::{collections::HashMap, io::Cursor, sync::Arc};
use tokio::io::BufReader;
@@ -28,7 +29,7 @@ use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::storage_api_contracts::range::HTTPRangeSpec;
use rustfs_filemeta::ObjectPartInfo;
use rustfs_rio::HashReader;
use rustfs_s3_types::S3ErrorCode;
use s3s::S3ErrorCode;
//#[derive(Clone)]
pub struct PutObjReader {
@@ -149,6 +150,24 @@ pub fn new_getobjectreader<'a>(
})
}
/// Convert a raw stored ETag into the strongly-typed `s3s::dto::ETag`.
///
/// Supports already quoted (`"abc"`), weak (`W/"abc"`), or plain (`abc`) values.
pub fn to_s3s_etag(etag: &str) -> ETag {
if let Some(rest) = etag.strip_prefix("W/\"") {
if let Some(body) = rest.strip_suffix('"') {
return ETag::Weak(body.to_string());
}
return ETag::Weak(rest.to_string());
}
if let Some(body) = etag.strip_prefix('"').and_then(|rest| rest.strip_suffix('"')) {
return ETag::Strong(body.to_string());
}
ETag::Strong(etag.to_string())
}
pub fn get_raw_etag(metadata: &HashMap<String, String>) -> String {
metadata
.get("etag")
@@ -213,42 +232,28 @@ mod tests {
}
#[test]
fn test_new_getobjectreader_rejects_a_range_past_the_object_as_invalid_range() {
let oi = multipart_object_info();
let past_the_end = Some(HTTPRangeSpec {
is_suffix_length: false,
start: oi.size,
end: oi.size + 10,
});
fn test_to_s3s_etag() {
// Test unquoted ETag - should become strong etag
assert_eq!(
to_s3s_etag("6af8d12c0c74b78094884349f3c8a079"),
ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
);
let err = new_getobjectreader(&past_the_end, &oi, &ObjectOptions::default(), &HeaderMap::new())
.err()
.expect("a range starting at the object size must be rejected");
assert_eq!(
to_s3s_etag("\"6af8d12c0c74b78094884349f3c8a079\""),
ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
);
assert_eq!(err.code, S3ErrorCode::InvalidRange);
assert_eq!(err.code.as_str(), "InvalidRange");
assert_eq!(err.key, None);
assert_eq!(err.bucket_name, None);
}
assert_eq!(
to_s3s_etag("W/\"6af8d12c0c74b78094884349f3c8a079\""),
ETag::Weak("6af8d12c0c74b78094884349f3c8a079".to_string())
);
#[test]
fn test_new_getobjectreader_rejects_an_unknown_size_without_a_range_as_invalid_range() {
let oi = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: -1,
..Default::default()
};
assert_eq!(to_s3s_etag(""), ETag::Strong(String::new()));
let err = new_getobjectreader(&None, &oi, &ObjectOptions::default(), &HeaderMap::new())
.err()
.expect("an object of unknown size cannot be read without a range");
assert_eq!(to_s3s_etag("\"incomplete"), ETag::Strong("\"incomplete".to_string()));
assert_eq!(err.code, S3ErrorCode::InvalidRange);
assert_eq!(err.code.as_str(), "InvalidRange");
assert_eq!(err.message, "Invalid range");
assert_eq!(err.key.as_deref(), Some("object"));
assert_eq!(err.bucket_name.as_deref(), Some("bucket"));
assert_eq!(to_s3s_etag("incomplete\""), ETag::Strong("incomplete\"".to_string()));
}
#[test]
-4
View File
@@ -1128,10 +1128,6 @@ pub struct ObjectOptions {
/// `ecstore` must leave it unset.
#[doc(hidden)]
pub data_movement_transferred_tier_versions: Option<crate::data_movement::TransferredTierVersions>,
/// Marks the physical copy-back write of a RestoreObject operation. This
/// is in-memory request context and must not be copied into object metadata.
#[doc(hidden)]
pub restore_copyback: bool,
/// Internal, in-memory protection context. Never populated from S3 metadata.
/// None selects the rollout default only for a new write, not for a rewrite.
#[doc(hidden)]
+3 -3
View File
@@ -275,16 +275,16 @@ pub(crate) type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;
///
/// # Returns
/// * None
pub fn set_global_region(region: rustfs_s3_types::Region) {
pub fn set_global_region(region: s3s::region::Region) {
current_ctx().set_region(region);
}
/// Get the global region
///
/// # Returns
/// * `Option<rustfs_s3_types::Region>` - The global region, if set
/// * `Option<s3s::region::Region>` - The global region, if set
///
pub fn get_global_region() -> Option<rustfs_s3_types::Region> {
pub fn get_global_region() -> Option<s3s::region::Region> {
current_ctx().region()
}
+2 -133
View File
@@ -52,13 +52,13 @@ use crate::services::event_notification::EventNotifier;
use crate::services::tier::tier::TierConfigMgr;
use crate::store::delete_recovery_runtime::DeleteRecoveryHints;
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
use rustfs_s3_types::Region;
use s3s::region::Region;
use std::collections::{HashMap, HashSet};
use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
@@ -187,38 +187,6 @@ impl DecommissionCountLedger {
}
}
#[derive(Debug)]
struct ListReadBudget {
permits: Arc<Semaphore>,
capacity: usize,
}
impl ListReadBudget {
fn new(capacity: usize) -> Self {
let capacity = capacity
.min(Semaphore::MAX_PERMITS)
.min(usize::try_from(u32::MAX).unwrap_or(usize::MAX));
Self {
permits: Arc::new(Semaphore::new(capacity)),
capacity,
}
}
async fn acquire(&self, fanout: usize) -> Result<Option<Arc<OwnedSemaphorePermit>>, tokio::sync::AcquireError> {
if self.capacity == 0 {
return Ok(None);
}
// A single walk must remain admissible even when the topology exceeds
// the configured budget. Tokio accepts at most u32::MAX in one grant.
let weight = u32::try_from(fanout.max(1).min(self.capacity)).unwrap_or(u32::MAX);
self.permits
.clone()
.acquire_many_owned(weight)
.await
.map(|permit| Some(Arc::new(permit)))
}
}
/// Runtime state owned by a single `ECStore` instance.
///
/// This is intentionally minimal in the first migration slice; subsequent
@@ -233,7 +201,6 @@ pub struct InstanceContext {
/// drift out of sync). Stored as one value so the three predicates can
/// never observe a torn intermediate state.
erasure_kind: RwLock<SetupType>,
list_read_budget: OnceLock<ListReadBudget>,
/// This instance's namespace lock manager (Phase 5 Slice 3, backlog#939).
///
/// Owned per-instance so two instances no longer share a lock namespace
@@ -387,24 +354,6 @@ impl InstanceContext {
Self::with_lock_manager(Arc::new(GlobalLockManager::new()))
}
/// Bound native listing fanout independently of GET body permits. A walk
/// keeps its shared grant until all of its disk producers have terminated.
/// Acquire before creating the merge channels or taking object locks.
pub(crate) async fn acquire_list_read_budget(
&self,
fanout: usize,
) -> Result<Option<Arc<OwnedSemaphorePermit>>, tokio::sync::AcquireError> {
self.list_read_budget
.get_or_init(|| {
ListReadBudget::new(rustfs_utils::get_env_usize(
rustfs_config::ENV_OBJECT_MAX_CONCURRENT_DISK_READS,
rustfs_config::DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS,
))
})
.acquire(fanout)
.await
}
/// Build a context bound to a specific lock manager.
///
/// [`bootstrap_ctx`] uses this to alias the process-global lock manager so
@@ -413,7 +362,6 @@ impl InstanceContext {
fn with_lock_manager(lock_manager: Arc<GlobalLockManager>) -> Self {
Self {
erasure_kind: RwLock::new(SetupType::Unknown),
list_read_budget: OnceLock::new(),
lock_manager,
region: OnceLock::new(),
deployment_id: OnceLock::new(),
@@ -1163,85 +1111,6 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
mod tests {
use super::*;
#[tokio::test]
async fn list_read_budget_weights_and_clamps_fanout() {
let budget = ListReadBudget::new(8);
let first = budget.acquire(3).await.expect("first walk").expect("bounded grant");
assert_eq!(budget.permits.available_permits(), 5);
let mut blocked = std::pin::pin!(budget.acquire(6));
assert!(futures::poll!(&mut blocked).is_pending());
drop(first);
let second = blocked.await.expect("queued walk").expect("bounded grant");
assert_eq!(budget.permits.available_permits(), 2);
drop(second);
let oversized = budget
.acquire(usize::MAX)
.await
.expect("oversized walk")
.expect("bounded grant");
assert_eq!(budget.permits.available_permits(), 0);
drop(oversized);
assert_eq!(budget.permits.available_permits(), 8);
}
#[tokio::test]
async fn list_read_budget_zero_preserves_unbounded_configuration() {
let budget = ListReadBudget::new(0);
assert!(budget.acquire(usize::MAX).await.expect("disabled budget").is_none());
}
#[tokio::test]
async fn list_read_budget_cancelled_waiter_returns_partial_grant() {
let budget = ListReadBudget::new(4);
let held = budget.acquire(3).await.expect("first walk");
{
let mut waiter = std::pin::pin!(budget.acquire(4));
assert!(futures::poll!(&mut waiter).is_pending());
}
assert_eq!(budget.permits.available_permits(), 1);
drop(held);
assert!(budget.acquire(4).await.expect("next walk").is_some());
}
#[tokio::test]
async fn list_read_budget_shared_grant_survives_parent_drop() {
let budget = ListReadBudget::new(4);
let parent = budget.acquire(4).await.expect("first walk").expect("bounded grant");
let producer = parent.clone();
drop(parent);
assert_eq!(budget.permits.available_permits(), 0);
drop(producer);
assert_eq!(budget.permits.available_permits(), 4);
}
#[tokio::test]
async fn list_read_budget_is_shared_by_context_and_isolated_between_instances() {
let first = Arc::new(InstanceContext::new());
first.list_read_budget.set(ListReadBudget::new(4)).expect("fresh context");
let same = first.clone();
let other = InstanceContext::new();
other.list_read_budget.set(ListReadBudget::new(4)).expect("fresh context");
let held = first.acquire_list_read_budget(4).await.expect("first context");
let mut waiter = std::pin::pin!(same.acquire_list_read_budget(1));
assert!(futures::poll!(&mut waiter).is_pending());
assert!(
other
.acquire_list_read_budget(4)
.await
.expect("independent context")
.is_some()
);
drop(held);
assert!(waiter.await.expect("shared context resumed").is_some());
}
#[test]
fn list_read_budget_extreme_configuration_is_bounded_without_panicking() {
let budget = ListReadBudget::new(usize::MAX);
assert!(budget.capacity <= Semaphore::MAX_PERMITS);
assert!(u32::try_from(budget.capacity).is_ok());
}
#[test]
fn decommission_count_ledger_bounds_retained_entries() {
let mut ledger = super::DecommissionCountLedger::default();
+3 -3
View File
@@ -44,10 +44,10 @@ use crate::{
};
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
use rustfs_config::server_config::{Config, get_global_server_config, set_global_server_config};
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_lock::client::LockClient;
use rustfs_s3_types::Region;
use s3s::dto::BucketLifecycleConfiguration;
use s3s::region::Region;
use tokio::sync::{OwnedRwLockReadGuard, RwLock};
use tokio_util::sync::CancellationToken;
#[cfg(test)]
@@ -371,7 +371,7 @@ pub fn global_tier_config_mgr() -> Arc<RwLock<TierConfigMgr>> {
get_global_tier_config_mgr()
}
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<PersistedLifecycleConfiguration> {
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
GLOBAL_LIFECYCLE_SYS.get(bucket).await
}
@@ -703,28 +703,6 @@ pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<L
.then_some(proof)
}
pub async fn require_restore_copyback_fleet_capability() -> Result<()> {
let Some(topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
#[cfg(any(test, feature = "test-util"))]
return Ok(());
#[cfg(not(any(test, feature = "test-util")))]
return Err(Error::other("restore copy-back capability topology is unavailable"));
};
let Some(notification_sys) = get_global_notification_sys() else {
#[cfg(any(test, feature = "test-util"))]
return Ok(());
#[cfg(not(any(test, feature = "test-util")))]
return Err(Error::other("restore copy-back capability probe is unavailable"));
};
timeout(
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
notification_sys.probe_restore_copyback_fleet(topology),
)
.await
.map_err(|_| Error::other("restore copy-back capability fleet probe timed out"))??;
Ok(())
}
fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
state: &FleetCapabilityProofState,
expected_topology: &str,
@@ -1782,30 +1760,6 @@ impl NotificationSys {
Ok(peer_epochs)
}
async fn probe_restore_copyback_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
if self.peer_clients.len() != self.peer_topology_hosts.len() {
return Err(Error::other("restore copy-back capability fleet membership is incomplete"));
}
let probes = self.peer_clients.iter().map(|client| async {
let client = client
.as_ref()
.ok_or_else(|| Error::other("restore copy-back capability peer is unreachable"))?;
client
.probe_restore_copyback_capability(topology_fingerprint.to_string())
.await
});
let mut peer_epochs = BTreeMap::new();
for result in join_all(probes).await {
let (peer, epoch) = result?;
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
}
let expected = self.peer_topology_hosts.iter().cloned().collect::<BTreeSet<_>>();
if expected.len() != self.peer_topology_hosts.len() || peer_epochs.keys().ne(expected.iter()) {
return Err(Error::other("restore copy-back capability fleet membership does not match topology"));
}
Ok(peer_epochs)
}
async fn probe_transition_transaction_compaction_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
if self.peer_clients.len() != self.peer_topology_hosts.len() {
return Err(Error::other(
@@ -593,7 +593,7 @@ impl ECStore {
#[cfg(test)]
crate::core::pools::observe_pool_activation_start_attempt(crate::core::pools::PoolActivationStartKind::Rebalance);
let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
let mut pool_meta_guard = self.pool_meta_save_gate.write().await;
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
pool_meta_guard.ensure_write_safe(stage)?;
let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?;
let pool_meta = self
@@ -625,7 +625,7 @@ impl ECStore {
{
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin. These
// shared fences admit a peer without waiting for the run's own entries.
let mut pool_meta_guard = self.pool_meta_save_gate.write().await;
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
let mut worker_fence = acquire_pool_rebalance_worker_locks(pool.clone()).await?;
let pool_meta = self
@@ -654,7 +654,7 @@ impl ECStore {
{
// The admission read fences must already be dropped. Reacquire in the
// original order and revalidate every durable boundary across that gap.
let mut pool_meta_guard = self.pool_meta_save_gate.write().await;
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
pool_meta_guard.ensure_write_safe("rebalance worker metadata update")?;
let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
let pool_meta = self
@@ -1714,7 +1714,7 @@ mod tests {
persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true)
.await
.expect("initialized pool metadata identity should persist");
*store.pool_meta_save_gate.write().await = write_state;
*store.pool_meta_save_gate.lock().await = write_state;
// Simulate physical loss; raw deletion of authoritative pool.bin is
// deliberately rejected by the native metadata mutation guard.
for pool in &store.pools {
+18 -19
View File
@@ -1043,9 +1043,8 @@ mod tests {
use crate::storage_api_contracts::object::ObjectIO as _;
use http::HeaderMap;
use rustfs_filemeta::{FileInfo, FileMeta, ObjectPartInfo, TransitionVersionState};
use rustfs_gateway_types::persistence::{
PersistedLifecycleConfiguration, PersistedLifecycleExpiration, PersistedLifecycleRule,
PersistedNoncurrentVersionExpiration,
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, NoncurrentVersionExpiration,
};
use std::time::Duration as StdDuration;
use time::OffsetDateTime;
@@ -1089,12 +1088,12 @@ mod tests {
}
}
fn expired_delete_marker_lifecycle() -> PersistedLifecycleConfiguration {
PersistedLifecycleConfiguration {
fn expired_delete_marker_lifecycle() -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
expiration: Some(PersistedLifecycleExpiration {
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
expired_object_delete_marker: Some(true),
..Default::default()
}),
@@ -2126,11 +2125,11 @@ mod tests {
))
.await
.expect("seed the expired source version");
let lifecycle = PersistedLifecycleConfiguration {
let lifecycle = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
expiration: Some(PersistedLifecycleExpiration {
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
expired_object_all_versions: Some(true),
..Default::default()
@@ -2568,7 +2567,7 @@ mod tests {
versioned,
version_id,
transition: crate::bucket::lifecycle::lifecycle::TransitionOptions {
restore_request: rustfs_lifecycle::RestoreRequest {
restore_request: s3s::dto::RestoreRequest {
days: Some(1),
..Default::default()
},
@@ -2648,7 +2647,7 @@ mod tests {
async fn run_entry_with(
&self,
entry: MetaCacheEntry,
lifecycle_config: Option<PersistedLifecycleConfiguration>,
lifecycle_config: Option<BucketLifecycleConfiguration>,
) -> RebalanceEntryOutcome {
Box::pin(Arc::clone(&self.store).rebalance_entry(
RebalanceEntryTarget {
@@ -2984,17 +2983,17 @@ mod tests {
}
/// A rule that expires every noncurrent version at once.
fn noncurrent_expire_all_lifecycle() -> PersistedLifecycleConfiguration {
PersistedLifecycleConfiguration {
fn noncurrent_expire_all_lifecycle() -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("noncurrent-expire-all".to_string()),
noncurrent_version_expiration: Some(PersistedNoncurrentVersionExpiration {
noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
noncurrent_days: Some(0),
newer_noncurrent_versions: None,
}),
+2 -2
View File
@@ -100,7 +100,7 @@ pub async fn test_store_with_persisted_rebalance_meta(
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
decommission_cancelers: tokio::sync::RwLock::new(vec![None]),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
pool_meta_save_gate: tokio::sync::Mutex::default(),
ctx,
bucket_fence_registry: std::sync::Arc::default(),
});
@@ -335,7 +335,7 @@ async fn test_pool_stores_with_contexts(
rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()),
decommission_cancelers: tokio::sync::RwLock::new(vec![None; pool_count]),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(pool_meta_write_state.independent_clone_for_test()),
pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()),
ctx: store_ctx,
bucket_fence_registry: std::sync::Arc::default(),
})
@@ -67,8 +67,8 @@ use crate::error::{Error, Result};
use crate::storage_api_contracts::range::HTTPRangeSpec;
use rustfs_filemeta::TRANSITION_COMPLETE;
use rustfs_filemeta::{FileInfo, MetaCacheEntry};
use rustfs_gateway_types::persistence::PersistedReplicationConfiguration;
use rustfs_rio::Index;
use s3s::dto::ReplicationConfiguration;
use serde::Serialize;
use std::io::Cursor;
use std::sync::Arc;
@@ -2676,7 +2676,7 @@ fn test_resolve_rebalance_optional_bucket_config_result_passthrough() {
let result = resolve_rebalance_optional_bucket_config_result(
"bucket-a",
"replication",
Ok((PersistedReplicationConfiguration::default(), OffsetDateTime::UNIX_EPOCH)),
Ok((ReplicationConfiguration::default(), OffsetDateTime::UNIX_EPOCH)),
)
.expect("bucket config should pass through");
assert!(result.is_some());
@@ -3412,7 +3412,7 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECSt
rebalance_meta: tokio::sync::RwLock::new(Some(meta)),
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
pool_meta_save_gate: tokio::sync::Mutex::default(),
ctx: crate::runtime::instance::bootstrap_ctx(),
bucket_fence_registry: std::sync::Arc::default(),
})
@@ -34,9 +34,9 @@ pub struct RebalanceStats {
#[derive(Debug, Default)]
pub(super) struct RebalanceBucketConfigs {
pub(super) bucket_incarnation_id: Option<uuid::Uuid>,
pub(super) lifecycle_config: Option<rustfs_gateway_types::persistence::PersistedLifecycleConfiguration>,
pub(super) object_lock_config: Option<rustfs_gateway_types::persistence::PersistedObjectLockConfiguration>,
pub(super) replication_config: Option<(rustfs_gateway_types::persistence::PersistedReplicationConfiguration, OffsetDateTime)>,
pub(super) lifecycle_config: Option<s3s::dto::BucketLifecycleConfiguration>,
pub(super) object_lock_config: Option<s3s::dto::ObjectLockConfiguration>,
pub(super) replication_config: Option<(s3s::dto::ReplicationConfiguration, OffsetDateTime)>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
+43 -94
View File
@@ -79,11 +79,10 @@ use crate::{
store::ECStore,
};
use rustfs_filemeta::FileInfo;
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
use rustfs_rio::HashReader;
use rustfs_s3_client::{admin_handler_utils::AdminError, provider_versions::ProviderVersionCapabilities};
use rustfs_s3_types::S3ErrorCode;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::{S3ErrorCode, dto::BucketLifecycleConfiguration};
use super::{
tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_PERM_ERR},
@@ -1103,7 +1102,7 @@ trait TierReferenceProofStore:
.await
}
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>>;
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<BucketLifecycleConfiguration>>;
async fn walk_bucket_for_deletion_check(
self: Arc<Self>,
@@ -1156,7 +1155,7 @@ impl TierReferenceProofStore for ECStore {
Ok(buckets.into_values().collect())
}
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>> {
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<BucketLifecycleConfiguration>> {
match get_lifecycle_config(bucket).await {
Ok((config, _updated_at)) => Ok(Some(config)),
Err(Error::ConfigNotFound) => Ok(None),
@@ -1635,7 +1634,7 @@ struct TierLifecycleReference<'a> {
}
fn lifecycle_config_target_reference<'a>(
config: &'a PersistedLifecycleConfiguration,
config: &'a BucketLifecycleConfiguration,
targets: &[TierMutationIntentTarget],
) -> Option<TierLifecycleReference<'a>> {
for rule in &config.rules {
@@ -7309,8 +7308,9 @@ mod tests {
endpoints::{Endpoints, PoolEndpoints, SetupType},
};
use crate::services::tier::tier_mutation_intent::TIER_MUTATION_INTENT_RECORD_PREFIX;
use rustfs_gateway_types::persistence::{
PersistedLifecycleConfiguration, PersistedLifecycleRule, PersistedNoncurrentVersionTransition, PersistedTransition,
use s3s::dto::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, NoncurrentVersionTransition, Transition,
TransitionStorageClass,
};
#[test]
@@ -8137,7 +8137,7 @@ mod tests {
#[async_trait::async_trait]
impl TierReferenceProofStore for LockingTierConfigStore {
async fn lifecycle_config_for_reference_proof(&self, _bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>> {
async fn lifecycle_config_for_reference_proof(&self, _bucket: &str) -> Result<Option<BucketLifecycleConfiguration>> {
Ok(None)
}
}
@@ -10870,42 +10870,6 @@ mod tests {
assert_eq!(observed.committed_config_etag.as_deref(), Some("etag-new"));
}
#[tokio::test]
async fn intent_advance_retries_cas_conflict_reload_lock_timeout() {
let store = Arc::new(CasConfigStore::default());
let mutation_id = uuid::Uuid::from_u128(39);
let prepared = prepared_remove_intent("COLD-A", mutation_id);
crate::services::tier::tier_mutation_intent::save_tier_mutation_intent_record(store.clone(), &prepared)
.await
.expect("prepared intent fixture should persist");
let committed = committed_remove_intent("COLD-A", mutation_id, "etag-new");
let committed_bytes = committed.encode().expect("committed intent fixture should encode");
let object = crate::services::tier::tier_mutation_intent::tier_mutation_intent_record_object_name(mutation_id)
.expect("record object should build");
let winning_etag = format!("etag-{}", store.next_etag.load(Ordering::SeqCst) + 1);
store.rewrite_on_next_if_match(object.clone(), committed_bytes.clone()).await;
*store.read_timeout_after_if_match_race.lock().await = Some(object.clone());
let (observed, applied) = crate::services::tier::tier_mutation_intent::advance_tier_mutation_intent_record_idempotent(
store.clone(),
mutation_id,
TierMutationIntentState::Committed,
Some("etag-new".to_string()),
)
.await
.expect("CAS conflict reload should retry its typed lock timeout");
assert!(!applied);
assert_eq!(observed, committed);
assert!(store.read_timeout_after_if_match_race.lock().await.is_none());
assert_eq!(
store.objects.lock().await.get(&object).cloned(),
Some((committed_bytes, winning_etag)),
"retry must preserve the complete winning record and its object ETag"
);
}
#[tokio::test]
async fn intent_advance_reconciles_matching_commit_after_final_cas_race() {
let store = Arc::new(CasConfigStore::default());
@@ -15406,7 +15370,6 @@ mod tests {
objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>,
legacy_state: tokio::sync::Mutex<Option<Vec<u8>>>,
if_match_race_rewrite: tokio::sync::Mutex<std::collections::VecDeque<(String, Vec<u8>)>>,
read_timeout_after_if_match_race: tokio::sync::Mutex<Option<String>>,
next_etag: AtomicUsize,
content_derived_etags: AtomicBool,
fail_put: AtomicBool,
@@ -15425,7 +15388,7 @@ mod tests {
lock_manager: Arc<rustfs_lock::GlobalLockManager>,
lock_requests: Mutex<Vec<(String, String)>>,
listed_versions: Mutex<Vec<ObjectInfo>>,
lifecycle_configs: Mutex<HashMap<String, PersistedLifecycleConfiguration>>,
lifecycle_configs: Mutex<HashMap<String, BucketLifecycleConfiguration>>,
}
impl Default for CasConfigStore {
@@ -15436,7 +15399,6 @@ mod tests {
objects: tokio::sync::Mutex::new(HashMap::new()),
legacy_state: tokio::sync::Mutex::new(None),
if_match_race_rewrite: tokio::sync::Mutex::new(std::collections::VecDeque::new()),
read_timeout_after_if_match_race: tokio::sync::Mutex::new(None),
next_etag: AtomicUsize::new(0),
content_derived_etags: AtomicBool::new(false),
fail_put: AtomicBool::new(false),
@@ -15489,7 +15451,7 @@ mod tests {
.retain(|version| version.bucket != bucket || version.name != object);
}
fn add_lifecycle_config(&self, bucket: &str, config: PersistedLifecycleConfiguration) {
fn add_lifecycle_config(&self, bucket: &str, config: BucketLifecycleConfiguration) {
self.lifecycle_configs
.lock()
.expect("tier reference fixture should not poison")
@@ -15598,19 +15560,6 @@ mod tests {
crate::api::diagnostic::current_read_version_entry_source(),
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
));
{
// The queued rewrite keeps the initial read successful. Only
// the first target read after that CAS race consumes the fault.
let race_completed = self.if_match_race_rewrite.lock().await.is_empty();
let mut timeout = self.read_timeout_after_if_match_race.lock().await;
if race_completed && timeout.as_deref() == Some(object) {
timeout.take();
return Err(Error::Lock(rustfs_lock::LockError::Timeout {
resource: format!("{bucket}/{object}@latest"),
timeout: std::time::Duration::from_secs(5),
}));
}
}
let legacy_config_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
let (data, etag) = if object == legacy_config_path {
let state = self.legacy_state.lock().await;
@@ -16239,7 +16188,7 @@ mod tests {
.await
}
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<PersistedLifecycleConfiguration>> {
async fn lifecycle_config_for_reference_proof(&self, bucket: &str) -> Result<Option<BucketLifecycleConfiguration>> {
if self.reference_lifecycle_error.load(Ordering::SeqCst) {
return Err(Error::CorruptedFormat);
}
@@ -16374,10 +16323,10 @@ mod tests {
});
store.add_lifecycle_config(
"photos",
PersistedLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
@@ -16386,10 +16335,10 @@ mod tests {
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: Some(vec![PersistedTransition {
transitions: Some(vec![Transition {
days: Some(1),
date: None,
storage_class: Some("COLD-A".to_owned()),
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
}]),
}],
},
@@ -17042,10 +16991,10 @@ mod tests {
.save_tiering_config_if_current(store.clone(), None)
.await
.expect("reference proof fixture should persist");
let config = PersistedLifecycleConfiguration {
let config = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
@@ -17054,10 +17003,10 @@ mod tests {
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: Some(vec![PersistedTransition {
transitions: Some(vec![Transition {
days: Some(1),
date: None,
storage_class: Some("COLD-A".to_owned()),
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
}]),
}],
};
@@ -17487,23 +17436,23 @@ mod tests {
});
store.add_lifecycle_config(
"private-bucket",
rustfs_gateway_types::persistence::PersistedLifecycleConfiguration {
BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![rustfs_gateway_types::persistence::PersistedLifecycleRule {
status: "Enabled".to_string(),
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
id: Some("private-rule".to_string()),
noncurrent_version_transitions: noncurrent.then(|| {
vec![rustfs_gateway_types::persistence::PersistedNoncurrentVersionTransition {
vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some("COLD-A".to_string()),
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
}]
}),
transitions: (!noncurrent).then(|| {
vec![rustfs_gateway_types::persistence::PersistedTransition {
vec![Transition {
days: Some(1),
date: None,
storage_class: Some("COLD-A".to_string()),
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
}]
}),
expiration: None,
@@ -17906,10 +17855,10 @@ mod tests {
old_backend_identity: Some(current_identity),
new_backend_identity: None,
};
let config = PersistedLifecycleConfiguration {
let config = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
@@ -17918,10 +17867,10 @@ mod tests {
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: Some(vec![PersistedTransition {
transitions: Some(vec![Transition {
days: Some(1),
date: None,
storage_class: Some("COLD-A".to_owned()),
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
}]),
}],
};
@@ -17951,20 +17900,20 @@ mod tests {
old_backend_identity: Some(current_identity),
new_backend_identity: None,
};
let config = PersistedLifecycleConfiguration {
let config = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
filter: None,
id: Some("move-noncurrent".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: Some(vec![PersistedNoncurrentVersionTransition {
noncurrent_version_transitions: Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some("COLD-A".to_owned()),
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
}]),
prefix: None,
transitions: None,
@@ -18000,10 +17949,10 @@ mod tests {
old_backend_identity: Some(current_identity),
new_backend_identity: None,
};
let config = PersistedLifecycleConfiguration {
let config = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![PersistedLifecycleRule {
status: "Enabled".to_owned(),
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: None,
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
@@ -18012,10 +17961,10 @@ mod tests {
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: Some(vec![PersistedTransition {
transitions: Some(vec![Transition {
days: Some(1),
date: None,
storage_class: Some("COLD-A".to_owned()),
storage_class: Some(TransitionStorageClass::from_static("COLD-A")),
}]),
}],
};
@@ -705,25 +705,9 @@ async fn advance_tier_mutation_intent_record_idempotent_at_prefix<S>(
where
S: EcstoreObjectIO,
{
let mut original_intent: Option<TierMutationIntent> = None;
for attempt in 0..TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS {
let (mut intent, current_etag) =
match load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await {
Ok(current) => current,
Err(Error::Lock(rustfs_lock::LockError::Timeout { .. }))
if attempt + 1 < TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS =>
{
continue;
}
Err(err) => return Err(err),
};
if let Some(original) = &original_intent {
if !intent.same_identity_as(original) {
return Err(Error::PreconditionFailed);
}
} else {
original_intent = Some(intent.clone());
}
load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await?;
let advanced = intent
.advance_idempotent(next, committed_config_etag.clone())
.map_err(tier_mutation_intent_store_error)?;
@@ -734,15 +718,7 @@ where
Ok(()) => return Ok((intent, true)),
Err(Error::PreconditionFailed) => {
let (mut current, _) =
match load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await {
Ok(current) => current,
Err(Error::Lock(rustfs_lock::LockError::Timeout { .. }))
if attempt + 1 < TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS =>
{
continue;
}
Err(err) => return Err(err),
};
load_tier_mutation_intent_record_with_etag_at_prefix(api.clone(), prefix, mutation_id).await?;
if !current.same_identity_as(&intent) {
return Err(Error::PreconditionFailed);
}
@@ -757,11 +733,6 @@ where
}
return Ok((current, false));
}
Err(Error::Lock(rustfs_lock::LockError::Timeout { .. }))
if attempt + 1 < TIER_MUTATION_INTENT_ADVANCE_CAS_ATTEMPTS =>
{
continue;
}
Err(err) => return Err(err),
}
}
@@ -37,9 +37,7 @@ use crate::services::tier::{
use bytes::Bytes;
use http::StatusCode;
use rustfs_filemeta::metadata_keys;
use rustfs_gateway_types::dto::{ObjectLockLegalHoldStatus, ObjectLockMode, ReplicationStatus};
use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
use rustfs_s3_client::remote_error_code::RemoteErrorCode;
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
@@ -52,6 +50,10 @@ use rustfs_utils::egress::validate_outbound_url;
use rustfs_utils::http::headers::{
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
};
use s3s::{
S3ErrorCode,
dto::{ObjectLockLegalHoldStatus, ObjectLockRetentionMode, ReplicationStatus},
};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
@@ -192,15 +194,12 @@ pub trait WarmBackend {
.await
{
Ok(_) => Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string())),
Err(err) if matches!(to_error_response(&err).code, RemoteErrorCode::INVALID_RANGE) => {
Err(err) if matches!(to_error_response(&err).code, S3ErrorCode::InvalidRange) => {
Ok(TransitionCandidateProbe::VersionedPresent(remote_version_id.to_string()))
}
Err(err)
if err.kind() == std::io::ErrorKind::NotFound
|| matches!(
to_error_response(&err).code,
RemoteErrorCode::NO_SUCH_KEY | RemoteErrorCode::NO_SUCH_VERSION
) =>
|| matches!(to_error_response(&err).code, S3ErrorCode::NoSuchKey | S3ErrorCode::NoSuchVersion) =>
{
Ok(TransitionCandidateProbe::Missing)
}
@@ -220,9 +219,9 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
let mut opts = PutObjectOptions {
storage_class,
send_content_md5: true,
legalhold: ObjectLockLegalHoldStatus::custom(""),
legalhold: ObjectLockLegalHoldStatus::from_static(""),
internal: AdvancedPutOptions {
replication_status: ReplicationStatus::custom(""),
replication_status: ReplicationStatus::from_static(""),
..Default::default()
},
..Default::default()
@@ -253,7 +252,7 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
}
if let Some(mode) = metadata.lookup(metadata_keys::OBJECT_LOCK_MODE) {
opts.mode = ObjectLockMode::custom(mode.to_ascii_uppercase());
opts.mode = ObjectLockRetentionMode::from(mode.to_ascii_uppercase());
}
if let Some(retain_until_date) = metadata
@@ -264,7 +263,7 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
}
if let Some(legalhold) = metadata.lookup(metadata_keys::OBJECT_LOCK_LEGAL_HOLD) {
opts.legalhold = ObjectLockLegalHoldStatus::custom(legalhold.to_ascii_uppercase());
opts.legalhold = ObjectLockLegalHoldStatus::from(legalhold.to_ascii_uppercase());
}
// Promoted keys are read above through `lookup`, which accepts more than
@@ -684,7 +683,7 @@ fn is_remote_tier_auth_rejection(err: &std::io::Error) -> bool {
matches!(response.status_code, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN)
|| matches!(
response.code,
RemoteErrorCode::ACCESS_DENIED | RemoteErrorCode::INVALID_ACCESS_KEY_ID | RemoteErrorCode::SIGNATURE_DOES_NOT_MATCH
S3ErrorCode::AccessDenied | S3ErrorCode::InvalidAccessKeyId | S3ErrorCode::SignatureDoesNotMatch
)
}
@@ -1332,7 +1331,7 @@ mod tests {
struct AuthRejectingProbePutBackend {
status_code: StatusCode,
code: RemoteErrorCode,
code: S3ErrorCode,
probes: Arc<AtomicUsize>,
removes: Arc<AtomicUsize>,
}
@@ -1795,10 +1794,10 @@ mod tests {
#[tokio::test]
async fn check_warm_backend_reports_auth_rejected_probe_put_as_invalid_credentials() {
for (status_code, code) in [
(StatusCode::FORBIDDEN, RemoteErrorCode::ACCESS_DENIED),
(StatusCode::UNAUTHORIZED, RemoteErrorCode::custom("")),
(StatusCode::OK, RemoteErrorCode::INVALID_ACCESS_KEY_ID),
(StatusCode::OK, RemoteErrorCode::SIGNATURE_DOES_NOT_MATCH),
(StatusCode::FORBIDDEN, S3ErrorCode::AccessDenied),
(StatusCode::UNAUTHORIZED, S3ErrorCode::Custom("".into())),
(StatusCode::OK, S3ErrorCode::InvalidAccessKeyId),
(StatusCode::OK, S3ErrorCode::SignatureDoesNotMatch),
] {
let probes = Arc::new(AtomicUsize::new(0));
let removes = Arc::new(AtomicUsize::new(0));
@@ -2193,31 +2192,18 @@ mod tests {
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
ObjectLockLegalHoldStatus::ON.to_string(),
);
metadata.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), ObjectLockMode::GOVERNANCE.to_string());
metadata.insert(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
ObjectLockRetentionMode::GOVERNANCE.to_string(),
);
let opts = build_transition_put_options("COLD".to_string(), metadata);
assert_eq!(opts.mode, ObjectLockMode::GOVERNANCE);
assert_eq!(opts.legalhold, ObjectLockLegalHoldStatus::ON);
assert_eq!(opts.mode.as_str(), ObjectLockRetentionMode::GOVERNANCE);
assert_eq!(opts.legalhold.as_str(), ObjectLockLegalHoldStatus::ON);
assert_ne!(opts.retain_until_date, OffsetDateTime::UNIX_EPOCH);
}
#[test]
fn build_transition_put_options_sends_no_object_lock_or_replication_headers_by_default() {
let opts = build_transition_put_options("COLD".to_string(), HashMap::new());
assert_eq!(opts.mode.as_str(), "");
assert_eq!(opts.legalhold.as_str(), "");
assert_eq!(opts.internal.replication_status.as_str(), "");
let header = opts.header();
for name in [
"x-amz-object-lock-mode",
"x-amz-object-lock-legal-hold",
"x-amz-replication-status",
] {
assert!(header.get(name).is_none(), "{name} must not be sent: {header:?}");
}
}
#[test]
fn build_transition_put_options_filters_promoted_headers_from_user_metadata() {
let mut metadata = HashMap::new();
@@ -28,8 +28,6 @@ use crate::services::tier::{
},
};
use http::HeaderMap;
use rustfs_gateway_types::dto::Status as BucketVersioningStatus;
use rustfs_s3_client::remote_error_code::RemoteErrorCode;
use rustfs_s3_client::{
api_error_response::to_error_response,
api_get_options::GetObjectOptions,
@@ -44,6 +42,7 @@ use rustfs_s3_client::{
};
use rustfs_utils::egress::validate_outbound_url_allow_private;
use rustfs_utils::path::SLASH_SEPARATOR;
use s3s::{S3ErrorCode, dto::BucketVersioningStatus};
pub struct WarmBackendS3 {
pub client: Arc<TransitionClient>,
@@ -62,8 +61,8 @@ enum RemoteBucketVersioning {
fn remote_bucket_versioning_from_status(status: Option<&str>) -> Result<RemoteBucketVersioning, std::io::Error> {
Ok(match status {
Some(status) if status == BucketVersioningStatus::ENABLED.as_str() => RemoteBucketVersioning::Enabled,
Some(status) if status == BucketVersioningStatus::SUSPENDED.as_str() => RemoteBucketVersioning::Suspended,
Some(BucketVersioningStatus::ENABLED) => RemoteBucketVersioning::Enabled,
Some(BucketVersioningStatus::SUSPENDED) => RemoteBucketVersioning::Suspended,
Some(status) => {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
@@ -238,7 +237,7 @@ impl WarmBackendS3 {
}
Err(err) => {
let response = to_error_response(&err);
if response.code == RemoteErrorCode::NO_SUCH_KEY {
if response.code == S3ErrorCode::NoSuchKey {
Ok(TransitionCandidateProbe::Missing)
} else {
Err(err)
@@ -325,7 +324,7 @@ impl WarmBackendS3 {
}
Err(err) => {
let response = to_error_response(&err);
if response.code == RemoteErrorCode::NO_SUCH_KEY {
if response.code == S3ErrorCode::NoSuchKey {
Ok(TransitionCandidateProbe::Missing)
} else {
Err(err)
@@ -689,7 +688,7 @@ mod tests {
.probe_transition_candidate("forbidden-probe")
.await
.expect_err("an authorization failure must not be mistaken for a missing key");
assert_eq!(to_error_response(&err).code, RemoteErrorCode::ACCESS_DENIED);
assert_eq!(to_error_response(&err).code, S3ErrorCode::AccessDenied);
assert_eq!(
backend
.probe_transition_candidate("delete-marker-hidden")
@@ -1067,13 +1066,11 @@ mod tests {
RemoteBucketVersioning::Disabled
);
assert_eq!(
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::ENABLED.as_str()))
.expect("enabled status should parse"),
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::ENABLED)).expect("enabled status should parse"),
RemoteBucketVersioning::Enabled
);
assert_eq!(
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::SUSPENDED.as_str()))
.expect("suspended status should parse"),
remote_bucket_versioning_from_status(Some(BucketVersioningStatus::SUSPENDED)).expect("suspended status should parse"),
RemoteBucketVersioning::Suspended
);
let err = remote_bucket_versioning_from_status(Some("UnexpectedStatus"))
@@ -18,6 +18,7 @@ use std::{
sync::atomic::{AtomicBool, Ordering},
};
use s3s::header::{X_AMZ_DELETE_MARKER, X_AMZ_VERSION_ID};
use uuid::Uuid;
use crate::services::tier::{
@@ -127,7 +128,7 @@ impl WarmBackend for WarmBackendWasabi {
self.ensure_unversioned().await?;
}
let (headers, reader) = self.s3.get_with_headers(object, rv, opts).await?;
if rv.is_empty() && (headers.contains_key("x-amz-version-id") || headers.contains_key("x-amz-delete-marker")) {
if rv.is_empty() && (headers.contains_key(X_AMZ_VERSION_ID) || headers.contains_key(X_AMZ_DELETE_MARKER)) {
self.versioning_drifted.store(true, Ordering::Release);
return Err(io::Error::other(
"Wasabi bucket versioning changed while reading an unversioned tier object",
+36 -15
View File
@@ -153,8 +153,8 @@ use rustfs_utils::http::headers::{
};
use rustfs_utils::http::{
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE,
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_COPYBACK, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK,
contains_key_str, get_header_map, get_str, insert_str, is_object_encryption_marker, remove_header_map,
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, contains_key_str, get_header_map,
get_str, insert_str, is_object_encryption_marker, remove_header_map,
};
use rustfs_utils::{
HashAlgorithm,
@@ -14047,7 +14047,10 @@ mod tests {
let requested_until = now + Duration::from_secs(60 * 60 * 24);
let mut user_defined = HashMap::new();
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
user_defined.insert(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
@@ -14059,7 +14062,7 @@ mod tests {
};
let opts = ObjectOptions {
object_lock_retention: Some(crate::storage_api_contracts::object::ObjectLockRetentionOptions {
mode: Some("COMPLIANCE".to_string()),
mode: Some(s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string()),
retain_until: Some(requested_until),
bypass_governance: true,
}),
@@ -14079,7 +14082,10 @@ mod tests {
let requested_until = now + Duration::from_secs(60 * 60 * 24);
let mut user_defined = HashMap::new();
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string());
user_defined.insert(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
);
user_defined.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
existing_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
@@ -14091,7 +14097,7 @@ mod tests {
};
let opts = ObjectOptions {
object_lock_retention: Some(crate::storage_api_contracts::object::ObjectLockRetentionOptions {
mode: Some("GOVERNANCE".to_string()),
mode: Some(s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string()),
retain_until: Some(requested_until),
bypass_governance: true,
}),
@@ -14106,7 +14112,10 @@ mod tests {
async fn test_check_object_lock_delete_blocks_compliance_version_delete() {
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
let mut user_defined = HashMap::new();
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
user_defined.insert(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
@@ -14148,7 +14157,10 @@ mod tests {
},
restore_expires: Some(restore_expiry),
user_defined: Arc::new(HashMap::from([
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
),
(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
@@ -14219,7 +14231,10 @@ mod tests {
async fn test_check_object_lock_delete_allows_versioned_delete_marker_creation() {
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
let mut user_defined = HashMap::new();
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
user_defined.insert(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
@@ -14244,7 +14259,10 @@ mod tests {
fn governance_retained_obj_info() -> ObjectInfo {
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
let mut user_defined = HashMap::new();
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string());
user_defined.insert(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
);
user_defined.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
@@ -14294,7 +14312,10 @@ mod tests {
async fn test_check_object_lock_delete_blocks_replicated_compliance_version_purge() {
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
let mut user_defined = HashMap::new();
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
user_defined.insert(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
@@ -14349,11 +14370,11 @@ mod tests {
#[test]
fn test_object_lock_delete_check_required_keeps_object_lock_config_buckets() {
use rustfs_gateway_types::persistence::PersistedObjectLockConfiguration as ObjectLockConfiguration;
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
let mut bm = crate::bucket::metadata::BucketMetadata::new("lock-config-bucket");
bm.object_lock_config = Some(ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
..Default::default()
});
assert!(object_lock_delete_check_required(Some(&bm)));
@@ -17250,8 +17271,8 @@ mod tests {
// otherwise `from_file_info` never synthesizes the null version id and
// the branch under test is not reached.
let suspended = crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test(
rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
status: Some("Suspended".to_owned()),
s3s::dto::VersioningConfiguration {
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
..Default::default()
},
None,
@@ -3039,86 +3039,6 @@ mod tests {
}
}
#[tokio::test]
async fn delete_recovery_batch_with_missing_version_recovers_after_preparation_failure() {
use crate::disk::RUSTFS_META_TMP_BUCKET;
use crate::storage_api_contracts::object::ObjectToDelete;
let bucket = "recover-batch-missing-version";
let (dirs, store, disks) = fixture(bucket).await;
let deleted = put(&store, bucket, &vec![48; 4096], true).await.expect("version to delete");
let remaining = vec![49; 4096];
let retained = put(&store, bucket, &remaining, true).await.expect("version to retain");
let missing = Uuid::new_v4();
let temporary = dirs[3].path().join(RUSTFS_META_TMP_BUCKET);
let saved = dirs[3].path().join("saved-delete-tmp");
tokio::fs::rename(&temporary, &saved)
.await
.expect("save returning disk's staging directory");
tokio::fs::write(&temporary, b"block preparation")
.await
.expect("fail staging after the recovery preflight");
let (objects, errors) = store
.delete_objects(
bucket,
[deleted, missing]
.into_iter()
.map(|version| ObjectToDelete {
object_name: "object".into(),
version_id: Some(version),
..Default::default()
})
.collect(),
ObjectOptions {
versioned: true,
..Default::default()
},
)
.await;
assert_eq!(objects.len(), 2);
assert!(errors.iter().all(Option::is_none), "batch deletion succeeds at quorum: {errors:?}");
assert!(
disks[3]
.discover_delete_transactions(bucket, Some("object"), None, 100)
.await
.expect("inspect failed preparation")
.transactions
.is_empty(),
"returning disk must adopt the quorum decision without a local preparation"
);
tokio::fs::remove_file(&temporary).await.expect("remove preparation failure");
tokio::fs::rename(&saved, &temporary)
.await
.expect("restore returning disk staging");
let result = store
.recover_delete_transactions(bucket, &HealOpts::default())
.await
.expect("recover batch containing a missing non-null version");
assert!(result.detail.contains("blocked=0"), "{}", result.detail);
assert_eq!(body(&store, bucket, Some(retained)).await, remaining);
for version in [deleted, missing] {
assert_eq!(
disks[3]
.read_version(bucket, bucket, "object", &version.to_string(), &ReadOptions::default())
.await
.expect_err("returning disk must not retain or create a deleted version"),
DiskError::FileVersionNotFound
);
}
for disk in &disks {
assert!(
disk.discover_delete_transactions(bucket, Some("object"), None, 100)
.await
.expect("inspect converged batch")
.transactions
.is_empty(),
"every participant must converge and release the decision"
);
}
}
#[tokio::test]
async fn delete_recovery_dry_run_matches_adoption_of_already_absent_non_null_version() {
let bucket = "recover-preview-non-null";
@@ -3485,140 +3405,6 @@ mod tests {
.expect("no retained version decision may keep the empty bucket nonempty");
}
#[tokio::test]
async fn delete_recovery_partial_batch_receipt_handles_missing_versions() {
use crate::disk::RUSTFS_META_TMP_BUCKET;
use crate::set_disk::ops::object::DeleteCleanupBarrier;
use crate::storage_api_contracts::object::ObjectToDelete;
for include_missing in [false, true] {
let bucket = "recover-prepared-batch-missing-versions";
let (dirs, store, disks) = fixture(bucket).await;
metadata_sys::update_in(
&store.ctx,
bucket,
crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
b"<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
)
.await
.expect("enable supported versioned bucket");
let first = put(&store, bucket, &vec![81; 1024 * 1024], true)
.await
.expect("first version");
let second = put(&store, bucket, &vec![82; 4096], true).await.expect("retained version");
let missing = Uuid::new_v4();
let barrier = DeleteCleanupBarrier::install(bucket);
let deleting = store.clone();
let task = tokio::spawn(async move {
let mut versions = vec![first];
if include_missing {
versions.push(missing);
}
deleting
.delete_objects(
bucket,
versions
.into_iter()
.map(|version| ObjectToDelete {
object_name: "object".into(),
version_id: Some(version),
..Default::default()
})
.collect(),
ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
});
barrier.wait_until_paused().await;
let transactions = disks[0]
.discover_delete_transactions(bucket, Some("object"), None, 100)
.await
.expect("discover batch preparation")
.transactions;
assert_eq!(transactions.len(), 1);
let transaction = transactions[0].transaction;
for disk in &disks {
let state = disk
.inspect_delete_transaction(bucket, "object", transaction)
.await
.expect("inspect prepared batch");
assert_eq!(state.state, DeleteRecoveryState::NewPrepared);
let journal = state.journal.expect("prepared journal");
assert!(journal.original_digest.is_some());
assert!(journal.original_versions.iter().all(|v| v.version_id != Some(missing)));
}
// Fail durable receipt staging on two participants after metadata quorum.
for dir in dirs.iter().skip(2) {
tokio::fs::rename(dir.path().join(RUSTFS_META_TMP_BUCKET), dir.path().join("saved-receipt-staging"))
.await
.expect("save receipt staging directory");
tokio::fs::write(dir.path().join(RUSTFS_META_TMP_BUCKET), b"receipt staging I/O failure")
.await
.expect("inject receipt staging failure");
}
barrier.release();
let (_, errors) = task.await.expect("batch deletion must join");
drop(barrier);
assert_eq!(errors.len(), if include_missing { 2 } else { 1 });
assert!(errors.iter().all(Option::is_some), "partial receipts are indeterminate: {errors:?}");
let mut receipts = 0;
let mut prepared = 0;
for disk in &disks {
match disk
.inspect_delete_transaction(bucket, "object", transaction)
.await
.expect("partial decision")
.state
{
DeleteRecoveryState::QuorumCommitted => receipts += 1,
DeleteRecoveryState::NewPrepared => prepared += 1,
state => panic!("unexpected post-publication state: {state:?}"),
}
}
assert_eq!((receipts, prepared), (2, 2));
for dir in dirs.iter().skip(2) {
tokio::fs::remove_file(dir.path().join(RUSTFS_META_TMP_BUCKET))
.await
.expect("remove injected staging failure");
tokio::fs::rename(dir.path().join("saved-receipt-staging"), dir.path().join(RUSTFS_META_TMP_BUCKET))
.await
.expect("restore receipt staging");
}
store
.delete_object(
bucket,
"object",
ObjectOptions {
versioned: true,
version_id: Some(second.to_string()),
..Default::default()
},
)
.await
.expect("later version deletion succeeds");
store
.recover_delete_transactions(bucket, &HealOpts::default())
.await
.expect("certified batch must recover after later version deletion");
for disk in &disks {
assert!(
disk.discover_delete_transactions(bucket, Some("object"), None, 100)
.await
.expect("settled transaction inventory")
.transactions
.is_empty()
);
}
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("settled receipts must not block removal of the empty bucket");
}
}
#[tokio::test]
async fn delete_recovery_partial_receipt_converges_after_later_version_absence() {
let bucket = "recover-partial-chained-versions";
+11 -25
View File
@@ -38,16 +38,16 @@ use super::super::{
ObjectLockDiagGuard, ObjectOptions, ObjectPartInfo, OffsetDateTime, PartInfo, PutObjReader, RUSTFS_META_MULTIPART_BUCKET,
RUSTFS_META_TMP_BUCKET, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, Result, SLASH_SEPARATOR,
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION_SIZE,
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_COPYBACK, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks,
SmallWritePath, StorageError, Uuid, WriteLayout, check_object_lock_for_deletion_with_state,
classify_multipart_part_write_path, coding, complete_multipart_part_error, complete_multipart_part_error_result,
complete_part_checksum, contains_key_str, create_bitrot_writer_with_orig_volume, debug, disk, error,
file_info_is_valid_for_metadata, get_complete_multipart_md5, get_header_map, get_str, insert_str, is_err_object_not_found,
is_err_version_not_found, is_min_allowed_part_size, log_multipart_write_quorum_failure, multipart_part_logical_size,
parts_after_marker, path_join_buf, prepare_complete_multipart_parts, record_compression_total_memory,
record_multipart_write_quorum_failure, reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map,
resolve_write_layout, restore_commit_operation_id_from_metadata, restore_operation_id_from_metadata,
should_persist_encryption_original_size, strip_internal_multipart_metadata, to_object_err, warn,
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError,
Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, contains_key_str,
create_bitrot_writer_with_orig_volume, debug, disk, error, file_info_is_valid_for_metadata, get_complete_multipart_md5,
get_header_map, get_str, insert_str, is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size,
log_multipart_write_quorum_failure, multipart_part_logical_size, parts_after_marker, path_join_buf,
prepare_complete_multipart_parts, record_compression_total_memory, record_multipart_write_quorum_failure,
reduce_read_quorum_errs, reduce_write_quorum_errs, remove_header_map, resolve_write_layout,
restore_commit_operation_id_from_metadata, restore_operation_id_from_metadata, should_persist_encryption_original_size,
strip_internal_multipart_metadata, to_object_err, warn,
};
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
#[cfg(test)]
@@ -3724,14 +3724,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
crate::object_api::ENCRYPTED_PART_LAYOUT_CANDIDATE_SUFFIX,
);
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX);
// Pending copy-back keeps its recovery identity until finalization.
if expected_restore_operation_id.is_some()
&& !fi
.metadata
.get(metadata_keys::RESTORE)
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
.is_some_and(|status| status.is_restore_in_progress == Some(true))
{
if expected_restore_operation_id.is_some() {
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, SUFFIX_RESTORE_OPERATION_ID);
rustfs_utils::http::metadata_compat::remove_str(&mut fi.metadata, SUFFIX_RESTORE_WORKER_LOCK);
}
@@ -4116,13 +4109,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
fi.metadata.remove(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM);
fi.metadata.remove(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE);
strip_internal_multipart_metadata(&mut fi.metadata);
if opts.restore_copyback {
insert_str(
&mut fi.metadata,
SUFFIX_RESTORE_COPYBACK,
rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1.to_string(),
);
}
// A previous coordinator can stop after a minority publication without
// undo. A fresh final directory keeps this attempt separate from old
+138 -227
View File
@@ -41,24 +41,23 @@ use super::super::{
ObjectKey, ObjectLockConfigSnapshot, ObjectLockConfigState, ObjectOptions, ObjectReader, ObjectToDelete, OffsetDateTime,
Ordering, Pin, PutObjReader, RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, ReadPathPlan, ReaderImpl, ReplicateDecision,
ReplicationObjectBridge, Result, SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS, SLASH_SEPARATOR, SUFFIX_ACTUAL_SIZE,
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_RESTORE_COPYBACK, SUFFIX_RESTORE_OPERATION_ID,
SUFFIX_RESTORE_WORKER_LOCK, SetDisks, SmallWritePath, StorageError, TRANSITION_COMPLETE, UpdateMetadataOpts, Uuid,
WriteLayout, adaptive_duplex_buffer_size, build_get_object_info, build_inline_bitrot_readers,
build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct, check_object_lock_delete,
check_object_lock_for_deletion_with_state, check_object_lock_retention_update, classify_get_codec_streaming_object_class,
classify_put_write_path, classify_storage_error, collect_inline_data_shard_fileinfos_by_index, contains_key_str,
create_bitrot_writer_with_orig_volume, debug, delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error,
explicit_delete_removed_marker, finish_set_disk_read_lock, get_codec_streaming_reader_gate_with_plan,
get_object_body_cache_hook, get_raw_etag, get_small_object_direct_memory_decision_with_threshold_and_plan,
get_small_object_direct_memory_threshold, get_stage_timer_if_enabled, get_str,
get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size, inline_erasure_shard_file_offset,
inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled, is_err_object_not_found, is_err_version_not_found,
is_explicit_null_version, is_get_codec_streaming_base_enabled, is_get_small_object_direct_memory_enabled,
is_lock_optimization_enabled, issue3031_diag_enabled, join_all, known_put_object_storage_size, path_join_buf,
put_restore_opts, record_compression_total_memory, record_get_codec_streaming_gate_decision,
record_get_direct_memory_decision, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path,
record_get_object_reader_path_observation, record_get_stage_duration_if_enabled, record_lock_acquire,
reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate,
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, SUFFIX_RESTORE_OPERATION_ID, SUFFIX_RESTORE_WORKER_LOCK, SetDisks,
SmallWritePath, StorageError, TRANSITION_COMPLETE, UpdateMetadataOpts, Uuid, WriteLayout, adaptive_duplex_buffer_size,
build_get_object_info, build_inline_bitrot_readers, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
check_object_lock_delete, check_object_lock_for_deletion_with_state, check_object_lock_retention_update,
classify_get_codec_streaming_object_class, classify_put_write_path, classify_storage_error,
collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer_with_orig_volume, debug,
delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error, explicit_delete_removed_marker,
finish_set_disk_read_lock, get_codec_streaming_reader_gate_with_plan, get_object_body_cache_hook, get_raw_etag,
get_small_object_direct_memory_decision_with_threshold_and_plan, get_small_object_direct_memory_threshold,
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_admission_shard_size,
inline_erasure_shard_file_offset, inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled,
is_err_object_not_found, is_err_version_not_found, is_explicit_null_version, is_get_codec_streaming_base_enabled,
is_get_small_object_direct_memory_enabled, is_lock_optimization_enabled, issue3031_diag_enabled, join_all,
known_put_object_storage_size, path_join_buf, put_restore_opts, record_compression_total_memory,
record_get_codec_streaming_gate_decision, record_get_direct_memory_decision, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_object_reader_path_observation, record_get_stage_duration_if_enabled,
record_lock_acquire, reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate,
republishes_data_movement_delete_marker, require_restore_operation_id, resolve_delete_version_state,
resolve_tiered_decommission_write_quorum_result, resolve_write_layout, restore_commit_operation_id_from_metadata,
restore_operation_id_from_metadata, send_event, set_disk_delete_creates_delete_marker,
@@ -2361,8 +2360,6 @@ fn is_restore_control_metadata(key: &str) -> bool {
.is_some_and(|remainder| remainder.is_empty())
|| rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_RESTORE_WORKER_LOCK)
.is_some_and(|remainder| remainder.is_empty())
|| rustfs_utils::http::internal_key_strip_suffix_prefix(key, SUFFIX_RESTORE_COPYBACK)
.is_some_and(|remainder| remainder.is_empty())
}
fn restore_metadata_update_preserves_protected_metadata(
@@ -4313,13 +4310,7 @@ impl SetDisks {
// state.
merge_replication_metadata_lww(&mut user_defined, &HashMap::new(), opts);
}
// Pending copy-back keeps its recovery identity until finalization.
if expected_restore_operation_id.is_some()
&& !user_defined
.get(metadata_keys::RESTORE)
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
.is_some_and(|status| status.is_restore_in_progress == Some(true))
{
if expected_restore_operation_id.is_some() {
rustfs_utils::http::metadata_compat::remove_str(&mut user_defined, SUFFIX_RESTORE_OPERATION_ID);
rustfs_utils::http::metadata_compat::remove_str(&mut user_defined, SUFFIX_RESTORE_WORKER_LOCK);
}
@@ -4720,29 +4711,19 @@ impl SetDisks {
inline_proof.to_owned(),
);
}
if opts.restore_copyback {
insert_str(
&mut user_defined,
SUFFIX_RESTORE_COPYBACK,
rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1.to_string(),
);
}
fi.metadata = user_defined;
let put_tier_free_version_id = if fi.version_id.is_none_or(|id| id.is_nil())
&& !opts.data_movement
&& expected_restore_operation_id.is_none()
&& !opts.restore_copyback
{
// Current disks publish the same cleanup owner alongside a
// replaced null version. Keep the intent outside metadata:
// older disks persist unknown metadata keys, which splits
// the replacement's read-quorum identity during an upgrade.
let free_version_id = Uuid::new_v4();
fi.overwrite_tier_free_version_id = Some(free_version_id);
Some(free_version_id)
} else {
None
};
let put_tier_free_version_id =
if fi.version_id.is_none_or(|id| id.is_nil()) && !opts.data_movement && expected_restore_operation_id.is_none() {
// Current disks publish the same cleanup owner alongside a
// replaced null version. Keep the intent outside metadata:
// older disks persist unknown metadata keys, which splits
// the replacement's read-quorum identity during an upgrade.
let free_version_id = Uuid::new_v4();
fi.overwrite_tier_free_version_id = Some(free_version_id);
Some(free_version_id)
} else {
None
};
fi.mod_time = mod_time;
fi.size = w_size as i64;
fi.versioned = opts.versioned || opts.version_suspended;
@@ -11912,14 +11893,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
if fi.transition_status == TRANSITION_COMPLETE {
return Ok(());
}
if fi
.metadata
.get(metadata_keys::RESTORE)
.and_then(|value| rustfs_filemeta::parse_restore_obj_status(value).ok())
.is_some_and(|status| status.is_restore_in_progress == Some(true))
{
return Ok(());
}
/*if fi.xlv1 {
if let Err(err) = self.heal_object(bucket, object, "", &HealOpts {no_lock: true, ..Default::default()}) {
return err.expect("err");
@@ -12426,9 +12400,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let actual_fi = actual.fi();
let oi = ObjectInfo::from_file_info(actual_fi, bucket, object, opts.versioned || opts.version_suspended);
let mut restore_cleanup_info = oi.clone();
let restore_result: Result<()> = async {
crate::services::notification_sys::require_restore_copyback_fleet_capability().await?;
// The tier reader releases its own lease at EOF, before PUT or
// CompleteMultipartUpload necessarily reaches metadata quorum. Keep a
// second exact-generation lease for the whole restore so the final
@@ -12507,7 +12479,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
.put_object(bucket, object, &mut p_reader, &ropts)
.await
.map_err(|err| to_object_err(err, vec![bucket, object]))?;
restore_cleanup_info = restored_info.clone();
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, &opts).await?;
send_event(EventArgs {
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
@@ -12627,7 +12598,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
bucket_lifecycle_lock_fence: opts.bucket_lifecycle_lock_fence.clone(),
user_defined: restore_commit_metadata,
no_lock: false,
restore_copyback: true,
decommission_capacity_admission: opts.decommission_capacity_admission.clone(),
// The composite ETag would otherwise be recomputed from the
// parts as they were written back, which for an encrypted or
@@ -12652,7 +12622,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
return Err(err);
}
};
restore_cleanup_info = restored_info.clone();
let restored_info = self_.finalize_restore_metadata(bucket, object, &restored_info, opts).await?;
send_event(EventArgs {
event_name: EventName::ObjectRestoreCompleted.as_str().to_string(),
@@ -12666,10 +12635,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
.await;
if let Err(primary_error) = restore_result {
if let Err(cleanup_error) = self_
.update_restore_metadata(bucket, object, &restore_cleanup_info, opts)
.await
{
if let Err(cleanup_error) = self_.update_restore_metadata(bucket, object, &oi, opts).await {
warn!(
event = EVENT_LIFECYCLE_RESTORE_CLEANUP,
component = LOG_COMPONENT_ECSTORE,
@@ -13642,8 +13608,8 @@ mod replication_quota_safety_tests {
let suspended_opts = ObjectOptions {
version_suspended: true,
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
status: Some("Suspended".to_owned()),
s3s::dto::VersioningConfiguration {
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
..Default::default()
},
None,
@@ -14276,11 +14242,11 @@ mod replication_lww_tests {
fn default_retention_snapshot(mode: &'static str) -> Arc<ObjectLockConfigSnapshot> {
Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::Configured {
config: rustfs_gateway_types::persistence::PersistedObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
rule: Some(rustfs_gateway_types::persistence::PersistedObjectLockRule {
default_retention: Some(rustfs_gateway_types::persistence::PersistedDefaultRetention {
mode: Some(mode.to_owned()),
config: s3s::dto::ObjectLockConfiguration {
object_lock_enabled: Some(s3s::dto::ObjectLockEnabled::from_static(s3s::dto::ObjectLockEnabled::ENABLED)),
rule: Some(s3s::dto::ObjectLockRule {
default_retention: Some(s3s::dto::DefaultRetention {
mode: Some(s3s::dto::ObjectLockRetentionMode::from_static(mode)),
days: Some(1),
years: None,
}),
@@ -14297,7 +14263,10 @@ mod replication_lww_tests {
/// timestamp the write passes and LWW judges it.
#[tokio::test]
async fn inbound_without_retention_timestamp_stays_rejected_under_bucket_default_retention() {
for mode in ["COMPLIANCE", "GOVERNANCE"] {
for mode in [
s3s::dto::ObjectLockRetentionMode::COMPLIANCE,
s3s::dto::ObjectLockRetentionMode::GOVERNANCE,
] {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-locked-default-retention";
let object = "object";
@@ -16450,8 +16419,8 @@ mod metadata_mutation_generation_tests {
#[tokio::test]
async fn restore_metadata_copy_preserves_tags_expiry_and_version_order() {
use rustfs_lifecycle::RestoreRequest;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use s3s::dto::RestoreRequest;
use time::format_description::well_known::Rfc3339;
let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
@@ -16728,7 +16697,7 @@ mod transition_commit_failure_tests {
use crate::services::tier::tier::TierConfigMgr;
use http::HeaderMap;
use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
use rustfs_lifecycle::RestoreRequest;
use s3s::dto::RestoreRequest;
use tokio::io::AsyncReadExt;
pub(super) fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap<String, String> {
@@ -19307,7 +19276,7 @@ mod transition_upload_integrity_tests {
for disk in disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut source = PutObjReader::from_vec(b"restore source body".repeat(64 * 1024));
let mut source = PutObjReader::from_vec(b"restore source body".repeat(1024));
set_disks
.put_object(bucket, object, &mut source, &ObjectOptions::default())
.await
@@ -19324,7 +19293,7 @@ mod transition_upload_integrity_tests {
.await
.expect("pending restore metadata should be installed");
let mut restored_reader = PutObjReader::from_vec(b"restored body".repeat(64 * 1024));
let mut restored_reader = PutObjReader::from_vec(b"restored body".repeat(1024));
set_disks
.put_object(
bucket,
@@ -19332,19 +19301,18 @@ mod transition_upload_integrity_tests {
&mut restored_reader,
&ObjectOptions {
user_defined: restore_metadata(operation_id, true),
restore_copyback: true,
..Default::default()
},
)
.await
.expect("matching restore copy-back should retain its pending operation id")
.expect("matching restore commit should consume its operation id")
}
fn restore_finalize_options(operation_id: Uuid) -> ObjectOptions {
ObjectOptions {
user_defined: restore_operation_id_metadata(operation_id),
transition: TransitionOptions {
restore_request: rustfs_lifecycle::RestoreRequest {
restore_request: s3s::dto::RestoreRequest {
days: Some(1),
..Default::default()
},
@@ -19354,87 +19322,17 @@ mod transition_upload_integrity_tests {
}
}
#[tokio::test]
#[serial_test::serial]
async fn restore_cleanup_reclaims_committed_copyback_data_dir() {
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "restore-cleanup-copyback-data-dir-bucket";
let object = "object.bin";
let operation_id = Uuid::new_v4();
let restored = write_committed_restore(&set_disks, &disk_stores, bucket, object, operation_id).await;
let data_dir = restored
.data_dir
.expect("committed copy-back must have a local data directory");
for temp_dir in &temp_dirs {
assert!(
temp_dir.path().join(bucket).join(object).join(data_dir.to_string()).exists(),
"committed copy-back must retain its local data directory before cleanup"
);
}
set_disks
.update_restore_metadata(
bucket,
object,
&restored,
&ObjectOptions {
user_defined: restore_operation_id_metadata(operation_id),
..Default::default()
},
)
.await
.expect("matching restore cleanup should remove the failed copy-back");
for temp_dir in &temp_dirs {
assert!(
!temp_dir.path().join(bucket).join(object).join(data_dir.to_string()).exists(),
"failed copy-back cleanup must reclaim its committed local data directory"
);
}
let cleaned = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("cleaned object metadata should remain readable");
assert!(!cleaned.user_defined.contains_key(rustfs_filemeta::metadata_keys::RESTORE));
assert!(
rustfs_utils::http::metadata_compat::get_consistent_str(
cleaned.user_defined.as_ref(),
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
)
.is_none(),
"failed copy-back cleanup must remove the local routing marker"
);
}
async fn assert_committed_restore_remains_pending(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, operation_id: Uuid) {
async fn assert_committed_restore_remains_pending(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) {
let current = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("pending restore metadata should remain readable");
assert_eq!(
restore_operation_id_from_metadata(current.user_defined.as_ref()).expect("operation id metadata should parse"),
Some(operation_id),
"unfinished restore copy-back must retain its exact operation id"
assert!(
restore_operation_id_from_metadata(current.user_defined.as_ref())
.expect("operation id metadata should parse")
.is_none(),
"successful restore commit must have consumed the operation id"
);
let operation_id = operation_id.to_string();
for (suffix, expected) in [
(rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID, operation_id.as_str()),
(
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
rustfs_utils::http::metadata_compat::RESTORE_WORKER_LOCK_PROTOCOL_V1,
),
] {
for prefix in [
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
] {
assert_eq!(
current.user_defined.get(&format!("{prefix}{suffix}")).map(String::as_str),
Some(expected),
"unfinished restore copy-back must retain each recovery marker alias"
);
}
}
assert!(
rustfs_filemeta::parse_restore_obj_status(
current
@@ -19492,7 +19390,7 @@ mod transition_upload_integrity_tests {
if error.kind() == std::io::ErrorKind::Other
&& error.to_string() == "restore finalization lock lost before metadata update"
));
assert_committed_restore_remains_pending(&set_disks, bucket, object, operation_id).await;
assert_committed_restore_remains_pending(&set_disks, bucket, object).await;
}
#[tokio::test]
@@ -19529,7 +19427,7 @@ mod transition_upload_integrity_tests {
..
}
));
assert_committed_restore_remains_pending(&set_disks, bucket, object, operation_id).await;
assert_committed_restore_remains_pending(&set_disks, bucket, object).await;
}
async fn assert_local_source_intact(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, payload: &[u8]) {
@@ -20079,7 +19977,7 @@ mod transition_upload_integrity_tests {
no_lock: true,
part_number: Some(1),
transition: TransitionOptions {
restore_request: rustfs_lifecycle::RestoreRequest {
restore_request: s3s::dto::RestoreRequest {
days: Some(1),
..Default::default()
},
@@ -24756,20 +24654,29 @@ mod put_object_tmp_cleanup_tests {
(
"compliance",
HashMap::from([
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string()),
(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until.clone()),
]),
),
(
"governance",
HashMap::from([
(metadata_keys::OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string()),
(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
),
(metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), retain_until),
]),
),
(
"legal-hold",
HashMap::from([(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string())]),
HashMap::from([(
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
s3s::dto::ObjectLockLegalHoldStatus::ON.to_string(),
)]),
),
];
@@ -24970,11 +24877,13 @@ mod put_object_tmp_cleanup_tests {
.expect("versioned PUT should return a version ID")
.to_string();
let default_compliance = ObjectLockConfigState::Configured {
config: rustfs_gateway_types::persistence::PersistedObjectLockConfiguration {
object_lock_enabled: Some("Enabled".to_owned()),
rule: Some(rustfs_gateway_types::persistence::PersistedObjectLockRule {
default_retention: Some(rustfs_gateway_types::persistence::PersistedDefaultRetention {
mode: Some("COMPLIANCE".to_owned()),
config: s3s::dto::ObjectLockConfiguration {
object_lock_enabled: Some(s3s::dto::ObjectLockEnabled::from_static(s3s::dto::ObjectLockEnabled::ENABLED)),
rule: Some(s3s::dto::ObjectLockRule {
default_retention: Some(s3s::dto::DefaultRetention {
mode: Some(s3s::dto::ObjectLockRetentionMode::from_static(
s3s::dto::ObjectLockRetentionMode::COMPLIANCE,
)),
days: Some(1),
years: None,
}),
@@ -25134,7 +25043,10 @@ mod put_object_tmp_cleanup_tests {
&ObjectOptions {
version_id: Some(destination_version.clone()),
versioned: true,
eval_metadata: Some(HashMap::from([(metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(), "ON".to_string())])),
eval_metadata: Some(HashMap::from([(
metadata_keys::OBJECT_LOCK_LEGAL_HOLD.to_string(),
s3s::dto::ObjectLockLegalHoldStatus::ON.to_string(),
)])),
..Default::default()
},
)
@@ -26167,7 +26079,7 @@ mod delete_objects_lock_gating_tests {
#[tokio::test]
async fn delete_objects_derives_per_object_versioning_from_the_request_snapshot() {
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
use s3s::dto::{BucketVersioningStatus, ExcludedPrefix, VersioningConfiguration};
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "batch-versioning-snapshot-bucket";
@@ -26179,8 +26091,10 @@ mod delete_objects_lock_gating_tests {
put_plain_object(&set_disks, bucket, "archive/unversioned-object").await;
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some("Enabled".to_owned()),
excluded_prefixes: Some(vec![Some("archive/".to_string())]),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
excluded_prefixes: Some(vec![ExcludedPrefix {
prefix: Some("archive/".to_string()),
}]),
..Default::default()
},
None,
@@ -26226,13 +26140,11 @@ mod delete_objects_lock_gating_tests {
#[tokio::test]
async fn batch_version_delete_uses_tags_read_under_the_delete_lock() {
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
use rustfs_gateway_types::persistence::{
PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationFilter,
PersistedReplicationRule, PersistedReplicationStatus, PersistedReplicationTag,
};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use s3s::dto::{
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration,
ReplicationRule, ReplicationRuleFilter, ReplicationRuleStatus, Tag, VersioningConfiguration,
};
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "batch-delete-locked-tags";
@@ -26268,23 +26180,23 @@ mod delete_objects_lock_gating_tests {
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![PersistedReplicationRule {
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: Some(PersistedReplicationStatus {
status: Status::ENABLED.to_string(),
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
}),
destination: PersistedReplicationDestination {
destination: Destination {
bucket: arn.to_string(),
..Default::default()
},
existing_object_replication: None,
filter: Some(PersistedReplicationFilter {
tag: Some(PersistedReplicationTag {
filter: Some(ReplicationRuleFilter {
tag: Some(Tag {
key: Some("generation".to_string()),
value: Some("locked".to_string()),
}),
@@ -26294,7 +26206,7 @@ mod delete_objects_lock_gating_tests {
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: Status::ENABLED.to_string(),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
));
@@ -26332,11 +26244,9 @@ mod delete_objects_lock_gating_tests {
#[tokio::test]
#[serial]
async fn lifecycle_delete_all_history_records_exact_replication_purge() {
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
use rustfs_gateway_types::persistence::{
PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
PersistedReplicationStatus,
use s3s::dto::{
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration,
ReplicationRule, ReplicationRuleStatus, VersioningConfiguration,
};
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
@@ -26378,17 +26288,17 @@ mod delete_objects_lock_gating_tests {
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![PersistedReplicationRule {
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: Some(PersistedReplicationStatus {
status: Status::ENABLED.to_string(),
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
}),
destination: PersistedReplicationDestination {
destination: Destination {
bucket: arn.to_string(),
..Default::default()
},
@@ -26398,7 +26308,7 @@ mod delete_objects_lock_gating_tests {
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: Status::ENABLED.to_string(),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
));
@@ -26440,11 +26350,10 @@ mod delete_objects_lock_gating_tests {
#[tokio::test]
async fn synthetic_directory_delete_uses_decoded_prefix_and_marker_switch() {
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
use rustfs_gateway_types::persistence::{
PersistedOptionalReplicationStatus, PersistedReplicationConfiguration, PersistedReplicationDestination,
PersistedReplicationFilter, PersistedReplicationRule, PersistedReplicationStatus,
use s3s::dto::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleFilter,
ReplicationRuleStatus, VersioningConfiguration,
};
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
@@ -26458,24 +26367,24 @@ mod delete_objects_lock_gating_tests {
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
VersioningConfiguration {
status: Some("Enabled".to_owned()),
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![PersistedReplicationRule {
delete_marker_replication: Some(PersistedOptionalReplicationStatus {
status: Some(Status::ENABLED.to_string()),
rules: vec![ReplicationRule {
delete_marker_replication: Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
}),
delete_replication: Some(PersistedReplicationStatus {
status: Status::DISABLED.to_string(),
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED),
}),
destination: PersistedReplicationDestination {
destination: Destination {
bucket: arn.to_string(),
..Default::default()
},
existing_object_replication: None,
filter: Some(PersistedReplicationFilter {
filter: Some(ReplicationRuleFilter {
prefix: Some("photos/".to_string()),
..Default::default()
}),
@@ -26483,7 +26392,7 @@ mod delete_objects_lock_gating_tests {
prefix: None,
priority: Some(1),
source_selection_criteria: None,
status: Status::ENABLED.to_string(),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
));
@@ -26615,7 +26524,10 @@ mod delete_objects_lock_gating_tests {
// stat and the #4297 rejection.
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 30);
let mut user_defined = HashMap::new();
user_defined.insert(metadata_keys::OBJECT_LOCK_MODE.to_string(), "COMPLIANCE".to_string());
user_defined.insert(
metadata_keys::OBJECT_LOCK_MODE.to_string(),
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
);
user_defined.insert(
metadata_keys::OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(),
retain_until
@@ -28245,10 +28157,9 @@ mod batch_delete_marker_tests {
#[tokio::test]
async fn batch_delete_explicit_marker_does_not_wait_for_pending_replication() {
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::{
PersistedReplicationConfiguration, PersistedReplicationDestination, PersistedReplicationRule,
PersistedReplicationStatus, PersistedVersioningConfiguration,
use s3s::dto::{
BucketVersioningStatus, DeleteReplication, DeleteReplicationStatus, Destination, ReplicationConfiguration,
ReplicationRule, ReplicationRuleStatus, VersioningConfiguration,
};
let (_dirs, store) = isolated_store_over_temp_disks().await;
@@ -28256,18 +28167,18 @@ mod batch_delete_marker_tests {
let (data_versions, markers) = marker_history(&store, bucket).await;
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
let snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
PersistedVersioningConfiguration {
status: Some("Enabled".to_owned()),
VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
},
Some(PersistedReplicationConfiguration {
Some(ReplicationConfiguration {
role: String::new(),
rules: vec![PersistedReplicationRule {
rules: vec![ReplicationRule {
delete_marker_replication: None,
delete_replication: Some(PersistedReplicationStatus {
status: Status::ENABLED.to_string(),
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
}),
destination: PersistedReplicationDestination {
destination: Destination {
bucket: arn.to_string(),
..Default::default()
},
@@ -28277,7 +28188,7 @@ mod batch_delete_marker_tests {
prefix: Some(String::new()),
priority: Some(1),
source_selection_criteria: None,
status: Status::ENABLED.to_string(),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}),
));
+2 -39
View File
@@ -2979,11 +2979,10 @@ fn build_legacy_per_part_fallback_reader(
part_length: usize,
part_size: usize,
read_repair: Option<(DecodeReadRepairContext, bool)>,
) -> Box<LegacyPerPartDecodeReader> {
) -> Box<dyn AsyncRead + Unpin + Send + Sync> {
let buffer = adaptive_duplex_buffer_size(part_size as i64);
let (read_half, write_half) = tokio::io::duplex(buffer);
let (read_half, mut write_half) = tokio::io::duplex(buffer);
let decode = tokio::spawn(async move {
let mut write_half = GetObjectDownstreamWriter::new(write_half);
let outcome = erasure
.decode_with_stripe_handles_and_reopeners_with_diagnostics(
&mut write_half,
@@ -8909,42 +8908,6 @@ mod tests {
assert!(err.to_string().contains("inconsistent read source shards"));
}
#[tokio::test]
async fn legacy_per_part_output_close_retains_downstream_classification() {
let erasure = coding::Erasure::new(2, 2, 64);
let data = (0..64u8).collect::<Vec<_>>();
let setup =
setup_codec_data_blocks_first_encoded_bitrot_readers(&erasure, &data, &[0], &[], &[], HashAlgorithm::HighwayHash256)
.await;
let mut reader = build_legacy_per_part_fallback_reader(
erasure,
setup.readers,
setup.deferred_stripe_handles,
Vec::new(),
None,
0,
data.len(),
data.len(),
None,
);
// Closing the actual output pipe models the body consumer disappearing
// while retaining the executor long enough to inspect its typed result.
assert!(!reader.finished);
let (replacement, _unused) = tokio::io::duplex(1);
drop(std::mem::replace(&mut reader.inner, replacement));
let handle = reader.decode.take().expect("fallback decode task");
let error = handle
.await
.expect("join the fallback task")
.expect("closed pipe must interrupt output");
assert_eq!(error.kind(), std::io::ErrorKind::BrokenPipe);
assert_eq!(
crate::diagnostics::get::classify_io_error(&error),
GetObjectFailureReason::DownstreamClosed,
"fallback output must carry the same marker as the normal GET path"
);
}
// backlog#879: a later multipart part that loses a shard (read quorum still
// holds) must be reconstructed in place through the legacy per-part decode
// bridge instead of failing the stream. This exercises the exact reader the
+3 -46
View File
@@ -27,7 +27,6 @@ use std::sync::Arc;
struct RestoreFinalizeBarrierState {
bucket: String,
object: String,
before_lock: bool,
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
}
@@ -44,18 +43,9 @@ pub(in crate::set_disk) struct RestoreFinalizeBarrier {
#[cfg(all(test, feature = "test-util"))]
impl RestoreFinalizeBarrier {
pub(in crate::set_disk) fn install(bucket: &str, object: &str) -> Self {
Self::install_at(bucket, object, false)
}
pub(in crate::set_disk) fn install_before_lock(bucket: &str, object: &str) -> Self {
Self::install_at(bucket, object, true)
}
fn install_at(bucket: &str, object: &str, before_lock: bool) -> Self {
let state = Arc::new(RestoreFinalizeBarrierState {
bucket: bucket.to_string(),
object: object.to_string(),
before_lock,
arrived: tokio::sync::Notify::new(),
release: tokio::sync::Notify::new(),
});
@@ -91,13 +81,13 @@ impl Drop for RestoreFinalizeBarrier {
}
#[cfg(all(test, feature = "test-util"))]
async fn maybe_pause_restore_finalize(bucket: &str, object: &str, before_lock: bool) {
async fn maybe_pause_restore_finalize(bucket: &str, object: &str) {
let barrier = RESTORE_FINALIZE_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("restore finalize barrier mutex should not poison")
.as_ref()
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.before_lock == before_lock)
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
.cloned();
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
@@ -160,8 +150,6 @@ impl SetDisks {
obj_info: &ObjectInfo,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
#[cfg(all(test, feature = "test-util"))]
maybe_pause_restore_finalize(bucket, object, true).await;
let expected = RestoreCleanupIdentity::from_object_info(obj_info);
let expected_operation_id = restore_operation_id_from_metadata(&opts.user_defined)?;
let expected_etag = obj_info
@@ -220,7 +208,7 @@ impl SetDisks {
return Err(Error::other("restored object changed before restore metadata finalization"));
}
#[cfg(all(test, feature = "test-util"))]
maybe_pause_restore_finalize(bucket, object, false).await;
maybe_pause_restore_finalize(bucket, object).await;
let restore_expiry =
lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), opts.transition.restore_request.days.unwrap_or(1));
fi.metadata.insert(
@@ -231,14 +219,7 @@ impl SetDisks {
}
.to_string(),
);
fi.transition_status = lifecycle::TRANSITION_COMPLETE.to_string();
rustfs_utils::http::metadata_compat::insert_str(
&mut fi.metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_STATUS,
lifecycle::TRANSITION_COMPLETE.to_string(),
);
for suffix in [
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
] {
@@ -334,20 +315,10 @@ impl SetDisks {
if !expected.matches_file_info(&fi, &expected_etag) {
return Ok(());
}
let copyback_data_dir = rustfs_utils::http::metadata_compat::get_consistent_str(
&fi.metadata,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
)
.filter(|value| *value == rustfs_utils::http::metadata_compat::RESTORE_COPYBACK_PROTOCOL_V1)
.and_then(|_| fi.data_dir.filter(|data_dir| !data_dir.is_nil()));
ensure_restore_metadata_lock_held(bucket, object, opts, "restore_cleanup_metadata")?;
fi.metadata.remove(metadata_keys::RESTORE);
fi.metadata.remove(metadata_keys::RESTORE_EXPIRY_DAYS);
fi.metadata.remove(metadata_keys::RESTORE_REQUEST_DATE);
rustfs_utils::http::metadata_compat::remove_str(
&mut fi.metadata,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_COPYBACK,
);
rustfs_utils::http::metadata_compat::remove_str(
&mut fi.metadata,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
@@ -377,20 +348,6 @@ impl SetDisks {
)
.await?;
self.invalidate_get_object_metadata_cache(bucket, object).await;
if let Some(data_dir) = copyback_data_dir {
let cleanup = self
.commit_rename_data_dir_and_mark_capacity(
&disks,
bucket,
object,
&data_dir.to_string(),
"",
self.default_write_quorum(),
)
.await;
self.report_old_data_dir_cleanup(bucket, object, &data_dir.to_string(), &cleanup)
.await;
}
Ok(())
}
}
@@ -274,452 +274,3 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot
"late GET must route to the remote generation instead of reopening the released local source"
);
}
#[tokio::test]
#[serial_test::serial]
async fn single_part_restore_copyback_stays_pending_before_finalization() {
restore_copyback_stays_pending_before_finalization(false).await;
}
#[tokio::test]
#[serial_test::serial]
async fn multipart_restore_copyback_stays_pending_before_finalization() {
restore_copyback_stays_pending_before_finalization(true).await;
}
async fn restore_copyback_stays_pending_before_finalization(multipart: bool) {
use crate::bucket::lifecycle::bucket_lifecycle_ops::LifecycleOps as _;
use crate::set_disk::replication::RestoreFinalizeBarrier;
use crate::storage_api_contracts::multipart::{CompletePart, MultipartOperations as _};
use rustfs_gateway_types::persistence::PersistedLifecycleConfiguration;
use rustfs_lifecycle::Lifecycle as _;
let (dirs, set_disks) = make_local_set_disks(4, 2).await;
let bucket = "restore-before-finalization-bucket";
let object = "object.bin";
let payload = vec![0x5au8; 6 * 1024 * 1024];
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let write_opts = ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
};
let original = if multipart {
let upload = set_disks
.new_multipart_upload(bucket, object, &write_opts)
.await
.expect("source multipart upload should be created");
let mut parts = Vec::new();
for (index, bytes) in payload.chunks(5 * 1024 * 1024).enumerate() {
let mut reader = PutObjReader::from_vec(bytes.to_vec());
let part = set_disks
.put_object_part(bucket, object, &upload.upload_id, index + 1, &mut reader, &write_opts)
.await
.expect("source part should be written");
parts.push(CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
});
}
set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload.upload_id, parts, &write_opts)
.await
.expect("source multipart upload should complete")
} else {
let mut reader = PutObjReader::from_vec(payload.clone());
set_disks
.put_object(bucket, object, &mut reader, &write_opts)
.await
.expect("source object should be written")
};
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&set_disks.instance_ctx().tier_config_mgr(), &tier_name).await;
set_disks
.transition_object(
bucket,
object,
&ObjectOptions {
transition: TransitionOptions {
status: TRANSITION_PENDING.to_string(),
tier: tier_name.clone(),
etag: original.etag.clone().expect("source ETag should be present"),
..Default::default()
},
version_id: original.version_id.map(|version| version.to_string()),
mod_time: original.mod_time,
..Default::default()
},
)
.await
.expect("source should transition before restore");
let operation_id = Uuid::new_v4().to_string();
let mut operation_metadata = std::collections::HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut operation_metadata,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
operation_id.clone(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut operation_metadata,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
rustfs_utils::http::metadata_compat::RESTORE_WORKER_LOCK_PROTOCOL_V1.to_string(),
);
operation_metadata.insert(
rustfs_filemeta::metadata_keys::RESTORE.to_string(),
"ongoing-request=\"true\"".to_string(),
);
set_disks
.put_object_metadata(
bucket,
object,
&ObjectOptions {
eval_metadata: Some(operation_metadata.clone()),
..Default::default()
},
)
.await
.expect("the restore operation generation should be installed");
let barrier = RestoreFinalizeBarrier::install_before_lock(bucket, object);
let restore_set = Arc::clone(&set_disks);
let restore = tokio::spawn(async move {
let mut opts = ObjectOptions {
user_defined: operation_metadata,
..Default::default()
};
opts.transition.restore_request.days = Some(1);
restore_set.restore_transitioned_object(bucket, object, &opts).await
});
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
.await
.expect("copy-back must reach the gap before finalization acquires its lock");
let pending = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("HEAD must remain available before finalization");
let pending_data_dir = pending
.data_dir
.expect("pending copy-back must retain its local data directory");
for dir in &dirs {
assert!(
dir.path()
.join(bucket)
.join(object)
.join(pending_data_dir.to_string())
.exists(),
"pending copy-back must retain its committed local data directory"
);
}
let status = parse_restore_obj_status(
pending
.user_defined
.get(rustfs_filemeta::metadata_keys::RESTORE)
.expect("copy-back must retain restore metadata"),
)
.expect("pending restore status should parse");
assert_eq!(
pending.transitioned_object.status, TRANSITION_COMPLETE,
"copy-back must retain the remote identity needed by recovery and cleanup"
);
backend.set_unreachable(true).await;
let mut restored_before_finalization = Vec::new();
set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("pending copy-back must be readable locally before metadata finalization")
.stream
.read_to_end(&mut restored_before_finalization)
.await
.expect("pending local body should drain");
assert_eq!(restored_before_finalization, payload, "pending local copy must preserve every byte");
set_disks
.transition_object(
bucket,
object,
&ObjectOptions {
transition: TransitionOptions {
status: TRANSITION_PENDING.to_string(),
tier: pending.transitioned_object.tier.clone(),
etag: pending.etag.clone().expect("pending ETag should be present"),
..Default::default()
},
version_id: pending.version_id.map(|version| version.to_string()),
mod_time: pending.mod_time,
..Default::default()
},
)
.await
.expect("a queued transition must skip the pending restore copy-back");
let after_queued_transition = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("restore metadata should remain readable after a queued transition");
assert_eq!(
after_queued_transition.transitioned_object.status, TRANSITION_COMPLETE,
"queued transition must not remove local copy-back data before finalization"
);
// Evaluate beyond any pre-copy deadline without sleeping or advancing the
// process clock. Pending copy-back must never become a restore-expiry candidate.
let lifecycle = PersistedLifecycleConfiguration::default();
let expiry_evaluation_time = OffsetDateTime::now_utc() + time::Duration::days(2);
let action = lifecycle
.eval_inner(&pending.to_lifecycle_opts(), expiry_evaluation_time, 0)
.await
.action;
barrier.release();
tokio::time::timeout(Duration::from_secs(30), restore)
.await
.expect("released restore should complete promptly")
.expect("restore task should join")
.expect("copy-back should finalize successfully");
assert!(status.on_going(), "copy-back must not advertise completion before metadata finalization");
assert!(status.expiry().is_none(), "pending copy-back must not expose a completed expiry");
assert!(
!action.delete_restored(),
"scanner must not reclaim copy-back before metadata finalization"
);
for (suffix, expected) in [
(rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID, operation_id.as_str()),
(
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
rustfs_utils::http::metadata_compat::RESTORE_WORKER_LOCK_PROTOCOL_V1,
),
] {
for prefix in [
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
] {
assert_eq!(
pending.user_defined.get(&format!("{prefix}{suffix}")).map(String::as_str),
Some(expected),
"pending copy-back must retain each recovery marker alias"
);
}
}
let completed = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("finalized metadata should be readable");
assert_eq!(
completed.data_dir,
Some(pending_data_dir),
"finalization must retain the local copy-back data directory"
);
assert_eq!(
completed.transitioned_object.status, TRANSITION_COMPLETE,
"finalization must restore the completed transition state"
);
let completed_status = parse_restore_obj_status(
completed
.user_defined
.get(rustfs_filemeta::metadata_keys::RESTORE)
.expect("finalized restore status should be present"),
)
.expect("finalized restore status should parse");
assert!(!completed_status.on_going(), "finalization must publish completion");
assert!(completed_status.expiry().is_some(), "finalization must publish the completed expiry");
for suffix in [
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
] {
for prefix in [
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
] {
assert!(
!completed.user_defined.contains_key(&format!("{prefix}{suffix}")),
"finalization must remove every recovery marker alias"
);
}
}
let completed_action = lifecycle
.eval_inner(&completed.to_lifecycle_opts(), expiry_evaluation_time, 0)
.await
.action;
assert!(
completed_action.delete_restored(),
"the same lifecycle evaluator must expire a completed restore beyond its deadline"
);
backend.set_unreachable(true).await;
let mut restored = Vec::new();
set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("completed restore must be readable with the remote tier unavailable")
.stream
.read_to_end(&mut restored)
.await
.expect("local restored body should drain");
assert_eq!(restored, payload, "finalized local copy must preserve every byte");
for dir in &dirs {
assert!(
dir.path()
.join(bucket)
.join(object)
.join(pending_data_dir.to_string())
.exists(),
"completed restore must retain its local data directory"
);
}
}
#[tokio::test]
#[serial_test::serial]
async fn restore_copyback_overwrite_preserves_remote_cleanup_owner() {
assert_transition_overwrite_preserves_remote_cleanup_owner(true).await;
}
#[tokio::test]
#[serial_test::serial]
async fn completed_transition_overwrite_preserves_remote_cleanup_owner() {
assert_transition_overwrite_preserves_remote_cleanup_owner(false).await;
}
async fn assert_transition_overwrite_preserves_remote_cleanup_owner(during_copyback: bool) {
use crate::set_disk::replication::RestoreFinalizeBarrier;
let (dirs, set_disks) = make_local_set_disks(4, 2).await;
let bucket = "restore-copyback-cleanup-owner";
let object = "object.bin";
let payload = vec![0x37u8; 6 * 1024 * 1024];
let write_opts = ObjectOptions {
write_completion: WriteCompletion::TailDrained,
..Default::default()
};
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("cleanup probe bucket should be created");
let mut reader = PutObjReader::from_vec(payload);
let original = set_disks
.put_object(bucket, object, &mut reader, &write_opts)
.await
.expect("cleanup probe source should be written");
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&set_disks.instance_ctx().tier_config_mgr(), &tier_name).await;
set_disks
.transition_object(
bucket,
object,
&ObjectOptions {
transition: TransitionOptions {
status: TRANSITION_PENDING.to_string(),
tier: tier_name.clone(),
etag: original.etag.clone().expect("source ETag should be present"),
..Default::default()
},
mod_time: original.mod_time,
..Default::default()
},
)
.await
.expect("cleanup probe source should transition");
let source = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("transitioned cleanup source should resolve");
assert_eq!(backend.object_count().await, 1, "transition must retain the real remote cleanup source");
let mut barrier = None;
let mut restore = None;
if during_copyback {
let operation_id = Uuid::new_v4();
let mut metadata = std::collections::HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
operation_id.to_string(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_WORKER_LOCK,
rustfs_utils::http::metadata_compat::RESTORE_WORKER_LOCK_PROTOCOL_V1.to_string(),
);
metadata.insert(
rustfs_filemeta::metadata_keys::RESTORE.to_string(),
"ongoing-request=\"true\"".to_string(),
);
set_disks
.put_object_metadata(
bucket,
object,
&ObjectOptions {
eval_metadata: Some(metadata.clone()),
..Default::default()
},
)
.await
.expect("cleanup probe restore generation should be installed");
let installed = RestoreFinalizeBarrier::install_before_lock(bucket, object);
let restore_set = Arc::clone(&set_disks);
restore = Some(tokio::spawn(async move {
let mut opts = ObjectOptions {
user_defined: metadata,
..Default::default()
};
opts.transition.restore_request.days = Some(1);
restore_set.restore_transitioned_object(bucket, object, &opts).await
}));
tokio::time::timeout(Duration::from_secs(30), installed.wait_until_paused())
.await
.expect("real restore should publish local bytes before finalization");
let pending = set_disks
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("copy-back metadata should resolve");
assert!(pending.restore_ongoing, "probe must hit an unfinished real restore");
barrier = Some(installed);
}
let mut replacement = PutObjReader::from_vec(vec![0x69u8; 6 * 1024 * 1024]);
set_disks
.put_object(bucket, object, &mut replacement, &write_opts)
.await
.expect("ordinary overwrite must succeed while restore finalization is paused");
if let Some(barrier) = barrier {
barrier.release();
}
if let Some(restore) = restore {
tokio::time::timeout(Duration::from_secs(30), restore)
.await
.expect("superseded restore must finish")
.expect("superseded restore must not panic")
.expect_err("restore finalization must not revive the overwritten generation");
}
let mut cleanup_owners = Vec::new();
for dir in &dirs {
let encoded = match tokio::fs::read(dir.path().join(bucket).join(object).join(crate::disk::STORAGE_FORMAT_FILE)).await {
Ok(encoded) => encoded,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
cleanup_owners.push(0);
continue;
}
Err(err) => panic!("cleanup source metadata should be readable: {err}"),
};
let versions = rustfs_filemeta::FileMeta::load(&encoded)
.expect("cleanup source metadata should decode")
.get_file_info_versions(bucket, object, false)
.expect("cleanup source versions should decode");
let count = versions
.free_versions
.iter()
.filter(|version| {
version.transition_tier == source.transitioned_object.tier
&& version.transitioned_objname == source.transitioned_object.name
})
.count();
cleanup_owners.push(count);
}
let remote_objects = backend.object_count().await;
println!("restore_cleanup_probe during_copyback={during_copyback} owners={cleanup_owners:?} remote_objects={remote_objects}");
assert!(
remote_objects == 0 || cleanup_owners.iter().filter(|count| **count > 0).count() >= 3,
"ordinary overwrite must remove the remote object or retain its cleanup owner at write quorum: during_copyback={during_copyback}, owners={cleanup_owners:?}, remote_objects={remote_objects}"
);
}
File diff suppressed because it is too large Load Diff
+19 -27
View File
@@ -527,7 +527,7 @@ impl ECStore {
async fn acquire_heal_format_fence(
&self,
) -> Result<(
crate::core::pools::PoolMetaSaveGateWriteGuard<'_>,
tokio::sync::MutexGuard<'_, PoolMetaWriteState>,
NamespaceLockGuard,
NamespaceLockGuard,
PoolMeta,
@@ -538,7 +538,7 @@ impl ECStore {
.first()
.cloned()
.ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
let mut write_state = self.pool_meta_save_gate.write().await;
let mut write_state = self.pool_meta_save_gate.lock().await;
write_state.ensure_write_safe("heal format fence failed")?;
// Metadata fence order is part of the decommission/rebalance protocol:
@@ -1079,7 +1079,7 @@ impl ECStore {
pub(super) async fn handle_heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
let movement_gate = self.ctx.data_movement_operation_gate();
let _movement_guard = movement_gate.read().await;
let save_guard = self.pool_meta_save_gate.write().await;
let save_guard = self.pool_meta_save_gate.lock().await;
save_guard.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")?;
let mut fenced_pools = BTreeSet::new();
{
@@ -2527,7 +2527,7 @@ mod tests {
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: Mutex::new(()),
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
pool_meta_save_gate: Mutex::default(),
ctx: crate::runtime::instance::bootstrap_ctx(),
bucket_fence_registry: std::sync::Arc::default(),
}
@@ -3550,14 +3550,10 @@ mod tests {
async fn full_format_heal_preblocked_pool_metadata_never_writes_format() {
let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
remove_heal_test_format(&temp_dir, &store, 0, 3).await;
store
.pool_meta_save_gate
.write()
.await
.observe_replicas(PoolMetaReplicaState {
needs_repair: true,
repair_write_safe: false,
});
store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
needs_repair: true,
repair_write_safe: false,
});
let err = store
.handle_heal_format(false)
@@ -3692,7 +3688,7 @@ mod tests {
let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
store.pools[0].disk_set[0].disks.write().await[3] = None;
let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
let recovery_latch = store.pool_meta_save_gate.write().await.aborted_transaction_latch_for_test();
let recovery_latch = store.pool_meta_save_gate.lock().await.aborted_transaction_latch_for_test();
let mut heal = tokio::spawn({
let store = Arc::clone(&store);
async move { store.handle_heal_format(false).await }
@@ -3852,7 +3848,7 @@ mod tests {
}
}
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
let state = source.pool_meta_save_gate.write().await.independent_clone_for_test();
let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
Arc::new(ECStore {
id: Uuid::new_v4(),
disk_map: Default::default(),
@@ -3862,7 +3858,7 @@ mod tests {
rebalance_meta: tokio::sync::RwLock::new(None),
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
start_gate: tokio::sync::Mutex::new(()),
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::new(state),
pool_meta_save_gate: tokio::sync::Mutex::new(state),
ctx: Arc::clone(&source.ctx),
bucket_fence_registry: Arc::default(),
})
@@ -4044,7 +4040,7 @@ mod tests {
let missing = remove_pool_meta_shard(&store, 0).await;
let (guard, admissions, mut state) = store.acquire_pool_meta_object_heal_fence(&[0]).await.expect("heal authority");
assert!(admissions.into_iter().all(|admission| admission.is_ok()));
assert!(store.pool_meta_save_gate.try_write().is_err(), "heal must retain the earlier save gate");
assert!(store.pool_meta_save_gate.try_lock().is_err(), "heal must retain the earlier save gate");
let writer_store = Arc::clone(&store);
let writer = tokio::spawn(async move { writer_store.save_current_pool_meta_for_test(&[0]).await });
tokio::task::yield_now().await;
@@ -4691,14 +4687,10 @@ mod tests {
#[tokio::test]
async fn bucket_heal_blocks_before_dispatch_after_unreadable_pool_meta_replica() {
let store = minimal_heal_store().await;
store
.pool_meta_save_gate
.write()
.await
.observe_replicas(PoolMetaReplicaState {
needs_repair: true,
repair_write_safe: false,
});
store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
needs_repair: true,
repair_write_safe: false,
});
let err = store
.handle_heal_bucket("bucket", &HealOpts::default())
@@ -4732,7 +4724,7 @@ mod tests {
.await
.expect("peer dispatch should start");
assert!(
store.pool_meta_save_gate.try_write().is_ok(),
store.pool_meta_save_gate.try_lock().is_ok(),
"coordinator must release its local save gate before waiting for peers"
);
assert!(
@@ -5124,7 +5116,7 @@ mod tests {
rebalance_meta: RwLock::new(None),
decommission_cancelers: RwLock::new(Vec::new()),
start_gate: Mutex::new(()),
pool_meta_save_gate: crate::core::pools::PoolMetaSaveGate::default(),
pool_meta_save_gate: Mutex::default(),
ctx: crate::runtime::instance::bootstrap_ctx(),
bucket_fence_registry: std::sync::Arc::default(),
};
@@ -5144,7 +5136,7 @@ mod tests {
.save_for_startup(store.pools.clone())
.await
.expect("pool metadata should be persisted before format heal");
*store.pool_meta_save_gate.write().await = crate::core::pools::PoolMetaWriteState::default();
*store.pool_meta_save_gate.lock().await = PoolMetaWriteState::default();
let replacement_endpoint = store.pools[1].endpoints.endpoints.as_ref()[2].clone();
tokio::fs::remove_file(
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More