mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-11 12:43:54 +00:00
Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| abfe23c02d | |||
| 04bcb57e6a | |||
| 4d3b8722ab | |||
| 22b0c42a2d | |||
| 9156ba24f4 | |||
| 325a181922 | |||
| 4836f7656a | |||
| 31e1e42ad7 | |||
| ca3128cfd3 | |||
| 8d04f1d2fc | |||
| 977060e457 | |||
| be4ef4bca0 | |||
| 1455f5f4a5 | |||
| 0e402b2534 | |||
| 4c7cc8a943 | |||
| db39cd5634 | |||
| 8172df44ec | |||
| 55d931353f | |||
| 7b3701ddc6 | |||
| 2df55c6dea |
@@ -62,10 +62,14 @@ script-tests: ## Run shell script tests
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
./scripts/check_s3s_footprint.sh --self-test
|
||||
./scripts/test_check_test_count_ratchet.sh
|
||||
./scripts/test_guard_source_set.sh
|
||||
./scripts/test_split_guard_rules.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_ecfs_legacy_edge.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_reference_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/release -p "test_*.py"
|
||||
@@ -92,6 +96,8 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_api_coverage.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Pinned RustFS release used as the "reference" target of parity lanes
|
||||
# (.github/workflows/reference-parity.yml). This is the single place that names
|
||||
# the reference release; .github/actions/reference-binary resolves, downloads and
|
||||
# verifies it, and fails on a missing pin, a missing asset or a digest mismatch.
|
||||
#
|
||||
# Bumping the reference is a reviewed event: every per-test difference between
|
||||
# the candidate and the reference is measured against this exact build.
|
||||
#
|
||||
# Each sha256 is the value published in the release's SHA256SUMS asset, which
|
||||
# matches the digest GitHub reports for the same asset. The action checks both.
|
||||
|
||||
schema = 1
|
||||
repository = "rustfs/rustfs"
|
||||
version = "1.0.5"
|
||||
tag = "1.0.5"
|
||||
|
||||
[platforms.linux-x86_64-gnu]
|
||||
asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
|
||||
sha256 = "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
|
||||
executable = "rustfs"
|
||||
@@ -0,0 +1,24 @@
|
||||
# Targets of the s3-tests reference parity lane (.github/workflows/reference-parity.yml).
|
||||
#
|
||||
# Exactly one target uses `binary = "reference"` (the release pinned in
|
||||
# .config/reference-binary.toml). Every other target uses `binary = "candidate"`
|
||||
# (the binary built from the branch under test) and is compared per node id
|
||||
# against the reference. All targets run the same harness, test list and server
|
||||
# configuration; `env` adds only the target's own RUSTFS_* settings.
|
||||
#
|
||||
# Adding a target is one more [[targets]] table, for example:
|
||||
#
|
||||
# [[targets]]
|
||||
# name = "candidate-gateway"
|
||||
# binary = "candidate"
|
||||
# env = { RUSTFS_S3_STACK = "gateway" }
|
||||
|
||||
schema = 1
|
||||
|
||||
[[targets]]
|
||||
name = "reference"
|
||||
binary = "reference"
|
||||
|
||||
[[targets]]
|
||||
name = "candidate-legacy"
|
||||
binary = "candidate"
|
||||
@@ -0,0 +1,12 @@
|
||||
# Tolerated per-test differences of the s3-tests reference parity lane.
|
||||
#
|
||||
# One entry per line, four fields separated by " | ":
|
||||
#
|
||||
# <target> | <pytest node id> | <tracking issue URL> | <reason>
|
||||
#
|
||||
# <target> is a candidate name from .config/reference-parity-targets.toml, the
|
||||
# node id is exactly as the lane prints it (s3tests/functional/test_s3.py::...),
|
||||
# and the issue URL is https://github.com/rustfs/<repo>/issues/<number>.
|
||||
#
|
||||
# Same semantics as scripts/junit_diff.py --allow: an entry whose difference no
|
||||
# longer occurs is stale and fails the lane, so remove it when it is fixed.
|
||||
@@ -0,0 +1,96 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: "Reference RustFS binary"
|
||||
description: "Download the pinned reference RustFS release, verify its sha256 and expose the executable path. A missing pin, missing asset or digest mismatch fails the step."
|
||||
|
||||
inputs:
|
||||
pin-file:
|
||||
description: "Pin file naming the reference release (repository-relative)"
|
||||
required: false
|
||||
default: ".config/reference-binary.toml"
|
||||
platform:
|
||||
description: "Platform table in the pin file"
|
||||
required: false
|
||||
default: "linux-x86_64-gnu"
|
||||
destination:
|
||||
description: "Directory that receives the executable; defaults to $RUNNER_TEMP/reference-binary"
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
outputs:
|
||||
path:
|
||||
description: "Absolute path of the verified executable"
|
||||
value: ${{ steps.unpack.outputs.path }}
|
||||
version:
|
||||
description: "Pinned reference version"
|
||||
value: ${{ steps.pin.outputs.version }}
|
||||
sha256:
|
||||
description: "Pinned and verified archive sha256"
|
||||
value: ${{ steps.pin.outputs.sha256 }}
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Resolve reference pin
|
||||
id: pin
|
||||
shell: bash
|
||||
env:
|
||||
PIN_TOOL: ${{ github.action_path }}/reference_binary.py
|
||||
PIN_FILE: ${{ inputs.pin-file }}
|
||||
PIN_PLATFORM: ${{ inputs.platform }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
resolved="$(python3 "$PIN_TOOL" resolve --pin "$PIN_FILE" --platform "$PIN_PLATFORM")"
|
||||
printf '%s\n' "$resolved"
|
||||
printf '%s\n' "$resolved" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify reference archive
|
||||
id: download
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
|
||||
REF_REPOSITORY: ${{ steps.pin.outputs.repository }}
|
||||
REF_TAG: ${{ steps.pin.outputs.tag }}
|
||||
REF_ASSET: ${{ steps.pin.outputs.asset }}
|
||||
REF_SHA256: ${{ steps.pin.outputs.sha256 }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
archive="$RUNNER_TEMP/reference-archive/$REF_ASSET"
|
||||
# Fails unless the release lists exactly one asset of this name whose
|
||||
# published digest equals the pin and whose downloaded bytes match it.
|
||||
python3 "$ASSET_DOWNLOADER" --repository "$REF_REPOSITORY" --tag "$REF_TAG" \
|
||||
--asset "$REF_ASSET" --output "$archive" --sha256 "$REF_SHA256"
|
||||
echo "archive=$archive" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Unpack reference executable
|
||||
id: unpack
|
||||
shell: bash
|
||||
env:
|
||||
PIN_TOOL: ${{ github.action_path }}/reference_binary.py
|
||||
REF_ARCHIVE: ${{ steps.download.outputs.archive }}
|
||||
REF_SHA256: ${{ steps.pin.outputs.sha256 }}
|
||||
REF_EXECUTABLE: ${{ steps.pin.outputs.executable }}
|
||||
REF_DESTINATION: ${{ inputs.destination }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
destination="${REF_DESTINATION:-$RUNNER_TEMP/reference-binary}"
|
||||
unpacked="$(python3 "$PIN_TOOL" unpack --archive "$REF_ARCHIVE" --sha256 "$REF_SHA256" \
|
||||
--executable "$REF_EXECUTABLE" --destination "$destination")"
|
||||
path="${unpacked#path=}"
|
||||
test -x "$path"
|
||||
echo "Reference executable: $path"
|
||||
"$path" --version
|
||||
echo "path=$path" >> "$GITHUB_OUTPUT"
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Resolve and unpack the pinned reference RustFS release binary.
|
||||
|
||||
`resolve` reads the pin file and prints the selected platform's identity as
|
||||
`key=value` lines (GitHub step-output format). `unpack` re-verifies the
|
||||
downloaded archive digest against the pin and extracts exactly one executable.
|
||||
Every missing or malformed input is a failure; nothing is skipped.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
import zipfile
|
||||
|
||||
REPOSITORY = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*")
|
||||
TAG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*")
|
||||
PLATFORM = re.compile(r"[a-z0-9][a-z0-9_-]*")
|
||||
ASSET = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*\.zip")
|
||||
SHA256 = re.compile(r"[0-9a-f]{64}")
|
||||
EXECUTABLE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
|
||||
MAX_EXECUTABLE_BYTES = 1024 * 1024 * 1024
|
||||
|
||||
|
||||
class PinError(Exception):
|
||||
"""A pin, archive or argument that cannot be trusted."""
|
||||
|
||||
|
||||
def field(table, key, pattern, where):
|
||||
value = table.get(key)
|
||||
if not isinstance(value, str) or not pattern.fullmatch(value):
|
||||
raise PinError(f"{where}: missing or invalid '{key}'")
|
||||
return value
|
||||
|
||||
|
||||
def resolve(pin_path, platform):
|
||||
if not PLATFORM.fullmatch(platform or ""):
|
||||
raise PinError("invalid platform name")
|
||||
try:
|
||||
data = tomllib.loads(Path(pin_path).read_text(encoding="utf-8"))
|
||||
except FileNotFoundError as exc:
|
||||
raise PinError(f"{pin_path}: reference pin file is missing") from exc
|
||||
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
|
||||
raise PinError(f"{pin_path}: cannot read reference pin: {exc}") from exc
|
||||
if data.get("schema") != 1 or type(data.get("schema")) is not int:
|
||||
raise PinError(f"{pin_path}: unsupported pin schema")
|
||||
identity = {
|
||||
"repository": field(data, "repository", REPOSITORY, pin_path),
|
||||
"version": field(data, "version", TAG, pin_path),
|
||||
"tag": field(data, "tag", TAG, pin_path),
|
||||
}
|
||||
platforms = data.get("platforms")
|
||||
if not isinstance(platforms, dict) or not platforms:
|
||||
raise PinError(f"{pin_path}: no pinned platforms")
|
||||
entry = platforms.get(platform)
|
||||
if not isinstance(entry, dict):
|
||||
raise PinError(f"{pin_path}: no pin for platform '{platform}'")
|
||||
where = f"{pin_path} [platforms.{platform}]"
|
||||
identity["asset"] = field(entry, "asset", ASSET, where)
|
||||
identity["sha256"] = field(entry, "sha256", SHA256, where)
|
||||
identity["executable"] = field(entry, "executable", EXECUTABLE, where)
|
||||
if identity["version"] not in identity["asset"]:
|
||||
raise PinError(f"{where}: asset name does not carry version {identity['version']}")
|
||||
unknown = set(entry) - {"asset", "sha256", "executable"}
|
||||
if unknown:
|
||||
raise PinError(f"{where}: unknown keys {sorted(unknown)}")
|
||||
return identity
|
||||
|
||||
|
||||
def file_sha256(path):
|
||||
with open(path, "rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def unpack(archive, sha256, executable, destination):
|
||||
if not SHA256.fullmatch(sha256 or "") or not EXECUTABLE.fullmatch(executable or ""):
|
||||
raise PinError("invalid expected digest or executable name")
|
||||
try:
|
||||
actual = file_sha256(archive)
|
||||
except OSError as exc:
|
||||
raise PinError(f"{archive}: reference archive is missing or unreadable") from exc
|
||||
if actual != sha256:
|
||||
raise PinError(f"{archive}: sha256 {actual} does not match the pinned digest")
|
||||
try:
|
||||
with zipfile.ZipFile(archive) as bundle:
|
||||
matches = [info for info in bundle.infolist() if info.filename == executable]
|
||||
if len(matches) != 1:
|
||||
raise PinError(f"{archive}: must contain exactly one top-level '{executable}'")
|
||||
info = matches[0]
|
||||
if info.is_dir() or not 0 < info.file_size <= MAX_EXECUTABLE_BYTES:
|
||||
raise PinError(f"{archive}: '{executable}' is not a plausible executable")
|
||||
destination = Path(destination)
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
target = destination / executable
|
||||
with bundle.open(info) as source, open(target, "wb") as sink:
|
||||
while chunk := source.read(1024 * 1024):
|
||||
sink.write(chunk)
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise PinError(f"{archive}: not a valid zip archive") from exc
|
||||
target.chmod(0o755)
|
||||
return target
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = parser.add_subparsers(dest="mode", required=True)
|
||||
res = sub.add_parser("resolve")
|
||||
res.add_argument("--pin", required=True)
|
||||
res.add_argument("--platform", required=True)
|
||||
unp = sub.add_parser("unpack")
|
||||
unp.add_argument("--archive", required=True)
|
||||
unp.add_argument("--sha256", required=True)
|
||||
unp.add_argument("--executable", required=True)
|
||||
unp.add_argument("--destination", required=True)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
if args.mode == "resolve":
|
||||
for key, value in resolve(args.pin, args.platform).items():
|
||||
print(f"{key}={value}")
|
||||
else:
|
||||
print(f"path={unpack(args.archive, args.sha256, args.executable, args.destination)}")
|
||||
except PinError as exc:
|
||||
print(f"reference binary: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -869,6 +869,33 @@
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "baseline"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/reference-parity.yml",
|
||||
"name": "Reference Parity",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"37 1 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "reference-parity.yml",
|
||||
"disposition": "s3-tests per-node-id parity of branch candidates against the pinned reference release",
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "added"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/functional-chain-health.yml",
|
||||
"name": "Functional chain health",
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# s3-tests reference parity lane (rustfs/backlog#2773).
|
||||
#
|
||||
# Runs the ceph/s3-tests implemented list (scripts/s3-tests/implemented_tests.txt)
|
||||
# against every target in .config/reference-parity-targets.toml and fails on
|
||||
# any per-node-id difference between a candidate and the reference:
|
||||
#
|
||||
# - reference: the release pinned in .config/reference-binary.toml, downloaded
|
||||
# and sha256-verified by .github/actions/reference-binary;
|
||||
# - candidate targets: the debug binary built from this checkout exactly as
|
||||
# ci.yml's build-rustfs-debug-binary job builds it for S3 Implemented Tests.
|
||||
#
|
||||
# Every target runs the same harness (scripts/s3-tests/run.sh from this
|
||||
# checkout), test list and server configuration on its own hosted runner. A
|
||||
# target's failures get one rerun against a fresh server; a case that passes it
|
||||
# is flaky, not a difference. scripts/s3-tests/reference_parity.py judges the
|
||||
# result and honours .config/reference-parity-tolerated.txt, whose entries must
|
||||
# each match a real difference.
|
||||
#
|
||||
# Report-only: not part of the Test and Lint aggregate. Scheduled failures are
|
||||
# owned by the scheduled-validation watchdog (.github/scheduled-validations.json).
|
||||
|
||||
name: Reference Parity
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/reference-parity.yml"
|
||||
- ".github/actions/reference-binary/**"
|
||||
- ".config/reference-binary.toml"
|
||||
- ".config/reference-parity-targets.toml"
|
||||
- ".config/reference-parity-tolerated.txt"
|
||||
- "scripts/s3-tests/reference_parity.py"
|
||||
- "scripts/junit_diff.py"
|
||||
schedule:
|
||||
- cron: "37 1 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
# Matches ci.yml: these variables are hashed into the rust-cache key, so the
|
||||
# candidate build reads the same ci-dev cache as build-rustfs-debug-binary.
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
plan:
|
||||
name: Plan parity targets
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
targets: ${{ steps.targets.outputs.targets }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Validate target list
|
||||
id: targets
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets="$(python3 scripts/s3-tests/reference_parity.py targets --config .config/reference-parity-targets.toml)"
|
||||
echo "targets=${targets}" >> "$GITHUB_OUTPUT"
|
||||
echo "Parity targets: ${targets}"
|
||||
|
||||
build-candidate:
|
||||
name: Build candidate binary
|
||||
# Critical-path CPU and cache ownership: the same debug build as ci.yml's
|
||||
# build-rustfs-debug-binary, reading the ci-dev cache written on sm-standard-4.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Build debug binary
|
||||
run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
- name: Upload candidate binary
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: reference-parity-candidate-binary
|
||||
overwrite: true
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e.json
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
s3tests:
|
||||
name: s3-tests (${{ matrix.target.name }})
|
||||
needs: [ plan, build-candidate ]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 75
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target: ${{ fromJSON(needs.plan.outputs.targets) }}
|
||||
env:
|
||||
TARGET_NAME: ${{ matrix.target.name }}
|
||||
TARGET_KIND: ${{ matrix.target.binary }}
|
||||
# Above this many first-run failures the target is systemically broken;
|
||||
# a rerun would only repeat it, so the first run is judged as is.
|
||||
RERUN_CAP: "50"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Fetch pinned reference binary
|
||||
id: reference
|
||||
if: matrix.target.binary == 'reference'
|
||||
uses: ./.github/actions/reference-binary
|
||||
|
||||
- name: Download candidate binary
|
||||
if: matrix.target.binary == 'candidate'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: reference-parity-candidate-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Select target binary
|
||||
env:
|
||||
REFERENCE_PATH: ${{ steps.reference.outputs.path }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "$TARGET_KIND" in
|
||||
reference) binary="$REFERENCE_PATH" ;;
|
||||
candidate) chmod +x target/debug/rustfs; binary="$PWD/target/debug/rustfs" ;;
|
||||
*) echo "unknown target binary kind: $TARGET_KIND" >&2; exit 1 ;;
|
||||
esac
|
||||
test -x "$binary"
|
||||
echo "TARGET_BINARY=$binary" >> "$GITHUB_ENV"
|
||||
mkdir -p "parity/$TARGET_NAME"
|
||||
{
|
||||
echo "target=$TARGET_NAME"
|
||||
echo "binary=$TARGET_KIND"
|
||||
echo "sha256=$(sha256sum "$binary" | cut -d' ' -f1)"
|
||||
echo "version=$("$binary" --version | tr '\n' ' ')"
|
||||
} | tee "parity/$TARGET_NAME/target.txt"
|
||||
|
||||
- name: Apply target environment
|
||||
env:
|
||||
TARGET_JSON: ${{ toJSON(matrix.target) }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 scripts/s3-tests/reference_parity.py export-env --target-json "$TARGET_JSON" \
|
||||
| tee -a "$GITHUB_ENV" "parity/$TARGET_NAME/target-env.txt"
|
||||
|
||||
- name: Run implemented s3-tests
|
||||
run: |
|
||||
set -uo pipefail
|
||||
report="parity/$TARGET_NAME"
|
||||
harness="artifacts/s3tests-single"
|
||||
|
||||
# Same invocation as ci.yml's S3 Implemented Tests, with a fresh data
|
||||
# root and port per run. $1 optionally replaces the test list.
|
||||
run_suite() {
|
||||
local list="$1" label="$2" root port
|
||||
root="${RUNNER_TEMP}/rustfs-s3tests-${label}"
|
||||
rm -rf "$harness" "$root"
|
||||
mkdir -p "$root"
|
||||
port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
|
||||
env DEPLOY_MODE=binary \
|
||||
RUSTFS_BINARY="$TARGET_BINARY" \
|
||||
TEST_MODE=single \
|
||||
MAXFAIL=0 \
|
||||
XDIST=4 \
|
||||
S3_PORT="$port" \
|
||||
DATA_ROOT="$root" \
|
||||
S3TESTS_CONF="$harness/s3tests.conf" \
|
||||
${list:+IMPLEMENTED_TESTS_FILE="$list"} \
|
||||
./scripts/s3-tests/run.sh
|
||||
}
|
||||
|
||||
# A run is usable only if it produced a JUnit report; test failures
|
||||
# are data for the comparison, a missing report is a lane failure.
|
||||
keep() {
|
||||
local prefix="$1"
|
||||
for file in pytest.log compat-report.md; do
|
||||
if [[ -f "$harness/$file" ]]; then cp "$harness/$file" "$report/${prefix}${file}"; fi
|
||||
done
|
||||
if [[ -f "$harness/rustfs-single/rustfs.log" ]]; then cp "$harness/rustfs-single/rustfs.log" "$report/${prefix}rustfs.log"; fi
|
||||
test -s "$harness/junit.xml" || { echo "::error::$TARGET_NAME produced no JUnit report (${prefix:-first} run)"; return 1; }
|
||||
cp "$harness/junit.xml" "$report/${prefix}junit.xml"
|
||||
}
|
||||
|
||||
run_suite "" first
|
||||
echo "first_run_exit=$?" >> "$report/target.txt"
|
||||
keep "" || exit 1
|
||||
cp "$harness/selected-nodeids.txt" "$report/selected-nodeids.txt" || exit 1
|
||||
|
||||
failures="$(python3 scripts/s3-tests/reference_parity.py failed --report "$report")" || exit 1
|
||||
count="$(printf '%s' "$failures" | grep -c . || true)"
|
||||
echo "first_run_failures=$count" >> "$report/target.txt"
|
||||
if (( count == 0 )); then
|
||||
exit 0
|
||||
fi
|
||||
if (( count > RERUN_CAP )); then
|
||||
echo "::warning::$TARGET_NAME has $count failures (cap $RERUN_CAP); judging the first run without a rerun"
|
||||
echo "rerun=skipped-over-cap" >> "$report/target.txt"
|
||||
exit 0
|
||||
fi
|
||||
printf '%s\n' "$failures" > "$RUNNER_TEMP/rerun-tests.txt"
|
||||
run_suite "$RUNNER_TEMP/rerun-tests.txt" rerun
|
||||
echo "rerun_exit=$?" >> "$report/target.txt"
|
||||
keep rerun- || exit 1
|
||||
|
||||
- name: Upload target report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: reference-parity-report-${{ matrix.target.name }}
|
||||
overwrite: true
|
||||
path: parity/${{ matrix.target.name }}/
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
compare:
|
||||
name: Compare targets
|
||||
needs: [ plan, s3tests ]
|
||||
if: always() && needs.plan.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download target reports
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: reference-parity-report-*
|
||||
path: parity-download
|
||||
|
||||
- name: Compare per node id
|
||||
env:
|
||||
TARGETS: ${{ needs.plan.outputs.targets }}
|
||||
S3TESTS_RESULT: ${{ needs.s3tests.result }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p parity
|
||||
for dir in parity-download/reference-parity-report-*; do
|
||||
if [[ -d "$dir" ]]; then mv "$dir" "parity/${dir#parity-download/reference-parity-report-}"; fi
|
||||
done
|
||||
python3 scripts/s3-tests/reference_parity.py compare \
|
||||
--targets-json "$TARGETS" \
|
||||
--reports parity \
|
||||
--tolerated .config/reference-parity-tolerated.txt \
|
||||
--markdown "$GITHUB_STEP_SUMMARY" \
|
||||
--json parity/diff.json > parity/diff.md
|
||||
status=$?
|
||||
cat parity/diff.md
|
||||
if [[ "$S3TESTS_RESULT" != "success" ]]; then
|
||||
echo "::error::s3-tests target jobs finished with '$S3TESTS_RESULT'"
|
||||
status=1
|
||||
fi
|
||||
exit "$status"
|
||||
|
||||
- name: Upload parity diff
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: reference-parity-diff
|
||||
overwrite: true
|
||||
path: |
|
||||
parity/diff.md
|
||||
parity/diff.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
@@ -11541,11 +11541,14 @@ impl LocalDisk {
|
||||
) -> Result<()> {
|
||||
multipart_staging::reject_reserved_anchor(path)?;
|
||||
let fuse_with_admission = admission.is_some() && !crash_inject::is_armed(CrashPoint::MetaWriteAfterTmpBeforeRename, path);
|
||||
// Held until the staging root is published, by the filesystem work
|
||||
// itself, so a concurrent listing cannot prune the marker in between.
|
||||
let mut index_pin = None;
|
||||
let prepared_index = if volume == RUSTFS_META_MULTIPART_BUCKET {
|
||||
if fuse_with_admission {
|
||||
self.multipart_index.prepare_record_head(path, &buf).await
|
||||
} else {
|
||||
self.multipart_index.record_head(path, &buf).await?;
|
||||
index_pin = self.multipart_index.record_head(path, &buf).await?;
|
||||
None
|
||||
}
|
||||
} else {
|
||||
@@ -11571,8 +11574,10 @@ impl LocalDisk {
|
||||
let root = self.publication_root.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner).await;
|
||||
os::run_blocking_namespace_operation_with_sync(lease, do_sync, move || {
|
||||
if let Some(record) = prepared_index {
|
||||
record.run_std()?;
|
||||
let _index_pin = prepared_index.map(|record| record.run_std()).transpose()?;
|
||||
#[cfg(test)]
|
||||
if let Some(pin) = &_index_pin {
|
||||
pin.pause_for_test();
|
||||
}
|
||||
super::fs::create_dir_all_std(&tmp_volume_dir)?;
|
||||
#[cfg(test)]
|
||||
@@ -11616,12 +11621,16 @@ impl LocalDisk {
|
||||
return Err(DiskError::Unexpected);
|
||||
}
|
||||
|
||||
let publication_owner: Option<Arc<dyn Send + Sync>> = match index_pin {
|
||||
Some(pin) => Some(Arc::new((namespace_owner.clone(), pin))),
|
||||
None => namespace_owner.clone(),
|
||||
};
|
||||
os::rename_all_with_precondition_and_parent_sync(
|
||||
&tmp_file_path,
|
||||
&file_path,
|
||||
&volume_dir,
|
||||
&self.publication_root,
|
||||
namespace_owner.clone(),
|
||||
publication_owner,
|
||||
admission.as_ref().map(|admission| admission.precondition()),
|
||||
admission.is_some() && sync && durability.syncs_commit_metadata(),
|
||||
)
|
||||
|
||||
@@ -521,8 +521,12 @@ impl LocalDisk {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_info(dst_path, &fi).await?;
|
||||
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|
||||
&& let Some(pin) = self.multipart_index.record_info(dst_path, &fi).await?
|
||||
{
|
||||
// The owner chain reaches every lease this rename publishes under,
|
||||
// so the index marker stays pinned until that work has returned.
|
||||
state.namespace_owner = Some(Arc::new((state.namespace_owner.take(), pin)));
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
|
||||
@@ -41,19 +41,53 @@ struct DiscoverySession {
|
||||
touched: Instant,
|
||||
}
|
||||
|
||||
/// Plain (non-transaction) children of an object-scoped discovery root cost one
|
||||
/// budget entry per this many names: enough to bound a hostile prefix without
|
||||
/// making a real folder prefix undeletable.
|
||||
/// Names an object-scoped scan skips without a stat (plain children of the
|
||||
/// key's directory, and entries of a UUID child that cannot be delete evidence)
|
||||
/// cost one budget entry per this many names: enough to bound a hostile prefix
|
||||
/// without making a real folder prefix undeletable.
|
||||
const PLAIN_CHILDREN_PER_BUDGET_ENTRY: u32 = 64;
|
||||
|
||||
/// Entries one step of the resumable (background) discovery may visit. The
|
||||
/// foreground bounded protocol has its own, larger per-call allowance.
|
||||
const DISCOVERY_STEP_ENTRIES: u32 = 4096;
|
||||
|
||||
/// Wall-clock allowance of one bounded discovery call: a third of the drive
|
||||
/// timeout that wraps the call (including its wait for a lane worker), capped at
|
||||
/// ten seconds, so an incomplete walk returns before the call can time out.
|
||||
fn bounded_discovery_time_limit() -> std::time::Duration {
|
||||
(crate::disk::disk_store::get_max_timeout_duration() / 3).min(std::time::Duration::from_secs(10))
|
||||
}
|
||||
|
||||
/// Whether an entry inside a UUID child of an object-scoped root can name one
|
||||
/// of the key's delete transactions: a journal file of a rollback directory or
|
||||
/// a data-directory marker. Every other name is inert to discovery.
|
||||
fn may_be_delete_evidence(name: &str) -> bool {
|
||||
[PREPARED, STORAGE_FORMAT_FILE_BACKUP, DELETE_QUORUM_COMMITTED_FILE].contains(&name)
|
||||
|| [
|
||||
DELETE_DATA_DIR_MARKER_PREFIX,
|
||||
RESERVED_DELETE_DATA_DIR_MARKER_PREFIX,
|
||||
RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX,
|
||||
]
|
||||
.iter()
|
||||
.any(|prefix| delete_transaction_marker_id(name, prefix).is_some())
|
||||
}
|
||||
|
||||
impl DiscoverySession {
|
||||
/// Consume one bounded step, returning ownership only when traversal remains.
|
||||
fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
|
||||
self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
|
||||
self.advance_bounded(limit, DISCOVERY_STEP_ENTRIES, &|| false)
|
||||
.map(|(transactions, next, _)| (transactions, next))
|
||||
}
|
||||
|
||||
fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
|
||||
/// Walk until `limit` transactions, `max_entries` charged entries, the end
|
||||
/// of the scope, or `should_stop` (checked before every entry); any early
|
||||
/// stop returns the remaining traversal as continuation state.
|
||||
fn advance_bounded(
|
||||
mut self,
|
||||
limit: u32,
|
||||
max_entries: u32,
|
||||
should_stop: &dyn Fn() -> bool,
|
||||
) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
|
||||
if self.frames.is_empty() {
|
||||
match std::fs::read_dir(&self.root) {
|
||||
Ok(entries) => self.frames.push((String::new(), entries)),
|
||||
@@ -66,10 +100,11 @@ impl DiscoverySession {
|
||||
let mut transactions = BTreeSet::new();
|
||||
let mut seen_candidates = HashSet::new();
|
||||
let mut visited = 0u32;
|
||||
let mut plain_children = 0u32;
|
||||
let mut skipped_by_name = 0u32;
|
||||
while !self.frames.is_empty()
|
||||
&& transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
|
||||
&& visited < max_entries
|
||||
&& !should_stop()
|
||||
{
|
||||
let (parent, entries) = self
|
||||
.frames
|
||||
@@ -90,13 +125,22 @@ impl DiscoverySession {
|
||||
return Err(std::io::Error::from(ErrorKind::InvalidData));
|
||||
}
|
||||
// An object-scoped scan only ever descends into transaction (UUID)
|
||||
// directories at the root. Plain child keys of a prefix directory are
|
||||
// skipped by name without a stat; they are charged at one entry per
|
||||
// readdir batch so the work stays bounded while deleting a missing key
|
||||
// that names a prefix with thousands of children can still complete.
|
||||
if self.object.is_some() && parent.is_empty() && !canonical_uuid(&name).is_some_and(|id| !id.is_nil()) {
|
||||
plain_children += 1;
|
||||
if plain_children.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
|
||||
// directories at the root, and inside them only journal files and
|
||||
// data-directory markers can be evidence. Every other name (plain
|
||||
// child keys of a prefix directory, and the `xl.meta`, data
|
||||
// directories or children of a UUID-named child key) is skipped by
|
||||
// name without a stat and charged at one entry per readdir batch, so
|
||||
// the work stays bounded while deleting a missing key that names a
|
||||
// folder prefix with many children, UUID-named or not, can complete.
|
||||
if self.object.is_some()
|
||||
&& if parent.is_empty() {
|
||||
!canonical_uuid(&name).is_some_and(|id| !id.is_nil())
|
||||
} else {
|
||||
!may_be_delete_evidence(&name)
|
||||
}
|
||||
{
|
||||
skipped_by_name += 1;
|
||||
if skipped_by_name.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
|
||||
visited += 1;
|
||||
}
|
||||
continue;
|
||||
@@ -1391,8 +1435,16 @@ impl LocalDisk {
|
||||
self.validate_delete_discovery_owner(&session)?;
|
||||
// The blocking owner drops every directory handle even if the RPC waiter disappears.
|
||||
// A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
|
||||
// A large folder prefix may take a while to walk, so the probe rides the
|
||||
// durable-mutation lane like bounded inspection, never the one-worker
|
||||
// foreground lane that every other mutation on the drive queues behind.
|
||||
// It also stops, incomplete, when its waiter goes away or well before the
|
||||
// drive timeout, so a slow walk answers SlowDown instead of marking the
|
||||
// drive failed or holding a lane worker after the request is gone.
|
||||
let deadline = Instant::now() + bounded_discovery_time_limit();
|
||||
os::run_cancellable_blocking_namespace_operation_with_sync(lease, true, move |is_cancelled| {
|
||||
let should_stop = || is_cancelled() || Instant::now() >= deadline;
|
||||
let (transactions, next, visited) = session.advance_bounded(256, max_entries, &should_stop)?;
|
||||
Ok(super::super::BoundedDeleteRecoveryDiscovery {
|
||||
transactions: if next.is_none() { transactions } else { Vec::new() },
|
||||
visited,
|
||||
@@ -6561,6 +6613,93 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_discovery_charges_uuid_named_children_once_and_keeps_evidence() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
|
||||
std::fs::create_dir_all(&path).expect("physical prefix");
|
||||
// UUID-named child keys of the folder: each holds its own `xl.meta` and a
|
||||
// data directory, neither of which can be evidence for the folder's key.
|
||||
const CHILDREN: u32 = 2000;
|
||||
for _ in 0..CHILDREN {
|
||||
let child = path.join(Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(child.join(Uuid::new_v4().to_string())).expect("child key data dir");
|
||||
std::fs::write(child.join(STORAGE_FORMAT_FILE), b"meta").expect("child key metadata");
|
||||
}
|
||||
// A UUID-named sub-prefix with children of its own.
|
||||
const SUB_PREFIX_CHILDREN: u32 = 640;
|
||||
let sub_prefix = path.join(Uuid::new_v4().to_string());
|
||||
for index in 0..SUB_PREFIX_CHILDREN {
|
||||
std::fs::create_dir_all(sub_prefix.join(format!("child-{index}"))).expect("sub-prefix child");
|
||||
}
|
||||
// Evidence of two interrupted transactions of the key itself: a journaled
|
||||
// rollback directory, and a data-directory marker whose rollback is gone.
|
||||
let journaled = Uuid::new_v4();
|
||||
std::fs::create_dir_all(path.join(journaled.to_string())).expect("rollback directory");
|
||||
std::fs::write(path.join(journaled.to_string()).join(PREPARED), b"{}").expect("journal");
|
||||
let marker_only = Uuid::new_v4();
|
||||
let data_dir = path.join(Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(&data_dir).expect("residual data directory");
|
||||
std::fs::write(data_dir.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{marker_only}")), [])
|
||||
.expect("reservation marker");
|
||||
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded(
|
||||
"recovery",
|
||||
"prefix",
|
||||
super::super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES,
|
||||
)
|
||||
.await
|
||||
.expect("bounded probe");
|
||||
assert!(page.complete, "a folder of UUID-named keys is a complete inventory");
|
||||
// One entry per descent (children, sub-prefix, rollback and data
|
||||
// directory) and per evidence file; inert names at 1/64.
|
||||
let inert = 2 * CHILDREN + SUB_PREFIX_CHILDREN;
|
||||
assert_eq!(page.visited, CHILDREN + 3 + 2 + inert / PLAIN_CHILDREN_PER_BUDGET_ENTRY);
|
||||
let found = page.transactions.iter().map(|id| id.transaction).collect::<BTreeSet<_>>();
|
||||
assert!(found.contains(&journaled), "a journaled rollback directory is evidence");
|
||||
assert!(found.contains(&marker_only), "a marker-only residue is evidence");
|
||||
assert_eq!(found.len(), 2, "no child key is mistaken for a transaction");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_discovery_stops_on_request_and_skips_only_inert_irregular_entries() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
|
||||
let child = path.join(Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(&child).expect("UUID-named child key");
|
||||
// A stop request (cancellation or the wall-clock limit) ends the walk
|
||||
// before any entry, leaving continuation state: never evidence of absence.
|
||||
let session = disk
|
||||
.new_delete_discovery_session("recovery", Some("prefix"), &path)
|
||||
.expect("private session");
|
||||
let (transactions, next, visited) = session
|
||||
.advance_bounded(256, super::super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, &|| true)
|
||||
.expect("stopped walk");
|
||||
assert!(transactions.is_empty());
|
||||
assert_eq!(visited, 0);
|
||||
assert!(next.is_some(), "a stopped walk is incomplete");
|
||||
drop(next);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
// An irregular entry whose name cannot be evidence is skipped unread.
|
||||
std::os::unix::fs::symlink(&path, child.join("inert-link")).expect("inert symlink");
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "prefix", 4096)
|
||||
.await
|
||||
.expect("inert irregular entries are not evidence");
|
||||
assert!(page.complete && page.transactions.is_empty());
|
||||
// One with an evidence name still fails closed.
|
||||
std::os::unix::fs::symlink(&path, child.join(PREPARED)).expect("evidence-named symlink");
|
||||
assert!(
|
||||
disk.discover_delete_transactions_bounded("recovery", "prefix", 4096)
|
||||
.await
|
||||
.is_err(),
|
||||
"an evidence-named symlink is corruption, not a skipped name"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
@@ -6663,8 +6802,11 @@ mod tests {
|
||||
let filler_id = Uuid::new_v4().to_string();
|
||||
let filler = path.join(&filler_id);
|
||||
fs::create_dir_all(&filler).await.expect("non-evidence directory");
|
||||
for index in 0..5000 {
|
||||
fs::write(filler.join(format!("part-{index:04}")), [])
|
||||
// Inert names are skipped at 1/64, so fill the step with entries that
|
||||
// pass the evidence-name filter but are directories: each costs a full
|
||||
// budget entry and names no transaction.
|
||||
for _ in 0..5000 {
|
||||
fs::create_dir(filler.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4())))
|
||||
.await
|
||||
.expect("non-evidence entry");
|
||||
}
|
||||
|
||||
@@ -4,6 +4,11 @@
|
||||
//! Bucket-to-staging-root candidates: one marker per staging root, recorded
|
||||
//! before the root is published and pruned once the root is gone. Source
|
||||
//! metadata stays authoritative.
|
||||
//!
|
||||
//! A marker without a staging root is either a leftover or an upload whose
|
||||
//! root is still being published. The publisher proves the second case by
|
||||
//! holding a [`PublicationPin`] from before it records the marker until its
|
||||
//! filesystem work has returned; no elapsed time is taken as that proof.
|
||||
|
||||
use super::{LocalDisk, read_metadata_only_std};
|
||||
use crate::disk::{
|
||||
@@ -29,9 +34,6 @@ use uuid::Uuid;
|
||||
|
||||
const INDEX: &str = ".rustfs.sys/multipart-index-v1";
|
||||
const REBUILD_DIRENTS_PER_JOB: usize = 64;
|
||||
/// A marker is recorded before its staging root is published. A marker younger
|
||||
/// than this without a root is an upload in flight, never a leftover to prune.
|
||||
const MULTIPART_INDEX_PRUNE_GRACE: std::time::Duration = std::time::Duration::from_secs(300);
|
||||
type DirectoryEntries = Box<dyn Iterator<Item = io::Result<String>> + Send>;
|
||||
type PhysicalRoot = (u64, [u8; 16]);
|
||||
static OWNERS: LazyLock<parking_lot::Mutex<HashMap<PhysicalRoot, Weak<IndexOwner>>>> =
|
||||
@@ -56,9 +58,15 @@ pub(super) struct IndexOwner {
|
||||
// drop so a cancelled scan can be resumed by the next caller.
|
||||
driving: AtomicBool,
|
||||
disabled: AtomicBool,
|
||||
// Markers whose staging root some publisher is creating right now, with
|
||||
// the number of publishers. Boot-local like the index itself: every writer
|
||||
// of this physical drive in this process shares this owner.
|
||||
publishing: parking_lot::Mutex<HashMap<PathBuf, usize>>,
|
||||
#[cfg(test)]
|
||||
rebuild_head_pause: parking_lot::Mutex<Option<RebuildHeadPause>>,
|
||||
#[cfg(test)]
|
||||
publication_pause: parking_lot::Mutex<Option<RebuildHeadPause>>,
|
||||
#[cfg(test)]
|
||||
observed_rebuild_queue: parking_lot::Mutex<Option<Arc<os::DiskBlockingQueue>>>,
|
||||
#[cfg(test)]
|
||||
rebuild_single_job: AtomicBool,
|
||||
@@ -120,23 +128,6 @@ impl IndexDirectory {
|
||||
std::fs::File::open(self.path.join(name))
|
||||
}
|
||||
}
|
||||
fn modified(&self, name: &str) -> io::Result<std::time::SystemTime> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.open_file(name)?.metadata()?.modified()
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
std::fs::metadata(self.path.join(name))?.modified()
|
||||
}
|
||||
}
|
||||
/// Whether the marker is still inside the publication grace period. A
|
||||
/// marker dated in the future (clock stepped back) is never a leftover.
|
||||
fn marker_is_young(&self, name: &str) -> bool {
|
||||
self.modified(name)
|
||||
.ok()
|
||||
.is_some_and(|modified| modified.elapsed().map_or(true, |age| age < MULTIPART_INDEX_PRUNE_GRACE))
|
||||
}
|
||||
fn remove_file(&self, name: &str) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
@@ -280,9 +271,12 @@ impl IndexOwner {
|
||||
initialized: OnceLock::new(),
|
||||
driving: AtomicBool::new(false),
|
||||
disabled: AtomicBool::new(false),
|
||||
publishing: parking_lot::Mutex::new(HashMap::new()),
|
||||
#[cfg(test)]
|
||||
rebuild_head_pause: parking_lot::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
publication_pause: parking_lot::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
observed_rebuild_queue: parking_lot::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
rebuild_single_job: AtomicBool::new(false),
|
||||
@@ -394,8 +388,8 @@ impl IndexOwner {
|
||||
// The index is boot-local: every start rescans the staging area
|
||||
// before serving it, so a marker lost to a crash is recreated before
|
||||
// it can be consulted. Creation is enough; no fsync of the marker or
|
||||
// its ancestors. An existing marker is touched so pruning sees the
|
||||
// new upload's age, not the previous one's.
|
||||
// its ancestors. An existing marker needs nothing: the caller's
|
||||
// publication pin, not the marker's age, keeps it from being pruned.
|
||||
match openat(
|
||||
&parent,
|
||||
source_root,
|
||||
@@ -407,16 +401,7 @@ impl IndexOwner {
|
||||
return Err(io::Error::other("index marker is not a file"));
|
||||
}
|
||||
}
|
||||
Err(rustix::io::Errno::EXIST) => {
|
||||
// Best effort: a filesystem that refuses to set times keeps
|
||||
// the previous upload's age, which only shortens the prune
|
||||
// grace for this key; index completeness does not depend on it.
|
||||
if let Ok(marker) =
|
||||
openat(&parent, source_root, OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::empty())
|
||||
{
|
||||
let _ = std::fs::File::from(marker).set_modified(std::time::SystemTime::now());
|
||||
}
|
||||
}
|
||||
Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
@@ -432,6 +417,22 @@ impl IndexOwner {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Declare that the caller is about to publish the staging root of
|
||||
/// `marker`. Must precede the marker lease the caller records under, so a
|
||||
/// prune that found no pin has released that lease before the marker is
|
||||
/// (re)created.
|
||||
fn pin_publication(self: &Arc<Self>, marker: PathBuf) -> PublicationPin {
|
||||
*self.publishing.lock().entry(marker.clone()).or_default() += 1;
|
||||
PublicationPin {
|
||||
owner: Arc::clone(self),
|
||||
marker,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_publishing(&self, marker: &Path) -> bool {
|
||||
self.publishing.lock().contains_key(marker)
|
||||
}
|
||||
|
||||
fn ensure_current_root(&self) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
let current = {
|
||||
@@ -513,9 +514,12 @@ impl IndexOwner {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn record_info(self: &Arc<Self>, path: &str, info: &FileInfo) -> Result<()> {
|
||||
/// Record the marker of `path`'s staging root. The returned pin must stay
|
||||
/// alive until the filesystem work that publishes the root has returned,
|
||||
/// including work that outlives a cancelled caller.
|
||||
pub(super) async fn record_info(self: &Arc<Self>, path: &str, info: &FileInfo) -> Result<Option<PublicationPin>> {
|
||||
if canonical_upload(path).is_none() {
|
||||
return Ok(());
|
||||
return Ok(None);
|
||||
}
|
||||
let owner = self.clone();
|
||||
let path = path.to_owned();
|
||||
@@ -531,6 +535,7 @@ impl IndexOwner {
|
||||
.join(path.split('/').next().unwrap_or_default())
|
||||
})
|
||||
.unwrap_or_else(|| owner.base.join(INDEX));
|
||||
let pin = self.pin_publication(marker.clone());
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
match owner.record(&path, &info) {
|
||||
@@ -544,7 +549,7 @@ impl IndexOwner {
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
Ok(())
|
||||
Ok(Some(pin))
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Option<PreparedHeadRecord> {
|
||||
@@ -578,20 +583,23 @@ impl IndexOwner {
|
||||
.join(upload.split('/').next().unwrap_or_default())
|
||||
})
|
||||
.unwrap_or_else(|| self.base.join(INDEX));
|
||||
let pin = self.pin_publication(marker.clone());
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
|
||||
Some(PreparedHeadRecord {
|
||||
owner: Arc::clone(self),
|
||||
upload: upload.to_owned(),
|
||||
info,
|
||||
pin,
|
||||
_lease: lease,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
/// Record the marker of the staging root that `path` belongs to. See
|
||||
/// [`Self::record_info`] for how long the returned pin must live.
|
||||
pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<Option<PublicationPin>> {
|
||||
let Some(prepared) = self.prepare_record_head(path, bytes).await else {
|
||||
return Ok(());
|
||||
return Ok(None);
|
||||
};
|
||||
prepared.run().await
|
||||
prepared.run().await.map(Some)
|
||||
}
|
||||
|
||||
async fn page(self: &Arc<Self>, bucket: &str, marker: Option<&str>, disk_id: Uuid) -> Result<MultipartIndexPage> {
|
||||
@@ -642,17 +650,20 @@ impl IndexOwner {
|
||||
// to every key ever multipart-initiated on this drive.
|
||||
let mut live = Vec::with_capacity(roots.len());
|
||||
let mut stale = Vec::new();
|
||||
let markers = owner.base.join(INDEX).join(sha(&bucket));
|
||||
for root in roots {
|
||||
// The marker precedes its staging root: a pinned one is an
|
||||
// upload in flight and stays a candidate. The pin is read
|
||||
// before the root, so a publisher that finished in between has
|
||||
// already made its root visible.
|
||||
if owner.is_publishing(&markers.join(&root)) {
|
||||
live.push(root);
|
||||
continue;
|
||||
}
|
||||
match owner.directory(&Path::new(RUSTFS_META_MULTIPART_BUCKET).join(&root)) {
|
||||
Ok(_) => live.push(root),
|
||||
Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) => {
|
||||
// The marker precedes its staging root; a young one is an
|
||||
// upload in flight and stays a candidate.
|
||||
if directory.as_ref().is_some_and(|index| index.marker_is_young(&root)) {
|
||||
live.push(root);
|
||||
} else {
|
||||
stale.push(root);
|
||||
}
|
||||
stale.push(root);
|
||||
}
|
||||
Err(error) => return Err(DiskError::from(error)),
|
||||
}
|
||||
@@ -677,9 +688,11 @@ impl IndexOwner {
|
||||
}
|
||||
|
||||
/// Remove one marker whose staging root was gone when a page examined it.
|
||||
/// The marker's own namespace lease serializes this with `record_info`; the
|
||||
/// staging root and the marker's age are re-checked under that lease, so an
|
||||
/// upload recorded in between keeps its marker.
|
||||
/// The marker's own namespace lease serializes this with recording. Under
|
||||
/// that lease the publication pin is read first and the staging root
|
||||
/// second: a publisher pinned before the read keeps the marker, one that
|
||||
/// already finished has made its root visible, and one that pins later
|
||||
/// records its marker only after this lease is released.
|
||||
async fn prune_marker(self: &Arc<Self>, bucket: &str, root: &str) -> Result<()> {
|
||||
let owner = self.clone();
|
||||
let bucket_directory = Path::new(INDEX).join(sha(bucket));
|
||||
@@ -690,6 +703,9 @@ impl IndexOwner {
|
||||
// not queue behind every other mutation on the one-worker foreground lane.
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
owner.ensure_current_root()?;
|
||||
if owner.is_publishing(&marker) {
|
||||
return Ok(());
|
||||
}
|
||||
match owner.directory(&Path::new(RUSTFS_META_MULTIPART_BUCKET).join(&root)) {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) => {}
|
||||
@@ -700,9 +716,6 @@ impl IndexOwner {
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if directory.marker_is_young(&root) {
|
||||
return Ok(());
|
||||
}
|
||||
match directory.remove_file(&root) {
|
||||
Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error),
|
||||
_ => Ok(()),
|
||||
@@ -713,34 +726,69 @@ impl IndexOwner {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PreparedHeadRecord {
|
||||
/// Proof that the staging root of one marker is being published. While any pin
|
||||
/// for a marker is alive, pruning leaves that marker alone.
|
||||
#[must_use = "dropping the pin lets a concurrent listing prune the marker before its staging root exists"]
|
||||
pub(super) struct PublicationPin {
|
||||
owner: Arc<IndexOwner>,
|
||||
marker: PathBuf,
|
||||
}
|
||||
|
||||
impl PublicationPin {
|
||||
/// Test seam between recording the marker and publishing the staging root.
|
||||
#[cfg(test)]
|
||||
pub(super) fn pause_for_test(&self) {
|
||||
let pause = self.owner.publication_pause.lock().take();
|
||||
if let Some(pause) = pause {
|
||||
let _pause = os::enter_blocking_test_hook();
|
||||
let _ = pause.started.send(());
|
||||
let _ = pause.release.recv_timeout(std::time::Duration::from_secs(10));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PublicationPin {
|
||||
fn drop(&mut self) {
|
||||
let mut publishing = self.owner.publishing.lock();
|
||||
if let Some(count) = publishing.get_mut(&self.marker) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
publishing.remove(&self.marker);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PreparedHeadRecord {
|
||||
upload: String,
|
||||
info: FileInfo,
|
||||
pin: PublicationPin,
|
||||
_lease: Arc<os::NamespaceMutationLease>,
|
||||
}
|
||||
|
||||
impl PreparedHeadRecord {
|
||||
pub(super) fn run_std(self) -> io::Result<()> {
|
||||
/// Record the marker and release its lease. The caller publishes the
|
||||
/// staging root while it holds the returned pin.
|
||||
pub(super) fn run_std(self) -> io::Result<PublicationPin> {
|
||||
let Self {
|
||||
owner,
|
||||
upload,
|
||||
info,
|
||||
pin,
|
||||
_lease,
|
||||
} = self;
|
||||
match owner.record(&upload, &info) {
|
||||
match pin.owner.record(&upload, &info) {
|
||||
Err(error) if error.kind() == io::ErrorKind::InvalidData => {
|
||||
owner.disabled.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
pin.owner.disabled.store(true, Ordering::Release);
|
||||
Ok(pin)
|
||||
}
|
||||
result => result,
|
||||
Err(error) => Err(error),
|
||||
Ok(()) => Ok(pin),
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(self) -> Result<()> {
|
||||
async fn run(self) -> Result<PublicationPin> {
|
||||
let lease = Arc::clone(&self._lease);
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || self.run_std()).await?;
|
||||
Ok(())
|
||||
Ok(os::run_blocking_namespace_operation_with_sync(lease, true, move || self.run_std()).await?)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -958,47 +1006,274 @@ mod tests {
|
||||
let live = format!("{:064x}", 1);
|
||||
let finished = format!("{:064x}", 2);
|
||||
let in_flight = format!("{:064x}", 3);
|
||||
let clock_stepped = format!("{:064x}", 4);
|
||||
for name in [&live, &finished, &in_flight, &clock_stepped] {
|
||||
for name in [&live, &finished, &in_flight] {
|
||||
std::fs::write(directory.join(name), b"1").expect("marker");
|
||||
}
|
||||
// Completion and abort remove the staging root and leave the marker.
|
||||
std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&live)).expect("live staging root");
|
||||
set_marker_modified(&directory.join(&finished), std::time::SystemTime::now() - MULTIPART_INDEX_PRUNE_GRACE * 2);
|
||||
// A marker dated after a backwards clock step is never a leftover.
|
||||
set_marker_modified(
|
||||
&directory.join(&clock_stepped),
|
||||
std::time::SystemTime::now() + MULTIPART_INDEX_PRUNE_GRACE * 2,
|
||||
);
|
||||
let page = disk.multipart_index.page("bucket", None, Uuid::new_v4()).await.expect("page");
|
||||
// The in-flight marker has no staging root yet but is young: a candidate, not a leftover.
|
||||
assert_eq!(page.roots, vec![live.clone(), in_flight.clone(), clock_stepped.clone()]);
|
||||
let owner = &disk.multipart_index;
|
||||
let publishing = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&in_flight));
|
||||
let second_publisher = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&in_flight));
|
||||
let page = owner.page("bucket", None, Uuid::new_v4()).await.expect("page");
|
||||
// The in-flight marker has no staging root yet but is pinned: a candidate, not a leftover.
|
||||
assert_eq!(page.roots, vec![live.clone(), in_flight.clone()]);
|
||||
assert!(page.next_marker.is_none());
|
||||
assert!(directory.join(&live).exists(), "a live root keeps its marker");
|
||||
assert!(directory.join(&in_flight).exists(), "a marker inside the grace period is kept");
|
||||
assert!(directory.join(&clock_stepped).exists(), "a future-dated marker is kept");
|
||||
assert!(!directory.join(&finished).exists(), "a finished root's marker is pruned");
|
||||
// The next page no longer pays for the pruned marker.
|
||||
let page = disk
|
||||
.multipart_index
|
||||
assert!(directory.join(&in_flight).exists(), "a pinned marker is kept");
|
||||
assert!(!directory.join(&finished).exists(), "a finished root's marker is pruned at once");
|
||||
// One of two publishers of the same root finishing does not unpin it.
|
||||
drop(publishing);
|
||||
let page = owner
|
||||
.page("bucket", None, Uuid::new_v4())
|
||||
.await
|
||||
.expect("page after pruning");
|
||||
assert_eq!(page.roots, vec![live, in_flight, clock_stepped]);
|
||||
.expect("page with one publisher left");
|
||||
assert_eq!(page.roots, vec![live.clone(), in_flight.clone()]);
|
||||
assert!(directory.join(&in_flight).exists(), "the remaining publisher keeps the marker");
|
||||
// A publication that ended without a staging root leaves a leftover.
|
||||
drop(second_publisher);
|
||||
let page = owner
|
||||
.page("bucket", None, Uuid::new_v4())
|
||||
.await
|
||||
.expect("page after the failed publication");
|
||||
assert_eq!(page.roots, vec![live]);
|
||||
assert!(!directory.join(&in_flight).exists(), "an unpinned marker without a root is pruned");
|
||||
assert!(owner.publishing.lock().is_empty(), "finished publishers leave no pin behind");
|
||||
}
|
||||
|
||||
/// Move a marker's modification time outside the publication grace period.
|
||||
fn age_marker(marker: &Path) {
|
||||
set_marker_modified(marker, std::time::SystemTime::now() - MULTIPART_INDEX_PRUNE_GRACE * 2);
|
||||
fn pause_next_publication(owner: &IndexOwner) -> (tokio::sync::oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
|
||||
let (started, waiting) = tokio::sync::oneshot::channel();
|
||||
let (release, released) = std::sync::mpsc::channel();
|
||||
*owner.publication_pause.lock() = Some(RebuildHeadPause {
|
||||
started,
|
||||
release: released,
|
||||
});
|
||||
(waiting, release)
|
||||
}
|
||||
|
||||
fn set_marker_modified(marker: &Path, modified: std::time::SystemTime) {
|
||||
/// The window the pin closes: the marker is recorded and its lease is
|
||||
/// released, the staging root is not published yet, and a listing examines
|
||||
/// the bucket. The marker is dated a day back so that no grace period could
|
||||
/// be what keeps it.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_index_listing_during_source_publication_keeps_the_new_upload_listed() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = Arc::new(open(root.path()).await);
|
||||
let (path, info) = upload("bucket", "published-while-listed");
|
||||
let source_root = path.split('/').next().expect("source sha").to_owned();
|
||||
let marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
|
||||
let source_head = root
|
||||
.path()
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&path)
|
||||
.join(STORAGE_FORMAT_FILE);
|
||||
let (paused, release) = pause_next_publication(&disk.multipart_index);
|
||||
let writer = tokio::spawn({
|
||||
let disk = Arc::clone(&disk);
|
||||
let path = path.clone();
|
||||
async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), paused)
|
||||
.await
|
||||
.expect("the publication reaches the pause")
|
||||
.expect("pause sender");
|
||||
assert!(marker.exists(), "the marker precedes the staging root");
|
||||
assert!(!source_head.exists(), "the staging root is not published yet");
|
||||
std::fs::File::options()
|
||||
.write(true)
|
||||
.open(marker)
|
||||
.open(&marker)
|
||||
.expect("marker")
|
||||
.set_modified(modified)
|
||||
.expect("set marker time");
|
||||
.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(86_400))
|
||||
.expect("age the marker");
|
||||
|
||||
let identity = Uuid::new_v4();
|
||||
let page = disk
|
||||
.multipart_index
|
||||
.page("bucket", None, identity)
|
||||
.await
|
||||
.expect("page during publication");
|
||||
assert_eq!(page.roots, vec![source_root.clone()], "an upload in flight stays a candidate");
|
||||
assert!(marker.exists(), "a listing must not prune the marker of an upload in flight");
|
||||
|
||||
release.send(()).expect("resume the publication");
|
||||
writer.await.expect("writer task").expect("the upload metadata is published");
|
||||
assert!(source_head.exists());
|
||||
let page = disk
|
||||
.multipart_index
|
||||
.page("bucket", None, identity)
|
||||
.await
|
||||
.expect("page after publication");
|
||||
assert_eq!(page.roots, vec![source_root], "the published upload is listed");
|
||||
assert!(marker.exists());
|
||||
assert!(disk.multipart_index.publishing.lock().is_empty());
|
||||
}
|
||||
|
||||
/// A timed-out caller drops its future while the filesystem work goes on.
|
||||
/// The pin belongs to that work, so the marker survives until the staging
|
||||
/// root the detached work publishes is visible.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_index_cancelled_publisher_keeps_its_marker_until_the_detached_work_returns() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = Arc::new(open(root.path()).await);
|
||||
let (path, info) = upload("bucket", "caller-timed-out");
|
||||
let source_root = path.split('/').next().expect("source sha").to_owned();
|
||||
let marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
|
||||
let source_head = root
|
||||
.path()
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&path)
|
||||
.join(STORAGE_FORMAT_FILE);
|
||||
let (paused, release) = pause_next_publication(&disk.multipart_index);
|
||||
let writer = tokio::spawn({
|
||||
let disk = Arc::clone(&disk);
|
||||
let path = path.clone();
|
||||
async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), paused)
|
||||
.await
|
||||
.expect("the publication reaches the pause")
|
||||
.expect("pause sender");
|
||||
writer.abort();
|
||||
assert!(writer.await.expect_err("the caller is gone").is_cancelled());
|
||||
|
||||
let identity = Uuid::new_v4();
|
||||
let page = disk
|
||||
.multipart_index
|
||||
.page("bucket", None, identity)
|
||||
.await
|
||||
.expect("page after the caller left");
|
||||
assert_eq!(page.roots, vec![source_root.clone()]);
|
||||
assert!(marker.exists(), "the detached publication still owns the marker");
|
||||
|
||||
release.send(()).expect("resume the detached publication");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
while !source_head.exists() || !disk.multipart_index.publishing.lock().is_empty() {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the detached publication completes");
|
||||
let page = disk
|
||||
.multipart_index
|
||||
.page("bucket", None, identity)
|
||||
.await
|
||||
.expect("page after the detached publication");
|
||||
assert_eq!(page.roots, vec![source_root], "the upload the detached work published is listed");
|
||||
assert!(marker.exists());
|
||||
}
|
||||
|
||||
/// A page classifies a marker as stale before the publisher pins it; the
|
||||
/// prune that follows must read the pin again under the marker lease.
|
||||
#[tokio::test]
|
||||
async fn multipart_index_prune_rechecks_the_pin_under_the_marker_lease() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = open(root.path()).await;
|
||||
let owner = &disk.multipart_index;
|
||||
let directory = root.path().join(INDEX).join(sha("bucket"));
|
||||
std::fs::create_dir_all(&directory).expect("marker directory");
|
||||
let name = format!("{:064x}", 9);
|
||||
std::fs::write(directory.join(&name), b"1").expect("marker");
|
||||
|
||||
let pin = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&name));
|
||||
owner.prune_marker("bucket", &name).await.expect("prune of a pinned marker");
|
||||
assert!(directory.join(&name).exists(), "a publisher that pinned after the page keeps its marker");
|
||||
|
||||
drop(pin);
|
||||
owner.prune_marker("bucket", &name).await.expect("prune of a leftover");
|
||||
assert!(!directory.join(&name).exists(), "the same marker is a leftover once nobody publishes it");
|
||||
}
|
||||
|
||||
/// The publisher pins before it queues on the marker lease. A prune that
|
||||
/// holds the lease and found no pin has therefore unlinked the marker
|
||||
/// before the publisher records it, and the record recreates it.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_index_publisher_pins_before_it_waits_for_the_marker_lease() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = Arc::new(open(root.path()).await);
|
||||
let owner = Arc::clone(&disk.multipart_index);
|
||||
let (path, info) = upload("bucket", "queued-behind-a-prune");
|
||||
let source_root = path.split('/').next().expect("source sha").to_owned();
|
||||
let marker = owner.base.join(INDEX).join(sha("bucket")).join(&source_root);
|
||||
let visible_marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
|
||||
|
||||
// Stand in for a prune that owns the marker lease and already read "no pin".
|
||||
let prune_lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
|
||||
assert!(!owner.is_publishing(&marker));
|
||||
let writer = tokio::spawn({
|
||||
let disk = Arc::clone(&disk);
|
||||
let path = path.clone();
|
||||
async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while !owner.is_publishing(&marker) {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(2)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the publisher pins while the marker lease is still held by the prune");
|
||||
assert!(!visible_marker.exists(), "the marker is recorded only under the lease");
|
||||
|
||||
drop(prune_lease);
|
||||
writer.await.expect("writer task").expect("the upload metadata is published");
|
||||
assert!(visible_marker.exists(), "the record after the prune recreates the marker");
|
||||
let page = owner.page("bucket", None, Uuid::new_v4()).await.expect("page");
|
||||
assert_eq!(page.roots, vec![source_root]);
|
||||
}
|
||||
|
||||
/// Stale markers at the start of a page, at its cursor, and filling a whole
|
||||
/// page: every page still satisfies the receiver's contract and advances.
|
||||
#[tokio::test]
|
||||
async fn multipart_index_pages_of_stale_markers_keep_valid_cursors() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = open(root.path()).await;
|
||||
let directory = root.path().join(INDEX).join(sha("bucket"));
|
||||
std::fs::create_dir_all(&directory).expect("marker directory");
|
||||
// Page one is entirely stale. Page two is live except its first name
|
||||
// and the name its cursor lands on. Two live names follow.
|
||||
let total = MULTIPART_INDEX_PAGE_SIZE * 2 + 2;
|
||||
let stale = |i: usize| i <= MULTIPART_INDEX_PAGE_SIZE || i == MULTIPART_INDEX_PAGE_SIZE * 2 - 1;
|
||||
for i in 0..total {
|
||||
let name = format!("{i:064x}");
|
||||
std::fs::write(directory.join(&name), b"1").expect("marker");
|
||||
if !stale(i) {
|
||||
std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&name)).expect("staging root");
|
||||
}
|
||||
}
|
||||
let disk_id = Uuid::new_v4();
|
||||
let first = disk.multipart_index.page("bucket", None, disk_id).await.expect("first page");
|
||||
first
|
||||
.validate("bucket", None, disk_id)
|
||||
.expect("an empty page with a cursor validates");
|
||||
assert!(first.roots.is_empty(), "a fully stale page has no roots");
|
||||
let cursor = first.next_marker.clone().expect("more markers follow a fully stale page");
|
||||
assert_eq!(cursor, format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1));
|
||||
|
||||
let second = disk
|
||||
.multipart_index
|
||||
.page("bucket", Some(&cursor), disk_id)
|
||||
.await
|
||||
.expect("second page");
|
||||
second
|
||||
.validate("bucket", Some(&cursor), disk_id)
|
||||
.expect("stale first and cursor names validate");
|
||||
assert_eq!(second.roots.len(), MULTIPART_INDEX_PAGE_SIZE - 2);
|
||||
assert_eq!(second.roots.first(), Some(&format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE + 1)));
|
||||
let cursor = second
|
||||
.next_marker
|
||||
.clone()
|
||||
.expect("the cursor is the stale last examined name");
|
||||
assert_eq!(cursor, format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE * 2 - 1));
|
||||
assert!(second.roots.last() < Some(&cursor));
|
||||
|
||||
let third = disk
|
||||
.multipart_index
|
||||
.page("bucket", Some(&cursor), disk_id)
|
||||
.await
|
||||
.expect("third page");
|
||||
third.validate("bucket", Some(&cursor), disk_id).expect("last page validates");
|
||||
assert_eq!(third.roots.len(), 2);
|
||||
assert!(third.next_marker.is_none());
|
||||
let remaining = std::fs::read_dir(&directory).expect("marker directory").count();
|
||||
assert_eq!(remaining, total - MULTIPART_INDEX_PAGE_SIZE - 2, "every stale marker is pruned");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1015,7 +1290,6 @@ mod tests {
|
||||
std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&name)).expect("staging root");
|
||||
}
|
||||
}
|
||||
age_marker(&directory.join(&stale));
|
||||
let disk_id = Uuid::new_v4();
|
||||
let first = disk.multipart_index.page("bucket", None, disk_id).await.expect("first page");
|
||||
first
|
||||
|
||||
@@ -813,7 +813,8 @@ impl SourceStore {
|
||||
Ok(())
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
self.multipart_index
|
||||
let index_pin = self
|
||||
.multipart_index
|
||||
.record_head(&format!("{}/xl.meta", self.path), bytes)
|
||||
.await?;
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
@@ -822,6 +823,7 @@ impl SourceStore {
|
||||
let root = self.publication_root.clone();
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
let _index_pin = index_pin;
|
||||
os::replace_multipart_source_metadata(&path, &bytes, &base, &root)
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -2585,7 +2585,13 @@ pub struct BoundedDeleteRecoveryDiscovery {
|
||||
pub complete: bool,
|
||||
}
|
||||
|
||||
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
|
||||
/// Entries one bounded discovery call may charge on one drive. Object-scoped
|
||||
/// discovery charges each transaction-shaped (UUID) child of the key's
|
||||
/// directory once and every name that cannot be delete evidence at
|
||||
/// 1/`PLAIN_CHILDREN_PER_BUDGET_ENTRY`, so this admits a missing-key delete of a
|
||||
/// folder prefix with this many UUID-named children per drive. The request
|
||||
/// budget (`DeleteRecoveryWorkBudget`) still bounds the total across keys.
|
||||
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 262_144;
|
||||
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
||||
@@ -975,9 +975,11 @@ pub struct DeleteRecoveryWorkBudget {
|
||||
remaining: std::sync::atomic::AtomicU32,
|
||||
}
|
||||
|
||||
/// Entries one disk may visit per recovery step (discovery plus inspection).
|
||||
/// Every disk of every set a request fans out to is admitted up to this much
|
||||
/// and charged on completion, so a budget must cover that many disks.
|
||||
/// Per-drive unit a request budget is sized by (a typical discovery plus one
|
||||
/// inspection). Admission charges what a call visited, not this unit, so it only
|
||||
/// scales the request total with the drives a request can reach; it is kept
|
||||
/// independent of the per-call caps, which may let one key's probe use more of
|
||||
/// the same total on a large folder.
|
||||
const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
|
||||
|
||||
impl Default for DeleteRecoveryWorkBudget {
|
||||
|
||||
@@ -11040,7 +11040,12 @@ mod tests {
|
||||
let live = Uuid::new_v4();
|
||||
let orphan = Uuid::new_v4();
|
||||
let object_dir = dir.path().join("bucket/obj");
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
|
||||
// Part files are inert to discovery and cost 1/64 of an entry, so the
|
||||
// first step is exhausted by what still costs a full entry each: the
|
||||
// data directories of many versions, all referenced by the metadata.
|
||||
let versions = (0..4200).map(|_| Uuid::new_v4()).collect::<Vec<_>>();
|
||||
let referenced = std::iter::once(live).chain(versions.iter().copied()).collect::<Vec<_>>();
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &referenced).await;
|
||||
let live_dir = object_dir.join(live.to_string());
|
||||
let orphan_dir = object_dir.join(orphan.to_string());
|
||||
fs::create_dir_all(&live_dir).await.expect("create live multipart directory");
|
||||
@@ -11049,10 +11054,14 @@ mod tests {
|
||||
.await
|
||||
.expect("write orphan payload");
|
||||
let parts_dir = live_dir.clone();
|
||||
let versions_root = object_dir.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
for part in 1..=4200 {
|
||||
std::fs::write(parts_dir.join(format!("part.{part}")), b"live").expect("write live multipart entry");
|
||||
}
|
||||
for version in versions {
|
||||
std::fs::create_dir(versions_root.join(version.to_string())).expect("create referenced version directory");
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("create multipart fixture");
|
||||
@@ -11062,7 +11071,10 @@ mod tests {
|
||||
.await
|
||||
.expect("read first discovery page");
|
||||
assert!(first.transactions.is_empty());
|
||||
assert!(first.next.is_some(), "ordinary multipart entries must exhaust the first scan budget");
|
||||
assert!(
|
||||
first.next.is_some(),
|
||||
"the data directories of many versions must exhaust the first scan step"
|
||||
);
|
||||
let last = disk
|
||||
.discover_delete_transactions("bucket", Some("obj"), first.next, 1)
|
||||
.await
|
||||
|
||||
@@ -1709,6 +1709,37 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_missing_key_naming_a_folder_of_uuid_named_keys_completes() {
|
||||
let bucket = "recover-uuid-named-folder";
|
||||
let (dirs, store, _) = fixture(bucket).await;
|
||||
// More UUID-named child keys per drive than the former 4096-entry probe
|
||||
// could cover once each child's `xl.meta` and data directory were charged
|
||||
// (three entries per child).
|
||||
const CHILDREN: usize = 1500;
|
||||
for dir in &dirs {
|
||||
let prefix = dir.path().join(bucket).join("uploads");
|
||||
for _ in 0..CHILDREN {
|
||||
let child = prefix.join(Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(child.join(Uuid::new_v4().to_string())).expect("child key data dir");
|
||||
std::fs::write(child.join(crate::disk::STORAGE_FORMAT_FILE), b"meta").expect("child key metadata");
|
||||
}
|
||||
}
|
||||
let error = store
|
||||
.delete_object(bucket, "uploads", ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the folder's own key does not exist");
|
||||
assert!(
|
||||
crate::error::is_err_object_not_found(&error),
|
||||
"a complete inventory keeps the missing-key no-op semantics: {error:?}"
|
||||
);
|
||||
assert!(dirs.iter().all(|dir| {
|
||||
std::fs::read_dir(dir.path().join(bucket).join("uploads"))
|
||||
.map(|entries| entries.count() == CHILDREN)
|
||||
.unwrap_or(false)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
|
||||
@@ -425,7 +425,10 @@ enum FolderScanSource {
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum FolderScanMode {
|
||||
Namespace,
|
||||
ErasureCandidate { reuse_namespace: bool },
|
||||
ErasureCandidate {
|
||||
reuse_namespace: bool,
|
||||
parent_has_metadata: bool,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
@@ -2065,7 +2068,9 @@ impl FolderScanner {
|
||||
if candidate_unclassified {
|
||||
// Apply namespace reuse only after the current listing
|
||||
// proves there is no direct erasure part.
|
||||
if let FolderScanMode::ErasureCandidate { reuse_namespace: true } = mode
|
||||
if let FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: true, ..
|
||||
} = mode
|
||||
&& !into.compacted
|
||||
&& self.old_cache.find(&folder.name).is_some()
|
||||
{
|
||||
@@ -2079,10 +2084,23 @@ impl FolderScanner {
|
||||
return Ok(FolderScanOutcome::Namespace);
|
||||
}
|
||||
candidate_unclassified = false;
|
||||
if !self.budget.try_start_directory() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
// An empty UUID directory beside object metadata has
|
||||
// no namespace work. Its listing was already charged;
|
||||
// nonempty namespaces still need directory admission.
|
||||
let empty_object_child = matches!(
|
||||
mode,
|
||||
FolderScanMode::ErasureCandidate {
|
||||
parent_has_metadata: true,
|
||||
..
|
||||
}
|
||||
) && deferred_metadata.is_none()
|
||||
&& queued_folders.is_empty();
|
||||
if !empty_object_child {
|
||||
if !self.budget.try_start_directory() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
wait_for_scanner_io(&ctx, self.sleeper.sleep_folder_preset()).await?;
|
||||
}
|
||||
wait_for_scanner_io(&ctx, self.sleeper.sleep_folder_preset()).await?;
|
||||
}
|
||||
let Some(entry) = deferred_metadata.take() else { break };
|
||||
(entry, true)
|
||||
@@ -2552,20 +2570,6 @@ impl FolderScanner {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
|
||||
if found_object_metadata && self.is_erasure_mode {
|
||||
// If we found an object in erasure mode, we skip subdirs (only datadirs)...
|
||||
debug!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
folder = %folder.name,
|
||||
state = "erasure_object_found",
|
||||
"Scanner folder descent stopped after erasure object"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
// Checkpointed sweeps need child entries to validate their resume
|
||||
// frontier (#8292). Outside that, also compact a wide bucket root:
|
||||
// excluding it left #8961's ~170k top-level prefixes unbounded until
|
||||
@@ -2654,6 +2658,7 @@ impl FolderScanner {
|
||||
let mut child_mode = if self.is_erasure_mode && is_erasure_candidate(&folder_item.name) {
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: reuse_covered,
|
||||
parent_has_metadata: found_object_metadata,
|
||||
}
|
||||
} else {
|
||||
FolderScanMode::Namespace
|
||||
@@ -2721,7 +2726,10 @@ impl FolderScanner {
|
||||
}
|
||||
// Sampling may skip namespace work, never the
|
||||
// classification that protects its parent object.
|
||||
child_mode = FolderScanMode::ErasureCandidate { reuse_namespace: true };
|
||||
child_mode = FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: true,
|
||||
parent_has_metadata: found_object_metadata,
|
||||
};
|
||||
} else {
|
||||
folder_item.object_heal_prob_div = data_usage_update_dir_cycles();
|
||||
}
|
||||
@@ -2816,6 +2824,11 @@ impl FolderScanner {
|
||||
updates.cache.delete_recursive(&h);
|
||||
}
|
||||
into.children.remove(&clone_cache_value(&h.0));
|
||||
// Object keys can also be prefixes. A classified payload
|
||||
// directory must not hide their other namespace children.
|
||||
if found_object_metadata {
|
||||
continue;
|
||||
}
|
||||
// Earlier namespace siblings may have completed, but none
|
||||
// can certify coverage of this unresolved parent object.
|
||||
self.coverage_gap |= forward_sweep;
|
||||
|
||||
@@ -183,6 +183,117 @@ fn charged_ops(root: &std::path::Path) -> u64 {
|
||||
status.charged_ops
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_object_metadata_preserves_child_object_namespaces() {
|
||||
for (corrupt_parent, cached_parent) in [(false, false), (true, false), (false, true)] {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, root.clone());
|
||||
scanner.old_cache.info.name = "bucket".into();
|
||||
scanner.new_cache.info.name = "bucket".into();
|
||||
scanner.is_erasure_mode = true;
|
||||
bind_budget(&mut scanner, &root, 1_000_000);
|
||||
|
||||
let namespace = format!("parent/{}/nested", Uuid::new_v4());
|
||||
for object in ["parent", "parent/child", namespace.as_str()] {
|
||||
write_test_object_metadata_bytes(&root, "bucket", object, &metadata_for_object_version("bucket", object, None)).await;
|
||||
}
|
||||
let payload = root.join("bucket/parent").join(Uuid::new_v4().to_string());
|
||||
tokio::fs::create_dir_all(&payload)
|
||||
.await
|
||||
.expect("create parent payload directory");
|
||||
tokio::fs::write(payload.join("part.1"), b"shard")
|
||||
.await
|
||||
.expect("create actual payload evidence");
|
||||
if corrupt_parent {
|
||||
tokio::fs::write(root.join("bucket/parent/xl.meta"), b"")
|
||||
.await
|
||||
.expect("corrupt only the parent metadata");
|
||||
}
|
||||
if cached_parent {
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
"bucket/parent",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 1,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let interval = data_usage_update_dir_cycles();
|
||||
scanner.old_cache.info.next_cycle = u64::from(
|
||||
(0..interval)
|
||||
.find(|cycle| hash_path("bucket/parent").mod_(*cycle, interval))
|
||||
.expect("one normal sampling cycle revisits the old parent-only cache"),
|
||||
);
|
||||
}
|
||||
|
||||
let mut into = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".into(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut into,
|
||||
)
|
||||
.await
|
||||
.expect("scan object and child namespaces independently");
|
||||
let total = scanner.new_cache.size_recursive("bucket").expect("retain bucket accounting");
|
||||
assert_eq!(total.objects, if corrupt_parent { 2 } else { 3 });
|
||||
assert_eq!(total.size, if corrupt_parent { 2 } else { 3 });
|
||||
assert_eq!(total.failed_objects, usize::from(corrupt_parent));
|
||||
assert_eq!(
|
||||
scanner.new_cache.info.failed_objects.len(),
|
||||
usize::from(corrupt_parent),
|
||||
"payload evidence must not invent missing parent metadata"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_object_prefix_children_keep_the_object_budget() {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, root.clone());
|
||||
scanner.old_cache.info.name = "bucket".into();
|
||||
scanner.new_cache.info.name = "bucket".into();
|
||||
scanner.is_erasure_mode = true;
|
||||
bind_budget(&mut scanner, &root, 1_000_000);
|
||||
for object in ["parent", "parent/child", "parent/other"] {
|
||||
write_test_object_metadata(&root, "bucket", object).await;
|
||||
}
|
||||
let visits = metadata_visits::observe(&mut scanner);
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
crate::scanner_budget::ScannerCycleBudgetConfig {
|
||||
max_objects: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
scanner.budget = budget.clone();
|
||||
let result = scanner
|
||||
.scan_folder(
|
||||
budget.token(),
|
||||
CachedFolder {
|
||||
name: "bucket".into(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut DataUsageEntry::default(),
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "parent metadata cannot hide children or bypass their budget");
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
|
||||
assert_eq!(budget.progress().0, 2);
|
||||
assert_eq!(visits.load(Ordering::Relaxed), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
|
||||
@@ -204,7 +315,10 @@ async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
|
||||
ctx,
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: false,
|
||||
parent_has_metadata: false,
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
@@ -271,7 +385,10 @@ async fn scanner_directory_probe_budget_wait_cancels_without_rate_delay() {
|
||||
ctx.clone(),
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: false,
|
||||
parent_has_metadata: false,
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
@@ -311,7 +428,10 @@ async fn scanner_directory_probe_cancels_during_entry_budget_wait() {
|
||||
ctx.clone(),
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: false,
|
||||
parent_has_metadata: false,
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
@@ -382,7 +502,16 @@ async fn erasure_candidate_defers_metadata_until_part_classification_finishes()
|
||||
scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
|
||||
let mut into = DataUsageEntry::default();
|
||||
let outcome = scanner
|
||||
.scan_folder_inner(ctx, folder, &mut into, FolderScanMode::ErasureCandidate { reuse_namespace: false }, false)
|
||||
.scan_folder_inner(
|
||||
ctx,
|
||||
folder,
|
||||
&mut into,
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: false,
|
||||
parent_has_metadata: false,
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("classify candidate");
|
||||
assert_eq!(outcome, FolderScanOutcome::ErasureData);
|
||||
@@ -594,7 +723,10 @@ async fn erasure_candidate_part_symlink_is_not_data_evidence() {
|
||||
CancellationToken::new(),
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: false,
|
||||
parent_has_metadata: false,
|
||||
},
|
||||
false,
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -130,7 +130,7 @@ These behaviors are client-visible and differ from AWS S3 or MinIO. They bound h
|
||||
| Bucket tag key or value outside the AWS tag character class (`[\p{L}\p{N}\p{Z}+\-=._:/@]`) | `400 InvalidTag` on `PutBucketTagging` | AWS enforces the same class; MinIO is more permissive | AWS parity. Legacy stored tags still load. |
|
||||
| `CopyObject` with the default `x-amz-tagging-directive: COPY` from a source that carries tags | `403 AccessDenied` unless the caller also holds `s3:GetObjectTagging` (or `s3:GetObjectVersionTagging`) on the source; the destination's `s3:RequestObjectTag/*` conditions are evaluated against the copied tag set | AWS and MinIO copy the tags with `s3:GetObject` alone | A copy would otherwise disclose source tags to a principal that may not read them, and source tags could satisfy destination conditions the caller cannot assert (rustfs/backlog#3034). Use `REPLACE` with an explicit `x-amz-tagging` to copy without the tagging grant. |
|
||||
| `ListObjectVersions` / `ListObjectsV2` with the include-deleted extension | Requires `s3:ListBucketVersions` | Vendor extension | Delete markers and non-current versions are version-listing data. |
|
||||
| `DeleteObject` of a missing key that names a prefix | `204 NoContent` when the recovery probe of the prefix fits the request budget (`DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES`, 4096 entries per drive, one entry per 64 plain children); `503 SlowDown` beyond that, deterministically, and for the duration of a rolling upgrade while a drive in the set still runs a release without the bounded recovery RPC | `204 NoContent` | The probe looks for interrupted delete transactions under the prefix and is bounded so a hostile prefix cannot pin the request. Prefixes with more than 262,144 direct children per drive, or roughly 1,400 to 4,096 UUID-named children, exceed the bound (rustfs/backlog#3038 residual). |
|
||||
| `DeleteObject` of a missing key that names a prefix | `204 NoContent` when the recovery probe of the prefix fits its per-drive allowance (`DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES`, 262,144 entries: one per UUID-named direct child, one per 64 other names), its per-call time limit (a third of the drive timeout, at most 10 seconds) and the request budget (`DeleteRecoveryWorkBudget`, shared by every key and drive of the request); `503 SlowDown` beyond that, and for the duration of a rolling upgrade while a drive in the set still runs a release without the bounded recovery RPC | `204 NoContent` | The probe looks for interrupted delete transactions of the key and is bounded so a hostile prefix cannot pin the request. A folder completes with up to 262,144 UUID-named direct children or about 16.7 million other direct children per drive when the drive can walk them within the time limit (a warm cache walks roughly 25,000 UUID-named children per second; a cold HDD far fewer); inside a UUID-named child only journal files and data-directory markers are examined, so a child key's own `xl.meta`, data directories or children cost 1/64 each. |
|
||||
| `PutObject` / `UploadPart` body delivering less than 64 KiB per 60 seconds of server wait | `400 RequestTimeout` (the minimum-progress floor, logged as `upload_body_progress_stalled`); independent of `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT`, including when that timeout is 0 | No minimum upload rate | A trickle upload otherwise pins upload admission and its buffers indefinitely; roughly 1.1 KiB/s is below any practical client. |
|
||||
| First write after `SetBucketQuota`, or after the quota changes, on a bucket without an authoritative usage ledger | `503 SlowDown` (retryable) until a background scan of the bucket publishes the exact ledger; a failed scan retries after 30 seconds | AWS has no bucket quota; MinIO admits with unknown usage | Quota admission uses only the authoritative ledger (scanner reporting is informational, see `reporting_retirement` tests), and the request must not hold its bucket metadata guard across a whole-bucket listing. The window is one bucket listing long. |
|
||||
|
||||
|
||||
@@ -101,6 +101,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
|
||||
| `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes (disabled) | `make coverage` |
|
||||
| `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
|
||||
| `e2e-upgrade.yml` (weekly, and on version tags) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
|
||||
| `reference-parity.yml` (nightly, and on PRs that change the lane's own files) | `plan`, `build-candidate`, `s3tests` (one job per target in `.config/reference-parity-targets.toml`), `compare` | report-only parity gate: fails on any per-node-id s3-tests difference between a branch candidate and the release pinned in `.config/reference-binary.toml` that `.config/reference-parity-tolerated.txt` does not list, and on a stale tolerated entry; per-target JUnit, logs and `diff.md`/`diff.json` | yes | `DEPLOY_MODE=binary RUSTFS_BINARY=<binary> MAXFAIL=0 XDIST=4 ./scripts/s3-tests/run.sh` once per target, then `python3 scripts/s3-tests/reference_parity.py compare` as in the `compare` job |
|
||||
| `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | yes | see the PR row |
|
||||
| `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | yes (disabled) | see the PR row |
|
||||
| `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | yes | dispatch, optionally with `source_ref` |
|
||||
|
||||
@@ -3912,27 +3912,21 @@ mod tests {
|
||||
req
|
||||
}
|
||||
|
||||
fn usecase_method_source<'a>(source: &'a str, method: &str) -> &'a str {
|
||||
let public_start_marker = format!("pub async fn {method}");
|
||||
let private_start_marker = format!("async fn {method}");
|
||||
let (start, marker_len) = if let Some(start) = source.find(&public_start_marker) {
|
||||
(start, public_start_marker.len())
|
||||
} else {
|
||||
let start = source.find(&private_start_marker).expect("method should exist");
|
||||
(start, private_start_marker.len())
|
||||
};
|
||||
let rest = &source[start + marker_len..];
|
||||
let end = [rest.find("\n pub async fn "), rest.find("\n async fn ")]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.min()
|
||||
.unwrap_or(rest.len());
|
||||
&rest[..end]
|
||||
/// Production text of this file plus `bucket_usecase/`, where a split moves its methods.
|
||||
fn bucket_usecase_sources() -> crate::guard_sources::SourceSet {
|
||||
crate::guard_sources::SourceSet::production("src/app/bucket_usecase.rs", "src/app/bucket_usecase")
|
||||
}
|
||||
|
||||
/// The one `async fn method` of the set, through its own closing line, so a neighbour of any
|
||||
/// visibility or a test module can never satisfy a check about it.
|
||||
fn usecase_method_source<'a>(source: &'a crate::guard_sources::SourceSet, method: &str) -> &'a str {
|
||||
source.method_body(method)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_create_and_delete_use_admitted_fresh_tasks() {
|
||||
let source = include_str!("bucket_usecase.rs");
|
||||
let sources = bucket_usecase_sources();
|
||||
let source = &sources;
|
||||
for (method, operation, inner_call) in [
|
||||
(
|
||||
"execute_create_bucket",
|
||||
@@ -3957,7 +3951,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn bucket_metadata_config_changes_notify_peer_metadata_reload() {
|
||||
let source = include_str!("bucket_usecase.rs");
|
||||
let sources = bucket_usecase_sources();
|
||||
let source = &sources;
|
||||
for (method, operation, scanner_maintenance_change) in [
|
||||
("execute_delete_bucket_policy", "delete bucket policy", false),
|
||||
("execute_put_bucket_policy", "put bucket policy", false),
|
||||
|
||||
@@ -0,0 +1,752 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Test-only source reader for text guards that must survive a mechanical file split.
|
||||
//!
|
||||
//! A text guard asserts that some literal appears in production source. When it reads one file
|
||||
//! with `include_str!`, moving the guarded function into another file either breaks the guard or,
|
||||
//! worse, makes it stop seeing the code it guards. A guard built on [`SourceSet`] instead names
|
||||
//! the file it reads today plus the sibling directory a split moves that file's code into, and
|
||||
//! reads every `.rs` file of both at test time.
|
||||
//!
|
||||
//! The named file is required: a missing file panics with its path, so a guard never passes on
|
||||
//! absent input. The directory is optional while the split has not happened; once it exists,
|
||||
//! every `.rs` file under it (recursively) is read in byte order of its manifest-relative path,
|
||||
//! after the named file.
|
||||
//!
|
||||
//! [`SourceSet::production`] keeps production text only, because a guard about production code
|
||||
//! must not be satisfied by a test that merely mentions the guarded call: it leaves out files
|
||||
//! that are test-only by path (a `tests` directory, `tests.rs`, `*_test.rs`, `*_tests.rs`,
|
||||
//! `test_support.rs`, a file with `#![cfg(test)]`, or a child of a `#[cfg(test)] mod x;`) and
|
||||
//! removes every item annotated `#[cfg(test)]` or `#[cfg(all(test, ...))]`. The stripper relies on
|
||||
//! rustfmt's layout and panics, naming the file and line, on any shape it does not recognise
|
||||
//! instead of guessing which text to keep.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// One source file of a [`SourceSet`].
|
||||
pub(crate) struct SourceFile {
|
||||
/// Path relative to the `rustfs` package root, with `/` separators.
|
||||
pub(crate) path: String,
|
||||
pub(crate) text: String,
|
||||
}
|
||||
|
||||
/// The ordered files a text guard reads: one required file, then the split directory.
|
||||
pub(crate) struct SourceSet {
|
||||
files: Vec<SourceFile>,
|
||||
}
|
||||
|
||||
impl SourceSet {
|
||||
/// Production text of `file` and of every `.rs` file under `split_dir`, both relative to the
|
||||
/// `rustfs` package root.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics, naming the path, when `file` is missing, unreadable or test-only, when `split_dir`
|
||||
/// exists but is not a directory, when a file cannot be read, or when a `#[cfg(test)]` item
|
||||
/// has a shape the stripper does not recognise.
|
||||
pub(crate) fn production(file: &str, split_dir: &str) -> Self {
|
||||
Self::production_under(Path::new(env!("CARGO_MANIFEST_DIR")), file, split_dir)
|
||||
}
|
||||
|
||||
/// [`SourceSet::production`] under another package root, for guard fixtures.
|
||||
pub(crate) fn production_under(root: &Path, file: &str, split_dir: &str) -> Self {
|
||||
assert!(
|
||||
!is_test_only_path(file),
|
||||
"guard source {file} is a test-only file; a production guard cannot be about it"
|
||||
);
|
||||
let mut stripped = Vec::new();
|
||||
let mut test_module_paths = Vec::new();
|
||||
for source in Self::read_under(root, file, split_dir).files {
|
||||
if is_test_only_path(&source.path) {
|
||||
continue;
|
||||
}
|
||||
let Some((text, test_modules)) = strip_test_items(&source.path, &source.text) else {
|
||||
assert!(source.path != file, "guard source {file} is test-only (#![cfg(test)])");
|
||||
continue;
|
||||
};
|
||||
let children = child_module_dir(&source.path);
|
||||
test_module_paths.extend(test_modules.into_iter().map(|name| format!("{children}/{name}")));
|
||||
stripped.push(SourceFile { path: source.path, text });
|
||||
}
|
||||
stripped.retain(|source| {
|
||||
!test_module_paths
|
||||
.iter()
|
||||
.any(|module| source.path == format!("{module}.rs") || source.path.starts_with(&format!("{module}/")))
|
||||
});
|
||||
Self { files: stripped }
|
||||
}
|
||||
|
||||
fn read_under(root: &Path, file: &str, split_dir: &str) -> Self {
|
||||
let required = root.join(file);
|
||||
let text = std::fs::read_to_string(&required)
|
||||
.unwrap_or_else(|err| panic!("guard source {} must exist and be readable: {err}", required.display()));
|
||||
let mut files = vec![SourceFile {
|
||||
path: file.to_owned(),
|
||||
text,
|
||||
}];
|
||||
|
||||
let dir = root.join(split_dir);
|
||||
if dir.exists() {
|
||||
assert!(dir.is_dir(), "guard split path {} must be a directory", dir.display());
|
||||
let mut paths = Vec::new();
|
||||
collect_rust_files(&dir, &mut paths);
|
||||
let mut split_files: Vec<SourceFile> = paths
|
||||
.into_iter()
|
||||
.map(|path| {
|
||||
let text = std::fs::read_to_string(&path)
|
||||
.unwrap_or_else(|err| panic!("guard source {} must be readable: {err}", path.display()));
|
||||
let relative = path
|
||||
.strip_prefix(root)
|
||||
.unwrap_or_else(|_| panic!("guard source {} must be under {}", path.display(), root.display()));
|
||||
let path = relative
|
||||
.components()
|
||||
.map(|component| component.as_os_str().to_string_lossy())
|
||||
.collect::<Vec<_>>()
|
||||
.join("/");
|
||||
SourceFile { path, text }
|
||||
})
|
||||
.collect();
|
||||
split_files.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
files.extend(split_files);
|
||||
}
|
||||
|
||||
Self { files }
|
||||
}
|
||||
|
||||
/// Whether any one file of the set contains `needle`.
|
||||
pub(crate) fn contains(&self, needle: &str) -> bool {
|
||||
self.files.iter().any(|file| file.text.contains(needle))
|
||||
}
|
||||
|
||||
/// Non-overlapping occurrences of `needle`, summed over the files; a match never spans two
|
||||
/// files.
|
||||
pub(crate) fn count(&self, needle: &str) -> usize {
|
||||
self.files.iter().map(|file| file.text.matches(needle).count()).sum()
|
||||
}
|
||||
|
||||
/// The text of the one `async fn name` method of the set: from its definition line, which
|
||||
/// must be indented by exactly four spaces (a method of a top-level `impl`), through the first
|
||||
/// following line that is exactly ` }`. A neighbouring method of any visibility, a test
|
||||
/// module, or the rest of the file is never part of it.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics unless exactly one such definition exists across the set and it has that closing
|
||||
/// line.
|
||||
pub(crate) fn method_body(&self, name: &str) -> &str {
|
||||
let definitions: Vec<(&SourceFile, usize)> = self
|
||||
.files
|
||||
.iter()
|
||||
.flat_map(|file| {
|
||||
line_starts(&file.text)
|
||||
.into_iter()
|
||||
.filter(move |&start| is_method_definition(line_at(&file.text, start), name))
|
||||
.map(move |start| (file, start))
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
definitions.len(),
|
||||
1,
|
||||
"method {name} must be defined exactly once, as a four-space-indented `async fn`, across {}",
|
||||
self.describe()
|
||||
);
|
||||
let (file, start) = definitions[0];
|
||||
let end = line_starts(&file.text)
|
||||
.into_iter()
|
||||
.filter(|&line| line > start)
|
||||
.find(|&line| line_at(&file.text, line) == " }")
|
||||
.unwrap_or_else(|| panic!("method {name} in {} has no closing ` }}` line", file.path));
|
||||
let close = end + " }".len();
|
||||
&file.text[start..close]
|
||||
}
|
||||
|
||||
/// Manifest-relative paths of the set, for panic messages.
|
||||
pub(crate) fn describe(&self) -> String {
|
||||
self.files
|
||||
.iter()
|
||||
.map(|file| file.path.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
}
|
||||
|
||||
/// Byte offsets at which each line of `text` starts.
|
||||
fn line_starts(text: &str) -> Vec<usize> {
|
||||
std::iter::once(0)
|
||||
.chain(text.match_indices('\n').map(|(index, _)| index + 1))
|
||||
.filter(|&start| start < text.len())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The line starting at `start`, without its line ending.
|
||||
fn line_at(text: &str, start: usize) -> &str {
|
||||
let rest = &text[start..];
|
||||
let line = rest.split('\n').next().unwrap_or(rest);
|
||||
line.strip_suffix('\r').unwrap_or(line)
|
||||
}
|
||||
|
||||
fn is_method_definition(line: &str, name: &str) -> bool {
|
||||
let Some(item) = line.strip_prefix(" ") else {
|
||||
return false;
|
||||
};
|
||||
if item.starts_with(' ') {
|
||||
return false;
|
||||
}
|
||||
let item = ["pub(crate) ", "pub(super) ", "pub "]
|
||||
.iter()
|
||||
.find_map(|visibility| item.strip_prefix(visibility))
|
||||
.unwrap_or(item);
|
||||
item.strip_prefix("async fn ")
|
||||
.and_then(|rest| rest.strip_prefix(name))
|
||||
.is_some_and(|rest| rest.starts_with(['(', '<']))
|
||||
}
|
||||
|
||||
fn is_test_only_path(path: &str) -> bool {
|
||||
let components: Vec<&str> = path.split('/').collect();
|
||||
let Some((file, directories)) = components.split_last() else {
|
||||
return false;
|
||||
};
|
||||
let stem = file.strip_suffix(".rs").unwrap_or(*file);
|
||||
directories.contains(&"tests")
|
||||
|| stem == "tests"
|
||||
|| stem == "test_support"
|
||||
|| stem.ends_with("_test")
|
||||
|| stem.ends_with("_tests")
|
||||
}
|
||||
|
||||
/// The directory that holds the child modules of the module in `path`.
|
||||
fn child_module_dir(path: &str) -> String {
|
||||
match path.strip_suffix("/mod.rs") {
|
||||
Some(dir) => dir.to_owned(),
|
||||
None => path.strip_suffix(".rs").unwrap_or(path).to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn indentation(line: &str) -> usize {
|
||||
line.len() - line.trim_start_matches(' ').len()
|
||||
}
|
||||
|
||||
fn is_test_cfg_attribute(trimmed: &str) -> bool {
|
||||
trimmed == "#[cfg(test)]" || (trimmed.starts_with("#[cfg(all(test,") && trimmed.ends_with(")]"))
|
||||
}
|
||||
|
||||
/// `text` without its `#[cfg(test)]` items, plus the names of the `#[cfg(test)] mod name;`
|
||||
/// declarations it removed; `None` when the whole file is `#![cfg(test)]`.
|
||||
fn strip_test_items(path: &str, text: &str) -> Option<(String, Vec<String>)> {
|
||||
let lines: Vec<&str> = text.split_inclusive('\n').collect();
|
||||
let mut kept = String::with_capacity(text.len());
|
||||
let mut test_modules = Vec::new();
|
||||
let mut index = 0;
|
||||
while index < lines.len() {
|
||||
let trimmed = lines[index].trim();
|
||||
if trimmed == "#![cfg(test)]" {
|
||||
return None;
|
||||
}
|
||||
if !is_test_cfg_attribute(trimmed) {
|
||||
assert!(
|
||||
!(trimmed.starts_with("#[cfg(test)]") || trimmed.starts_with("#[cfg(all(test,")),
|
||||
"{path}:{}: a test-only item on its attribute line is not a shape the guard stripper reads",
|
||||
index + 1
|
||||
);
|
||||
kept.push_str(lines[index]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
let indent = indentation(lines[index]);
|
||||
let mut item = index + 1;
|
||||
while item < lines.len() && is_attribute_or_comment(lines[item].trim()) {
|
||||
item += 1;
|
||||
}
|
||||
assert!(item < lines.len(), "{path}:{}: #[cfg(test)] is not followed by an item", index + 1);
|
||||
assert_eq!(
|
||||
indentation(lines[item]),
|
||||
indent,
|
||||
"{path}:{}: the #[cfg(test)] item is not at its attribute's indentation",
|
||||
item + 1
|
||||
);
|
||||
if let Some(name) = declared_module(lines[item].trim()) {
|
||||
test_modules.push(name.to_owned());
|
||||
}
|
||||
index = test_item_end(path, &lines, item, indent) + 1;
|
||||
}
|
||||
Some((kept, test_modules))
|
||||
}
|
||||
|
||||
fn is_attribute_or_comment(trimmed: &str) -> bool {
|
||||
(trimmed.starts_with("#[") && trimmed.ends_with(']')) || trimmed.starts_with("//")
|
||||
}
|
||||
|
||||
fn declared_module(item: &str) -> Option<&str> {
|
||||
let item = ["pub(crate) ", "pub(super) ", "pub "]
|
||||
.iter()
|
||||
.find_map(|visibility| item.strip_prefix(visibility))
|
||||
.unwrap_or(item);
|
||||
let name = item.strip_prefix("mod ")?.strip_suffix(';')?;
|
||||
(!name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')).then_some(name)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Lexeme {
|
||||
Code,
|
||||
Str,
|
||||
RawStr(usize),
|
||||
BlockComment,
|
||||
}
|
||||
|
||||
fn is_ident(c: char) -> bool {
|
||||
c.is_alphanumeric() || c == '_'
|
||||
}
|
||||
|
||||
/// Index of the last line of the item that starts at `lines[start]` with indentation `indent`.
|
||||
fn test_item_end(path: &str, lines: &[&str], start: usize, indent: usize) -> usize {
|
||||
let mut depth: i64 = 0;
|
||||
let mut state = Lexeme::Code;
|
||||
// Inside a `where` clause a trailing comma separates bounds instead of ending the item.
|
||||
let mut in_where_clause = false;
|
||||
for (offset, line) in lines[start..].iter().enumerate() {
|
||||
let number = start + offset + 1;
|
||||
if state == Lexeme::Code && depth == 0 && indentation(line) == indent && line.trim().starts_with("where") {
|
||||
in_where_clause = true;
|
||||
}
|
||||
let chars: Vec<char> = line.chars().collect();
|
||||
let mut last_code = None;
|
||||
let mut i = 0;
|
||||
while i < chars.len() {
|
||||
let c = chars[i];
|
||||
let next = chars.get(i + 1).copied();
|
||||
match state {
|
||||
Lexeme::Code => match c {
|
||||
'/' if next == Some('/') => break,
|
||||
'/' if next == Some('*') => {
|
||||
state = Lexeme::BlockComment;
|
||||
i += 1;
|
||||
}
|
||||
'"' => state = Lexeme::Str,
|
||||
'r' if matches!(next, Some('"' | '#'))
|
||||
&& (i == 0
|
||||
|| !is_ident(chars[i - 1])
|
||||
|| (chars[i - 1] == 'b' && (i == 1 || !is_ident(chars[i - 2])))) =>
|
||||
{
|
||||
let mut quote = i + 1;
|
||||
while chars.get(quote) == Some(&'#') {
|
||||
quote += 1;
|
||||
}
|
||||
if chars.get(quote) == Some(&'"') {
|
||||
state = Lexeme::RawStr(quote - i - 1);
|
||||
i = quote;
|
||||
} else {
|
||||
last_code = Some(c);
|
||||
}
|
||||
}
|
||||
'\'' => {
|
||||
if next == Some('\\') {
|
||||
let close = chars
|
||||
.get(i + 3..)
|
||||
.and_then(|rest| rest.iter().position(|&candidate| candidate == '\''))
|
||||
.unwrap_or_else(|| panic!("{path}:{number}: unterminated character literal"));
|
||||
i += 3 + close;
|
||||
last_code = Some('\'');
|
||||
} else if chars.get(i + 2) == Some(&'\'') {
|
||||
i += 2;
|
||||
last_code = Some('\'');
|
||||
} else {
|
||||
last_code = Some(c);
|
||||
}
|
||||
}
|
||||
'{' | '(' | '[' => {
|
||||
depth += 1;
|
||||
last_code = Some(c);
|
||||
}
|
||||
'}' | ')' | ']' => {
|
||||
depth -= 1;
|
||||
assert!(depth >= 0, "{path}:{number}: unbalanced `{c}` in a #[cfg(test)] item");
|
||||
last_code = Some(c);
|
||||
}
|
||||
c if !c.is_whitespace() => last_code = Some(c),
|
||||
_ => {}
|
||||
},
|
||||
Lexeme::Str => match c {
|
||||
'\\' => i += 1,
|
||||
'"' => {
|
||||
state = Lexeme::Code;
|
||||
last_code = Some('"');
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Lexeme::RawStr(hashes) => {
|
||||
if c == '"' && chars.len() > i + hashes && chars[i + 1..=i + hashes].iter().all(|&h| h == '#') {
|
||||
i += hashes;
|
||||
state = Lexeme::Code;
|
||||
last_code = Some('"');
|
||||
}
|
||||
}
|
||||
Lexeme::BlockComment => {
|
||||
if c == '*' && next == Some('/') {
|
||||
state = Lexeme::Code;
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
if state != Lexeme::Code || depth != 0 {
|
||||
continue;
|
||||
}
|
||||
let line_indent = indentation(line);
|
||||
assert!(
|
||||
offset == 0 || line_indent >= indent,
|
||||
"{path}:{number}: a #[cfg(test)] item continues below its own indentation"
|
||||
);
|
||||
match last_code {
|
||||
Some(';') => return start + offset,
|
||||
Some(',') if !in_where_clause => return start + offset,
|
||||
Some('}') => {
|
||||
assert!(
|
||||
offset == 0 || line_indent == indent,
|
||||
"{path}:{number}: a #[cfg(test)] item block closes at indentation {line_indent}, expected {indent}"
|
||||
);
|
||||
return start + offset;
|
||||
}
|
||||
_ => {
|
||||
let following = lines
|
||||
.get(start + offset + 1)
|
||||
.unwrap_or_else(|| panic!("{path}:{number}: a #[cfg(test)] item never ends"));
|
||||
let following_trimmed = following.trim();
|
||||
assert!(
|
||||
indentation(following) > indent
|
||||
|| (indentation(following) == indent
|
||||
&& ["where", "{", "->", ".", ")", "]"]
|
||||
.iter()
|
||||
.any(|prefix| following_trimmed.starts_with(prefix))),
|
||||
"{path}:{}: the line after an unterminated #[cfg(test)] item line does not continue it",
|
||||
number + 1
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
panic!("{path}:{}: a #[cfg(test)] item never ends", start + 1);
|
||||
}
|
||||
|
||||
fn collect_rust_files(dir: &Path, out: &mut Vec<PathBuf>) {
|
||||
let entries =
|
||||
std::fs::read_dir(dir).unwrap_or_else(|err| panic!("guard split directory {} must be readable: {err}", dir.display()));
|
||||
for entry in entries {
|
||||
let entry = entry.unwrap_or_else(|err| panic!("guard split directory {} must be listable: {err}", dir.display()));
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
collect_rust_files(&path, out);
|
||||
} else if path.extension().is_some_and(|extension| extension == "rs") {
|
||||
out.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes `files` under a fresh temporary package root, for guard fixtures.
|
||||
pub(crate) fn fixture_root(files: &[(&str, &str)]) -> tempfile::TempDir {
|
||||
let root = tempfile::tempdir().expect("fixture root");
|
||||
for (relative, text) in files {
|
||||
let path = root.path().join(relative);
|
||||
std::fs::create_dir_all(path.parent().expect("fixture path has a parent")).expect("create fixture dir");
|
||||
std::fs::write(path, text).expect("write fixture");
|
||||
}
|
||||
root
|
||||
}
|
||||
|
||||
/// Runs `run`, which must panic, and returns its panic message.
|
||||
pub(crate) fn panic_message(run: impl FnOnce() + std::panic::UnwindSafe) -> String {
|
||||
let payload = std::panic::catch_unwind(run).expect_err("the guard input must be rejected");
|
||||
payload
|
||||
.downcast_ref::<String>()
|
||||
.cloned()
|
||||
.or_else(|| payload.downcast_ref::<&str>().map(|message| (*message).to_owned()))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{SourceSet, fixture_root, panic_message};
|
||||
|
||||
const MAIN: &str = "src/app/usecase.rs";
|
||||
const SPLIT: &str = "src/app/usecase";
|
||||
|
||||
#[test]
|
||||
fn absent_split_directory_reads_only_the_required_file() {
|
||||
let root = fixture_root(&[(MAIN, "guarded();\n")]);
|
||||
|
||||
let set = SourceSet::read_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
assert_eq!(set.describe(), MAIN);
|
||||
assert!(set.contains("guarded();"));
|
||||
assert_eq!(set.count("guarded();"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_directory_files_follow_the_required_file_in_path_order() {
|
||||
let root = fixture_root(&[
|
||||
(MAIN, "guarded();\n"),
|
||||
("src/app/usecase/zeta.rs", "guarded();\n"),
|
||||
("src/app/usecase/alpha.rs", "guarded();\nguarded();\n"),
|
||||
("src/app/usecase/tests/nested.rs", "guarded();\n"),
|
||||
("src/app/usecase/notes.txt", "guarded();\n"),
|
||||
]);
|
||||
|
||||
let set = SourceSet::read_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
assert_eq!(
|
||||
set.describe(),
|
||||
"src/app/usecase.rs, src/app/usecase/alpha.rs, src/app/usecase/tests/nested.rs, src/app/usecase/zeta.rs"
|
||||
);
|
||||
assert_eq!(set.count("guarded();"), 5, "non-.rs files are not source");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_required_file_fails_even_when_the_split_directory_exists() {
|
||||
let root = fixture_root(&[("src/app/usecase/moved.rs", "guarded();\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains(MAIN), "panic must name the missing file: {message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_path_that_is_a_file_fails() {
|
||||
let root = fixture_root(&[(MAIN, "guarded();\n"), (SPLIT, "not a directory\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains("must be a directory"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_does_not_join_a_needle_across_two_files() {
|
||||
let root = fixture_root(&[(MAIN, "guard"), ("src/app/usecase/tail.rs", "ed();\n")]);
|
||||
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
assert_eq!(set.count("guarded();"), 0);
|
||||
assert!(!set.contains("guarded();"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_text_keeps_production_items_and_drops_every_test_only_one() {
|
||||
let main = "\
|
||||
use crate::kept;
|
||||
#[cfg(test)]
|
||||
use crate::test_only_import;
|
||||
#[cfg(any(test, feature = \"x\"))]
|
||||
fn kept_in_some_production_build() { guarded(); }
|
||||
|
||||
impl Usecase {
|
||||
#[cfg(test)]
|
||||
field_for_tests: Option<u8>,
|
||||
#[cfg(test)]
|
||||
pub fn without_context() -> Self {
|
||||
guarded();
|
||||
}
|
||||
fn run(&self) {
|
||||
#[cfg(test)]
|
||||
let value = injected()
|
||||
.map_or(value, |err| guarded());
|
||||
#[cfg(test)]
|
||||
if let Some(hook) = self.hook {
|
||||
guarded();
|
||||
}
|
||||
guarded();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn helper<F>(f: F) -> u8
|
||||
where
|
||||
F: Fn(),
|
||||
{
|
||||
guarded();
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
const CANARY: &str = \"guarded();\";
|
||||
|
||||
#[cfg(test)]
|
||||
mod declared;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
const BRACES: &str = \"}}} {{\";
|
||||
const RAW: &[u8] = br#\"{\"a\": \"}\"}\"#;
|
||||
fn t() { let c = '}'; let d = '\\''; guarded(); }
|
||||
}
|
||||
";
|
||||
let root = fixture_root(&[
|
||||
(MAIN, main),
|
||||
("src/app/usecase/declared.rs", "guarded();\n"),
|
||||
("src/app/usecase/declared/deeper.rs", "guarded();\n"),
|
||||
("src/app/usecase/ops.rs", "fn op() { guarded(); }\n"),
|
||||
("src/app/usecase/tests/op_tests.rs", "guarded();\n"),
|
||||
("src/app/usecase/ops_test.rs", "guarded();\n"),
|
||||
("src/app/usecase/test_support.rs", "guarded();\n"),
|
||||
("src/app/usecase/gated.rs", "#![cfg(test)]\nguarded();\n"),
|
||||
]);
|
||||
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
assert_eq!(set.describe(), "src/app/usecase.rs, src/app/usecase/ops.rs");
|
||||
assert_eq!(set.count("guarded();"), 3, "production calls: kept_in_some_production_build, run, op");
|
||||
assert!(set.contains("use crate::kept;") && set.contains("impl Usecase {") && set.contains("fn run(&self) {"));
|
||||
assert!(!set.contains("test_only_import") && !set.contains("without_context") && !set.contains("mod tests"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_test_only_item_on_its_attribute_line_fails_loudly() {
|
||||
let root = fixture_root(&[(MAIN, "#[cfg(test)] use crate::hidden;\nguarded();\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains("src/app/usecase.rs:1"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_test_only_item_that_never_closes_fails_loudly() {
|
||||
let root = fixture_root(&[(MAIN, "#[cfg(test)]\nmod tests {\n fn t() {}\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains("never ends"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_test_only_block_closing_at_another_indentation_fails_loudly() {
|
||||
let root = fixture_root(&[(MAIN, "impl X {\n #[cfg(test)]\n fn t() {\n}\n fn kept() {}\n}\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains("src/app/usecase.rs:4"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unterminated_test_only_line_followed_by_production_fails_loudly() {
|
||||
let root = fixture_root(&[(MAIN, "fn f() {\n #[cfg(test)]\n let hook = injected()\n guarded();\n}\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains("does not continue it"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_test_only_required_file_is_rejected() {
|
||||
let root = fixture_root(&[("src/app/usecase_test.rs", "guarded();\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, "src/app/usecase_test.rs", "src/app/usecase_test");
|
||||
});
|
||||
|
||||
assert!(message.contains("test-only"), "{message}");
|
||||
}
|
||||
|
||||
const METHODS: &str = "\
|
||||
impl Usecase {
|
||||
pub async fn guarded(&self) {
|
||||
first();
|
||||
}
|
||||
|
||||
pub(crate) async fn neighbour(&self) {
|
||||
needle();
|
||||
}
|
||||
|
||||
pub(super) async fn other(&self) {
|
||||
needle();
|
||||
}
|
||||
}
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn method_body_stops_at_its_own_closing_line_before_any_neighbour() {
|
||||
let root = fixture_root(&[
|
||||
(MAIN, METHODS),
|
||||
("src/app/usecase/later.rs", "impl Usecase {\n fn x() { needle(); }\n}\n"),
|
||||
]);
|
||||
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
let body = set.method_body("guarded");
|
||||
|
||||
assert_eq!(body, " pub async fn guarded(&self) {\n first();\n }");
|
||||
assert!(!body.contains("needle"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_body_of_the_last_method_stops_before_the_test_module() {
|
||||
let text = "impl Usecase {\n async fn last(&self) {\n first();\n }\n}\n\n#[cfg(test)]\nmod tests {\n fn t() { needle(); }\n}\n";
|
||||
let root = fixture_root(&[("src/app/usecase/moved.rs", text), (MAIN, "\n")]);
|
||||
|
||||
let set = SourceSet::read_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
assert!(!set.method_body("last").contains("needle"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_body_does_not_match_a_longer_name_or_a_nested_definition() {
|
||||
let text = "impl Usecase {\n pub async fn guarded_more(&self) {}\n async fn guarded(&self) {}\n}\n";
|
||||
let root = fixture_root(&[(MAIN, text)]);
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
let message = panic_message(move || {
|
||||
set.method_body("guarded");
|
||||
});
|
||||
|
||||
assert!(message.contains("exactly once"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_body_defined_in_two_files_fails() {
|
||||
let root = fixture_root(&[(MAIN, METHODS), ("src/app/usecase/copy.rs", METHODS)]);
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
let message = panic_message(move || {
|
||||
set.method_body("guarded");
|
||||
});
|
||||
|
||||
assert!(message.contains("src/app/usecase/copy.rs"), "panic must name the set: {message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_body_without_a_closing_line_fails() {
|
||||
let root = fixture_root(&[(MAIN, "impl Usecase {\n async fn guarded(&self) {\n first();\n }\n}\n")]);
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
let message = panic_message(move || {
|
||||
set.method_body("guarded");
|
||||
});
|
||||
|
||||
assert!(message.contains("no closing"), "{message}");
|
||||
}
|
||||
}
|
||||
@@ -89,6 +89,8 @@ pub mod diagnose;
|
||||
pub mod embedded;
|
||||
pub mod error;
|
||||
pub mod foreground_observation;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod guard_sources;
|
||||
pub mod init;
|
||||
pub mod inspect;
|
||||
pub(crate) mod kms_deletion_gate;
|
||||
|
||||
@@ -1334,12 +1334,16 @@ fn test_delete_bucket_broadcast_fences_target_membership_through_delivery() {
|
||||
"a failed destructive delivery must remain visible for operator repair"
|
||||
);
|
||||
|
||||
let usecase = include_str!("../app/bucket_usecase.rs");
|
||||
let delete = usecase
|
||||
.split("async fn execute_delete_bucket_inner")
|
||||
.nth(1)
|
||||
.and_then(|rest| rest.split("pub async fn execute_head_bucket").next())
|
||||
.expect("delete bucket usecase");
|
||||
assert_delete_bucket_reserves_before_local_delete(&crate::guard_sources::SourceSet::production(
|
||||
"src/app/bucket_usecase.rs",
|
||||
"src/app/bucket_usecase",
|
||||
));
|
||||
}
|
||||
|
||||
/// The reservation and the local delete must both be inside `execute_delete_bucket_inner`, in
|
||||
/// that order; the slice ends at that method's own closing line.
|
||||
fn assert_delete_bucket_reserves_before_local_delete(usecase: &crate::guard_sources::SourceSet) {
|
||||
let delete = usecase.method_body("execute_delete_bucket_inner");
|
||||
assert!(
|
||||
delete
|
||||
.find("prepare_site_replication_delete_bucket")
|
||||
@@ -1349,6 +1353,67 @@ fn test_delete_bucket_broadcast_fences_target_membership_through_delivery() {
|
||||
);
|
||||
}
|
||||
|
||||
const DELETE_BUCKET_INNER_FIXTURE: &str = "\
|
||||
impl DefaultBucketUsecase {
|
||||
async fn execute_delete_bucket_inner(&self) {
|
||||
let intent = prepare_site_replication_delete_bucket(&bucket, force).await?;
|
||||
LOCAL_DELETE
|
||||
}
|
||||
}
|
||||
";
|
||||
|
||||
fn delete_bucket_fixture(core: &str) -> crate::guard_sources::SourceSet {
|
||||
let root = crate::guard_sources::fixture_root(&[
|
||||
("src/app/bucket_usecase.rs", "pub struct DefaultBucketUsecase;\n"),
|
||||
("src/app/bucket_usecase/core.rs", core),
|
||||
]);
|
||||
crate::guard_sources::SourceSet::production_under(root.path(), "src/app/bucket_usecase.rs", "src/app/bucket_usecase")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_bucket_order_guard_passes_after_the_method_moves_into_the_split_directory() {
|
||||
let core = DELETE_BUCKET_INNER_FIXTURE.replace("LOCAL_DELETE", "store.delete_bucket(&bucket).await;");
|
||||
assert_delete_bucket_reserves_before_local_delete(&delete_bucket_fixture(&core));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_bucket_order_guard_fails_when_the_local_delete_leaves_the_method() {
|
||||
let core = format!(
|
||||
"{}\nimpl DefaultBucketUsecase {{\n pub(crate) async fn delete_helper(&self) {{\n store.delete_bucket(&bucket).await;\n }}\n}}\n",
|
||||
DELETE_BUCKET_INNER_FIXTURE.replace("LOCAL_DELETE", "")
|
||||
);
|
||||
let usecase = delete_bucket_fixture(&core);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_delete_bucket_reserves_before_local_delete(&usecase));
|
||||
|
||||
assert!(message.contains("local delete"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_bucket_order_guard_fails_when_the_delete_precedes_the_reservation() {
|
||||
let core = DELETE_BUCKET_INNER_FIXTURE.replace(
|
||||
" let intent = prepare_site_replication_delete_bucket(&bucket, force).await?;\n LOCAL_DELETE",
|
||||
" store.delete_bucket(&bucket).await;\n let intent = prepare_site_replication_delete_bucket(&bucket, force).await?;",
|
||||
);
|
||||
let usecase = delete_bucket_fixture(&core);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_delete_bucket_reserves_before_local_delete(&usecase));
|
||||
|
||||
assert!(message.contains("destructive peer liabilities"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_bucket_order_guard_fails_when_the_method_has_no_closing_line() {
|
||||
let core = DELETE_BUCKET_INNER_FIXTURE
|
||||
.replace("LOCAL_DELETE", "store.delete_bucket(&bucket).await;")
|
||||
.replace(" }\n}\n", "");
|
||||
let usecase = delete_bucket_fixture(&core);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_delete_bucket_reserves_before_local_delete(&usecase));
|
||||
|
||||
assert!(message.contains("no closing"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_retry_settlement_preserves_a_newer_same_path_failure() {
|
||||
let peer = peer("remote", "https://remote.example.com");
|
||||
|
||||
+91
-1756
File diff suppressed because it is too large
Load Diff
@@ -53,7 +53,7 @@ mod tests {
|
||||
remove_object_lock_retention_metadata, set_bucket_metadata, validate_bucket_object_lock_enabled,
|
||||
validate_list_object_unordered_with_delimiter,
|
||||
};
|
||||
use http::{Extensions, HeaderMap, HeaderValue, Method, StatusCode, Uri};
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
@@ -62,108 +62,14 @@ mod tests {
|
||||
};
|
||||
use rustfs_zip::CompressionFormat;
|
||||
use s3s::dto::{
|
||||
DefaultRetention, DeleteObjectTaggingInput, Delimiter, FilterRule, FilterRuleName, GetBucketAclInput, GetObjectAclInput,
|
||||
GetObjectLegalHoldInput, GetObjectRetentionInput, GetObjectTaggingInput, LambdaFunctionConfiguration,
|
||||
NotificationConfigurationFilter, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold,
|
||||
ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule, PutBucketAclInput,
|
||||
PutObjectAclInput, PutObjectLegalHoldInput, PutObjectLockConfigurationInput, PutObjectRetentionInput,
|
||||
PutObjectTaggingInput, QueueConfiguration, S3KeyFilter, Tag, Tagging, TopicConfiguration,
|
||||
DefaultRetention, Delimiter, FilterRule, FilterRuleName, LambdaFunctionConfiguration, NotificationConfigurationFilter,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention,
|
||||
ObjectLockRetentionMode, ObjectLockRule, QueueConfiguration, S3KeyFilter, TopicConfiguration,
|
||||
};
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request, s3_error};
|
||||
use s3s::{S3Error, S3ErrorCode, s3_error};
|
||||
use std::collections::HashMap;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
input,
|
||||
method,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_apis_report_storage_read_quorum_as_unavailable() {
|
||||
crate::app::gating_test_env::run_large_stack_test("list-quorum-surface", || async {
|
||||
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use s3s::dto::{ListObjectVersionsInput, ListObjectsInput, ListObjectsV2Input};
|
||||
crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let bucket = format!("list-quorum-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create listing bucket");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let saved_disks = set.disks.read().await.clone();
|
||||
{
|
||||
let mut disks = set.disks.write().await;
|
||||
for disk in disks.iter_mut().skip(1) {
|
||||
*disk = None;
|
||||
}
|
||||
}
|
||||
let fs = FS::new();
|
||||
let errors = [
|
||||
S3::list_objects(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectsInput {
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("V1 cannot list below storage read quorum"),
|
||||
S3::list_objects_v2(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectsV2Input {
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("V2 cannot list below storage read quorum"),
|
||||
S3::list_object_versions(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectVersionsInput {
|
||||
bucket,
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("versions cannot list below storage read quorum"),
|
||||
];
|
||||
*set.disks.write().await = saved_disks;
|
||||
for error in errors {
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
|
||||
let source = error.source().expect("retain storage diagnostics");
|
||||
assert!(
|
||||
matches!(
|
||||
source.downcast_ref::<StorageError>(),
|
||||
Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
|
||||
),
|
||||
"the production LIST boundary must expose a typed read-quorum failure: {source}"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fs_creation() {
|
||||
let _fs = FS::new();
|
||||
@@ -293,164 +199,6 @@ mod tests {
|
||||
assert_eq!(gz_format.extension(), "gz");
|
||||
}
|
||||
|
||||
/// Premise guard for the `*_returns_internal_error_when_store_uninitialized`
|
||||
/// tests (backlog#1830): they assert the error path taken while the global
|
||||
/// object layer is absent. Under nextest — the authoritative runner — every
|
||||
/// test owns its process, so the premise always holds and the assertion
|
||||
/// always runs. Under the documented shared-process `cargo test` fallback a
|
||||
/// sibling test may have initialized the store first; the premise is then
|
||||
/// unattainable, so the test skips instead of asserting against a scenario
|
||||
/// it does not describe.
|
||||
fn store_uninitialized_premise_holds() -> bool {
|
||||
if crate::runtime_sources::current_object_store_handle().is_some() {
|
||||
eprintln!("skipping store-uninitialized assertion: a sibling test already initialized the global object layer");
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_acl(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_bucket_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetBucketAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_bucket_acl(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_legal_hold_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectLegalHoldInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_legal_hold(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_retention_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetObjectRetentionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_retention(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_legal_hold_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectLegalHoldInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_legal_hold(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_bucket_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutBucketAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_bucket_acl(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_acl(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_retention_returns_internal_error_when_store_uninitialized() {
|
||||
let input = PutObjectRetentionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_retention(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectLockConfigurationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.object_lock_configuration(Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs
|
||||
.put_object_lock_configuration(build_request(input, Method::PUT))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
/// The bucket default retention path reads the persisted shape; the request
|
||||
/// literals in these tests stay on the wire DTO and convert once here.
|
||||
fn persisted_lock(cfg: &ObjectLockConfiguration) -> rustfs_gateway_types::persistence::PersistedObjectLockConfiguration {
|
||||
@@ -777,104 +525,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_tagging(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_rejects_too_many_tags() {
|
||||
let tag_set = (0..11)
|
||||
.map(|index| Tag {
|
||||
key: Some(format!("k{index}")),
|
||||
value: Some(format!("v{index}")),
|
||||
})
|
||||
.collect();
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging { tag_set })
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Cannot have more than 10 tags per object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_rejects_empty_tag_key_before_store_lookup() {
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some(String::new()),
|
||||
value: Some("v1".to_string()),
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Tag key cannot be empty"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some("k".to_string()),
|
||||
value: Some("v".to_string()),
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delete_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = DeleteObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs
|
||||
.delete_object_tagging(build_request(input, Method::DELETE))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_adaptive_buffer_size_with_profile() {
|
||||
const KB: i64 = 1024;
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of bucket policy, policy status, versioning, notification and ACL: batch A2 of
|
||||
//! the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api::{self, acl};
|
||||
use crate::storage::storage_api::ecfs_consumer::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage::{persisted_from_s3s, persisted_to_s3s};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
/// Handler bodies that [`S3`] runs through `audit_only_operation`.
|
||||
impl FS {
|
||||
async fn put_bucket_acl_inner(&self, req: S3Request<PutBucketAclInput>) -> S3Result<S3Response<PutBucketAclOutput>> {
|
||||
let PutBucketAclInput {
|
||||
bucket,
|
||||
access_control_policy,
|
||||
..
|
||||
} = req.input;
|
||||
record_s3_op(S3Operation::PutBucketAcl);
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
if access_control_policy.is_some() {
|
||||
return Err(s3_error!(
|
||||
NotImplemented,
|
||||
"ACL XML grants are not supported; use canned ACL headers or omit ACL"
|
||||
));
|
||||
}
|
||||
|
||||
Ok(S3Response::new(PutBucketAclOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
impl FS {
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_policy_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketPolicyInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketPolicyOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketPolicy, |req| usecase.execute_delete_bucket_policy(req)).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_acl_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketAclInput>,
|
||||
) -> S3Result<S3Response<GetBucketAclOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketAcl);
|
||||
let GetBucketAclInput { bucket, .. } = req.input;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(acl::build_get_bucket_acl_output()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_notification_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketNotificationConfigurationInput>,
|
||||
) -> S3Result<S3Response<GetBucketNotificationConfigurationOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketNotificationConfiguration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let notification_configuration = usecase
|
||||
.execute_get_bucket_notification_configuration(req)
|
||||
.await?
|
||||
.as_ref()
|
||||
.map(persisted_to_s3s::<_, NotificationConfiguration>)
|
||||
.transpose()?;
|
||||
match notification_configuration {
|
||||
Some(NotificationConfiguration {
|
||||
event_bridge_configuration,
|
||||
lambda_function_configurations,
|
||||
queue_configurations,
|
||||
topic_configurations,
|
||||
}) => Ok(S3Response::new(GetBucketNotificationConfigurationOutput {
|
||||
event_bridge_configuration,
|
||||
lambda_function_configurations,
|
||||
queue_configurations,
|
||||
topic_configurations,
|
||||
})),
|
||||
None => Ok(S3Response::new(GetBucketNotificationConfigurationOutput::default())),
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_policy_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketPolicyInput>,
|
||||
) -> S3Result<S3Response<GetBucketPolicyOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketPolicy);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_get_bucket_policy(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_policy_status_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketPolicyStatusInput>,
|
||||
) -> S3Result<S3Response<GetBucketPolicyStatusOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketPolicyStatus);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_get_bucket_policy_status(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "get_bucket_versioning", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_versioning_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketVersioningInput>,
|
||||
) -> S3Result<S3Response<GetBucketVersioningOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketVersioning);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let config = usecase.execute_get_bucket_versioning(req).await?;
|
||||
Ok(S3Response::new(GetBucketVersioningOutput {
|
||||
status: config.status.map(BucketVersioningStatus::from),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_acl_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketAclInput>,
|
||||
) -> S3Result<S3Response<PutBucketAclOutput>> {
|
||||
audit_only_operation(req, S3Operation::PutBucketAcl, |req| self.put_bucket_acl_inner(req)).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_notification_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketNotificationConfigurationInput>,
|
||||
) -> S3Result<S3Response<PutBucketNotificationConfigurationOutput>> {
|
||||
let prepared = persisted_from_s3s(&req.input.notification_configuration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketNotificationConfiguration, |req| {
|
||||
usecase.execute_put_bucket_notification_configuration(req, prepared)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_policy_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketPolicyInput>,
|
||||
) -> S3Result<S3Response<PutBucketPolicyOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketPolicy, |req| usecase.execute_put_bucket_policy(req)).await
|
||||
}
|
||||
|
||||
#[instrument(name = "put_bucket_versioning", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_versioning_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketVersioningInput>,
|
||||
) -> S3Result<S3Response<PutBucketVersioningOutput>> {
|
||||
let prepared = persisted_from_s3s(&req.input.versioning_configuration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketVersioning, |req| {
|
||||
usecase.execute_put_bucket_versioning(req, prepared)
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of the bucket CORS, tagging, public access block and encryption configurations:
|
||||
//! batch A1 of the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use crate::storage::{persisted_from_s3s, persisted_to_s3s, tagging_from_s3s, tagging_to_s3s};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "delete_bucket_cors", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_cors_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketCorsInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketCorsOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketCors, |req| usecase.execute_delete_bucket_cors(req)).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_encryption_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketEncryptionInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketEncryptionOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketEncryption, |req| {
|
||||
usecase.execute_delete_bucket_encryption(req)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(name = "delete_bucket_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketTaggingInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketTaggingOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketTagging, |req| usecase.execute_delete_bucket_tagging(req)).await
|
||||
}
|
||||
|
||||
#[instrument(name = "delete_public_access_block", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_public_access_block_edge(
|
||||
&self,
|
||||
req: S3Request<DeletePublicAccessBlockInput>,
|
||||
) -> S3Result<S3Response<DeletePublicAccessBlockOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeletePublicAccessBlock, |req| {
|
||||
usecase.execute_delete_public_access_block(req)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(name = "get_bucket_cors", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_cors_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketCorsInput>,
|
||||
) -> S3Result<S3Response<GetBucketCorsOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketCors);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let cors_configuration = usecase.execute_get_bucket_cors(req).await?;
|
||||
let cors_configuration: CORSConfiguration = persisted_to_s3s(&cors_configuration)?;
|
||||
Ok(S3Response::new(GetBucketCorsOutput {
|
||||
cors_rules: Some(cors_configuration.cors_rules),
|
||||
}))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_encryption_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketEncryptionInput>,
|
||||
) -> S3Result<S3Response<GetBucketEncryptionOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketEncryption);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let server_side_encryption_configuration = usecase
|
||||
.execute_get_bucket_encryption(req)
|
||||
.await?
|
||||
.as_ref()
|
||||
.map(persisted_to_s3s::<_, ServerSideEncryptionConfiguration>)
|
||||
.transpose()?;
|
||||
Ok(S3Response::new(GetBucketEncryptionOutput {
|
||||
server_side_encryption_configuration,
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "get_bucket_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketTaggingInput>,
|
||||
) -> S3Result<S3Response<GetBucketTaggingOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketTagging);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let tagging = usecase.execute_get_bucket_tagging(req).await?;
|
||||
Ok(S3Response::new(GetBucketTaggingOutput {
|
||||
tag_set: tagging_to_s3s(&tagging).tag_set,
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "get_public_access_block", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_public_access_block_edge(
|
||||
&self,
|
||||
req: S3Request<GetPublicAccessBlockInput>,
|
||||
) -> S3Result<S3Response<GetPublicAccessBlockOutput>> {
|
||||
record_s3_op(S3Operation::GetPublicAccessBlock);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let config = usecase.execute_get_public_access_block(req).await?;
|
||||
Ok(S3Response::new(GetPublicAccessBlockOutput {
|
||||
public_access_block_configuration: Some(persisted_to_s3s(&config)?),
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "put_bucket_cors", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_cors_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketCorsInput>,
|
||||
) -> S3Result<S3Response<PutBucketCorsOutput>> {
|
||||
let prepared = persisted_from_s3s(&req.input.cors_configuration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketCors, |req| usecase.execute_put_bucket_cors(req, prepared)).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_encryption_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketEncryptionInput>,
|
||||
) -> S3Result<S3Response<PutBucketEncryptionOutput>> {
|
||||
let prepared = persisted_from_s3s(&req.input.server_side_encryption_configuration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketEncryption, |req| {
|
||||
usecase.execute_put_bucket_encryption(req, prepared)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(name = "put_public_access_block", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_public_access_block_edge(
|
||||
&self,
|
||||
req: S3Request<PutPublicAccessBlockInput>,
|
||||
) -> S3Result<S3Response<PutPublicAccessBlockOutput>> {
|
||||
let prepared = persisted_from_s3s(&req.input.public_access_block_configuration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutPublicAccessBlock, |req| {
|
||||
usecase.execute_put_public_access_block(req, prepared)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(name = "put_bucket_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketTaggingInput>,
|
||||
) -> S3Result<S3Response<PutBucketTaggingOutput>> {
|
||||
let tagging = tagging_from_s3s(&req.input.tagging);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketTagging, |req| usecase.execute_put_bucket_tagging(req, tagging)).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of bucket create, delete, head and location, and the website, accelerate,
|
||||
//! logging and request-payment configurations: batch A4 of the gateway migration
|
||||
//! (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::access::bucket_config_mutation_incarnation;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use crate::storage::storage_api::ecfs_consumer::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage::storage_api::ecstore_bucket::utils::serialize as serialize_bucket_config;
|
||||
use crate::storage::validate_website_configuration;
|
||||
use crate::storage::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_WEBSITE_CONFIG, StorageError,
|
||||
delete_bucket_metadata_config_if_incarnation, get_bucket_accelerate_config, get_bucket_logging_config,
|
||||
get_bucket_request_payment_config, get_bucket_website_config, update_bucket_metadata_config_if_incarnation,
|
||||
};
|
||||
use crate::storage::{persisted_from_s3s, persisted_to_s3s};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
/// Handler bodies that [`S3`] runs through `audit_only_operation`.
|
||||
impl FS {
|
||||
async fn put_bucket_logging_inner(
|
||||
&self,
|
||||
req: S3Request<PutBucketLoggingInput>,
|
||||
) -> S3Result<S3Response<PutBucketLoggingOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
record_s3_op(S3Operation::PutBucketLogging);
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
let bucket_logging_status = persisted_from_s3s::<_, rustfs_gateway_types::persistence::PersistedBucketLoggingStatus>(
|
||||
&req.input.bucket_logging_status,
|
||||
)?;
|
||||
let logging_config = serialize_bucket_config(&bucket_logging_status)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?;
|
||||
update_bucket_metadata_config_if_incarnation(
|
||||
&req.input.bucket,
|
||||
BUCKET_LOGGING_CONFIG,
|
||||
logging_config,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(PutBucketLoggingOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
impl FS {
|
||||
#[instrument(
|
||||
name = "create_bucket",
|
||||
level = "debug",
|
||||
skip(self, req),
|
||||
fields(start_time=?time::OffsetDateTime::now_utc())
|
||||
)]
|
||||
pub(in crate::storage::ecfs) async fn create_bucket_edge(
|
||||
&self,
|
||||
req: S3Request<CreateBucketInput>,
|
||||
) -> S3Result<S3Response<CreateBucketOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_create_bucket(req).await
|
||||
}
|
||||
|
||||
/// Delete a bucket
|
||||
#[instrument(name = "delete_bucket", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_delete_bucket(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_website_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketWebsiteInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketWebsiteOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
delete_bucket_metadata_config_if_incarnation(&req.input.bucket, BUCKET_WEBSITE_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(DeleteBucketWebsiteOutput::default()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_accelerate_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketAccelerateConfigurationInput>,
|
||||
) -> S3Result<S3Response<GetBucketAccelerateConfigurationOutput>> {
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
match get_bucket_accelerate_config(&req.input.bucket).await {
|
||||
Ok((accelerate, _)) => Ok(S3Response::new(GetBucketAccelerateConfigurationOutput {
|
||||
status: accelerate.status.map(BucketAccelerateStatus::from),
|
||||
..Default::default()
|
||||
})),
|
||||
Err(StorageError::ConfigNotFound) => Ok(S3Response::new(GetBucketAccelerateConfigurationOutput::default())),
|
||||
Err(err) => Err(crate::error::ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get bucket location
|
||||
#[instrument(name = "get_bucket_location", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_location_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketLocationInput>,
|
||||
) -> S3Result<S3Response<GetBucketLocationOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketLocation);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_get_bucket_location(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_request_payment_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketRequestPaymentInput>,
|
||||
) -> S3Result<S3Response<GetBucketRequestPaymentOutput>> {
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
match get_bucket_request_payment_config(&req.input.bucket).await {
|
||||
Ok((payment, _)) => Ok(S3Response::new(GetBucketRequestPaymentOutput {
|
||||
payer: Some(Payer::from(payment.payer)),
|
||||
})),
|
||||
Err(StorageError::ConfigNotFound) => Ok(S3Response::new(GetBucketRequestPaymentOutput {
|
||||
payer: Some(Payer::from_static(Payer::BUCKET_OWNER)),
|
||||
})),
|
||||
Err(err) => Err(crate::error::ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_website_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketWebsiteInput>,
|
||||
) -> S3Result<S3Response<GetBucketWebsiteOutput>> {
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
match get_bucket_website_config(&req.input.bucket).await {
|
||||
Ok((website, _)) => {
|
||||
let website: WebsiteConfiguration = persisted_to_s3s(&website)?;
|
||||
Ok(S3Response::new(GetBucketWebsiteOutput {
|
||||
error_document: website.error_document,
|
||||
index_document: website.index_document,
|
||||
redirect_all_requests_to: website.redirect_all_requests_to,
|
||||
routing_rules: website.routing_rules,
|
||||
}))
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => Err(s3_error!(NoSuchWebsiteConfiguration)),
|
||||
Err(err) => Err(crate::error::ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(name = "head_bucket", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn head_bucket_edge(
|
||||
&self,
|
||||
req: S3Request<HeadBucketInput>,
|
||||
) -> S3Result<S3Response<HeadBucketOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_head_bucket(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_accelerate_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketAccelerateConfigurationInput>,
|
||||
) -> S3Result<S3Response<PutBucketAccelerateConfigurationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
let accelerate_configuration = persisted_from_s3s::<
|
||||
_,
|
||||
rustfs_gateway_types::persistence::PersistedAccelerateConfiguration,
|
||||
>(&req.input.accelerate_configuration)?;
|
||||
let accelerate_config = serialize_bucket_config(&accelerate_configuration)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?;
|
||||
update_bucket_metadata_config_if_incarnation(
|
||||
&req.input.bucket,
|
||||
BUCKET_ACCELERATE_CONFIG,
|
||||
accelerate_config,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(PutBucketAccelerateConfigurationOutput::default()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_logging_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketLoggingInput>,
|
||||
) -> S3Result<S3Response<GetBucketLoggingOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketLogging);
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
match get_bucket_logging_config(&req.input.bucket).await {
|
||||
Ok((logging, _)) => {
|
||||
let logging: BucketLoggingStatus = persisted_to_s3s(&logging)?;
|
||||
Ok(S3Response::new(GetBucketLoggingOutput {
|
||||
logging_enabled: logging.logging_enabled,
|
||||
}))
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => Ok(S3Response::new(GetBucketLoggingOutput::default())),
|
||||
Err(err) => Err(crate::error::ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_logging_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketLoggingInput>,
|
||||
) -> S3Result<S3Response<PutBucketLoggingOutput>> {
|
||||
audit_only_operation(req, S3Operation::PutBucketLogging, |req| self.put_bucket_logging_inner(req)).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_request_payment_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketRequestPaymentInput>,
|
||||
) -> S3Result<S3Response<PutBucketRequestPaymentOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
let request_payment_configuration = persisted_from_s3s::<
|
||||
_,
|
||||
rustfs_gateway_types::persistence::PersistedRequestPaymentConfiguration,
|
||||
>(&req.input.request_payment_configuration)?;
|
||||
let payment_config = serialize_bucket_config(&request_payment_configuration)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?;
|
||||
update_bucket_metadata_config_if_incarnation(
|
||||
&req.input.bucket,
|
||||
BUCKET_REQUEST_PAYMENT_CONFIG,
|
||||
payment_config,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(PutBucketRequestPaymentOutput::default()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_website_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketWebsiteInput>,
|
||||
) -> S3Result<S3Response<PutBucketWebsiteOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
let website_configuration = persisted_from_s3s::<_, rustfs_gateway_types::persistence::PersistedWebsiteConfiguration>(
|
||||
&req.input.website_configuration,
|
||||
)?;
|
||||
validate_website_configuration(&website_configuration)?;
|
||||
|
||||
let website_config = serialize_bucket_config(&website_configuration)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?;
|
||||
update_bucket_metadata_config_if_incarnation(
|
||||
&req.input.bucket,
|
||||
BUCKET_WEBSITE_CONFIG,
|
||||
website_config,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(PutBucketWebsiteOutput::default()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of the bucket lifecycle, replication and Object Lock configurations: batch A3 of
|
||||
//! the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook;
|
||||
use crate::app::storage_api::object_usecase::bucket::replication::OperatorRuleContract;
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::StorageVersioningConfigExt as _;
|
||||
use crate::storage::access::bucket_config_mutation_incarnation;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use crate::storage::s3_compat::lifecycle::{lifecycle_configuration_from_s3s, lifecycle_configuration_to_s3s};
|
||||
use crate::storage::s3_compat::replication::{replication_configuration_from_s3s, replication_configuration_to_s3s};
|
||||
use crate::storage::storage_api::ecfs_consumer::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage::storage_api::ecstore_bucket::utils::serialize as serialize_bucket_config;
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
use crate::storage::{
|
||||
BUCKET_VERSIONING_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, StorageError, get_bucket_object_lock_config,
|
||||
update_bucket_metadata_config_if_incarnation,
|
||||
};
|
||||
use crate::storage::{persisted_from_s3s, persisted_to_s3s};
|
||||
use http::StatusCode;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::{instrument, warn};
|
||||
|
||||
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) static SITE_REPLICATION_GATE_TEST_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) const SITE_REPLICATION_GATE_FORCE_DISABLED: u8 = 1;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) const SITE_REPLICATION_GATE_FORCE_ENABLED: u8 = 2;
|
||||
|
||||
async fn site_replication_gate_enabled() -> S3Result<bool> {
|
||||
#[cfg(test)]
|
||||
match SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED => return Ok(false),
|
||||
SITE_REPLICATION_GATE_FORCE_ENABLED => return Ok(true),
|
||||
_ => {}
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_enabled()
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Remote site-replication peer deployment ids and the operator-rule contract
|
||||
/// the peers support, handed to the bucket usecase so an S3 replication-config
|
||||
/// edit keeps exactly the reconciler-owned rules and merges the way every
|
||||
/// peer will (issue #1948). Read here, in the interface layer, because the
|
||||
/// usecase must not import the admin handlers (layer guard); a state-read
|
||||
/// failure propagates so the edit fails closed.
|
||||
async fn site_replication_edit_context() -> S3Result<(std::collections::HashSet<String>, OperatorRuleContract)> {
|
||||
// While the gate override is in effect the test exercises the deny/allow
|
||||
// branch, not the peer set; there is no persisted state to read.
|
||||
#[cfg(test)]
|
||||
if SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) != 0 {
|
||||
return Ok((std::collections::HashSet::new(), OperatorRuleContract::Derived));
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_edit_context()
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// MinIO `ErrReplicationDenyEditError`.
|
||||
fn replication_deny_edit_error() -> S3Error {
|
||||
let mut err = S3Error::with_message(
|
||||
S3ErrorCode::Custom("XMinioReplicationDenyEdit".into()),
|
||||
"Sub-User is not allowed to edit Replication configuration",
|
||||
);
|
||||
err.set_status_code(StatusCode::BAD_REQUEST);
|
||||
err
|
||||
}
|
||||
|
||||
/// Site-replication gate for S3 replication-config edits (issue #1948).
|
||||
///
|
||||
/// On a site-replication deployment the bucket's replication config carries
|
||||
/// the operator-managed `site-repl-*` rules that keep every peer in sync, and
|
||||
/// a successful edit is broadcast to all peers — so a user holding only
|
||||
/// bucket-scoped `s3:PutReplicationConfiguration` could rewrite or erase
|
||||
/// replication net-wide. MinIO parity (`ErrReplicationDenyEditError`): only
|
||||
/// owner credentials (root or root-parented) may edit. Runs after the policy
|
||||
/// authorization in the access layer and only on the external S3 path — the
|
||||
/// reconciler and peer bucket-meta ingestion never route through these
|
||||
/// handlers.
|
||||
async fn deny_replication_config_edit_for_non_owner<T>(req: &S3Request<T>) -> S3Result<()> {
|
||||
if crate::storage::access::req_info_ref(req)?.is_owner {
|
||||
return Ok(());
|
||||
}
|
||||
if site_replication_gate_enabled().await? {
|
||||
return Err(replication_deny_edit_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handler bodies that [`S3`] runs through `audit_only_operation`.
|
||||
impl FS {
|
||||
async fn delete_bucket_replication_inner(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let (site_peers, contract) = site_replication_edit_context().await?;
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_delete_bucket_replication(req, site_peers, contract).await
|
||||
}
|
||||
|
||||
async fn put_bucket_replication_inner(
|
||||
&self,
|
||||
mut req: S3Request<PutBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let (site_peers, contract) = site_replication_edit_context().await?;
|
||||
let replication_configuration =
|
||||
replication_configuration_from_s3s(std::mem::take(&mut req.input.replication_configuration));
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase
|
||||
.execute_put_bucket_replication(req, replication_configuration, site_peers, contract)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_object_lock_configuration_inner(
|
||||
&self,
|
||||
req: S3Request<PutObjectLockConfigurationInput>,
|
||||
) -> S3Result<S3Response<PutObjectLockConfigurationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let PutObjectLockConfigurationInput {
|
||||
bucket,
|
||||
object_lock_configuration,
|
||||
..
|
||||
} = req.input;
|
||||
|
||||
let Some(input_cfg) = object_lock_configuration else { return Err(s3_error!(InvalidArgument)) };
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
validate_object_lock_configuration_input(&input_cfg)?;
|
||||
|
||||
match get_bucket_object_lock_config(&bucket).await {
|
||||
Ok(_) => {}
|
||||
Err(err) => {
|
||||
if err == StorageError::ConfigNotFound {
|
||||
if !BucketVersioningSys::enabled(&bucket).await {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidBucketState,
|
||||
"Object Lock configuration cannot be enabled on existing buckets".to_string(),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
warn!("get_object_lock_config err {:?}", err);
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
"Failed to get bucket ObjectLockConfiguration".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let persisted_cfg =
|
||||
persisted_from_s3s::<_, rustfs_gateway_types::persistence::PersistedObjectLockConfiguration>(&input_cfg)?;
|
||||
let data = serialize_bucket_config(&persisted_cfg)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")))?;
|
||||
let object_lock_config =
|
||||
String::from_utf8(data.clone()).map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")))?;
|
||||
|
||||
let updated_at = update_bucket_metadata_config_if_incarnation(&bucket, OBJECT_LOCK_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
// When Object Lock is enabled, automatically enable versioning if not already enabled.
|
||||
// This matches S3-compatible behavior.
|
||||
let versioning_config = BucketVersioningSys::get(&bucket).await.map_err(ApiError::from)?;
|
||||
if !versioning_config.enabled() {
|
||||
let enable_versioning_config = rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let versioning_data = serialize_bucket_config(&enable_versioning_config)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")))?;
|
||||
update_bucket_metadata_config_if_incarnation(
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
versioning_data,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
|
||||
if let Some(notification_sys) =
|
||||
runtime_sources::current_notification_system_for_context(self.server_ctx.app_context().as_deref())
|
||||
{
|
||||
propagate_object_lock_peer_reload(notification_sys.load_bucket_metadata(&bucket).await)?;
|
||||
}
|
||||
|
||||
if let Err(err) = site_replication_bucket_meta_hook(SRBucketMeta {
|
||||
bucket: bucket.clone(),
|
||||
r#type: "object-lock-config".to_string(),
|
||||
object_lock_config: Some(object_lock_config),
|
||||
updated_at: Some(updated_at),
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT_LOCK,
|
||||
event = "put_object_lock_configuration",
|
||||
bucket = %bucket,
|
||||
result = "site_replication_hook_failed",
|
||||
error = ?err,
|
||||
"storage object lock state"
|
||||
);
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket_from_producer(
|
||||
&bucket,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata,
|
||||
);
|
||||
Ok(S3Response::new(PutObjectLockConfigurationOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "delete_bucket_lifecycle", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_lifecycle_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketLifecycleInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketLifecycleOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketLifecycle, |req| {
|
||||
usecase.execute_delete_bucket_lifecycle(req)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_replication_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
audit_only_operation(req, S3Operation::DeleteBucketReplication, |req| self.delete_bucket_replication_inner(req)).await
|
||||
}
|
||||
|
||||
#[instrument(name = "get_bucket_lifecycle_configuration", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_lifecycle_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketLifecycleConfigurationInput>,
|
||||
) -> S3Result<S3Response<GetBucketLifecycleConfigurationOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketLifecycleConfiguration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let config = usecase.execute_get_bucket_lifecycle_configuration(req).await?;
|
||||
let config = lifecycle_configuration_to_s3s(config)
|
||||
.map_err(|error| S3Error::with_message(S3ErrorCode::InternalError, error.to_string()))?;
|
||||
Ok(S3Response::new(GetBucketLifecycleConfigurationOutput {
|
||||
rules: Some(config.rules),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_replication_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<GetBucketReplicationOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketReplication);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let replication_configuration = usecase.execute_get_bucket_replication(req).await?;
|
||||
Ok(S3Response::new(GetBucketReplicationOutput {
|
||||
replication_configuration: Some(replication_configuration_to_s3s(replication_configuration)),
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "get_object_lock_configuration", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_object_lock_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectLockConfigurationInput>,
|
||||
) -> S3Result<S3Response<GetObjectLockConfigurationOutput>> {
|
||||
record_s3_op(S3Operation::GetObjectLockConfiguration);
|
||||
let GetObjectLockConfigurationInput { bucket, .. } = req.input;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let object_lock_configuration = match get_bucket_object_lock_config(&bucket).await {
|
||||
Ok((cfg, _created)) => Some(persisted_to_s3s::<_, ObjectLockConfiguration>(&cfg)?),
|
||||
Err(err) => {
|
||||
if err == StorageError::ConfigNotFound {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::ObjectLockConfigurationNotFoundError,
|
||||
"Object Lock configuration does not exist for this bucket".to_string(),
|
||||
));
|
||||
}
|
||||
warn!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT_LOCK,
|
||||
event = "object_lock_config_load_failed",
|
||||
bucket = %bucket,
|
||||
error = ?err,
|
||||
"Failed to load bucket object lock configuration"
|
||||
);
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
"Failed to load Object Lock configuration".to_string(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
Ok(S3Response::new(GetObjectLockConfigurationOutput {
|
||||
object_lock_configuration,
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "put_bucket_lifecycle_configuration", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_lifecycle_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketLifecycleConfigurationInput>,
|
||||
) -> S3Result<S3Response<PutBucketLifecycleConfigurationOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let prepared = req
|
||||
.input
|
||||
.lifecycle_configuration
|
||||
.clone()
|
||||
.map(lifecycle_configuration_from_s3s)
|
||||
.transpose();
|
||||
audit_only_operation(req, S3Operation::PutBucketLifecycleConfiguration, |req| {
|
||||
usecase.execute_put_bucket_lifecycle_configuration(req, prepared)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_replication_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
audit_only_operation(req, S3Operation::PutBucketReplication, |req| self.put_bucket_replication_inner(req)).await
|
||||
}
|
||||
|
||||
#[instrument(name = "put_object_lock_configuration", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_object_lock_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectLockConfigurationInput>,
|
||||
) -> S3Result<S3Response<PutObjectLockConfigurationOutput>> {
|
||||
audit_only_operation(req, S3Operation::PutObjectLockConfiguration, |req| {
|
||||
self.put_object_lock_configuration_inner(req)
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of the object and multipart-upload listings, CreateMultipartUpload and
|
||||
//! AbortMultipartUpload: batch B2 of the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "abort_multipart_upload", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn abort_multipart_upload_edge(
|
||||
&self,
|
||||
req: S3Request<AbortMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<AbortMultipartUploadOutput>> {
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::AbortMultipartUpload, |req| usecase.execute_abort_multipart_upload(req)).await
|
||||
}
|
||||
|
||||
#[instrument(name = "create_multipart_upload", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn create_multipart_upload_edge(
|
||||
&self,
|
||||
req: S3Request<CreateMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CreateMultipartUploadOutput>> {
|
||||
crate::hp_guard!("S3::create_multipart_upload");
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
usecase.execute_create_multipart_upload(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn list_multipart_uploads_edge(
|
||||
&self,
|
||||
req: S3Request<ListMultipartUploadsInput>,
|
||||
) -> S3Result<S3Response<ListMultipartUploadsOutput>> {
|
||||
record_s3_op(S3Operation::ListMultipartUploads);
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
usecase.execute_list_multipart_uploads(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn list_object_versions_edge(
|
||||
&self,
|
||||
req: S3Request<ListObjectVersionsInput>,
|
||||
) -> S3Result<S3Response<ListObjectVersionsOutput>> {
|
||||
record_s3_op(S3Operation::ListObjectVersions);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_list_object_versions(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "list_objects", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn list_objects_edge(
|
||||
&self,
|
||||
req: S3Request<ListObjectsInput>,
|
||||
) -> S3Result<S3Response<ListObjectsOutput>> {
|
||||
record_s3_op(S3Operation::ListObjects);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_list_objects(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "list_objects_v2", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn list_objects_v2_edge(
|
||||
&self,
|
||||
req: S3Request<ListObjectsV2Input>,
|
||||
) -> S3Result<S3Response<ListObjectsV2Output>> {
|
||||
crate::hp_guard!("S3::list_objects_v2");
|
||||
record_s3_op(S3Operation::ListObjectsV2);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_list_objects_v2(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "list_parts", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn list_parts_edge(
|
||||
&self,
|
||||
req: S3Request<ListPartsInput>,
|
||||
) -> S3Result<S3Response<ListPartsOutput>> {
|
||||
record_s3_op(S3Operation::ListParts);
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
usecase.execute_list_parts(req).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The bodies of `impl S3 for FS`, one file per batch of the s3s-to-gateway migration
|
||||
//! (rustfs/backlog#2765), so that parallel batches edit disjoint files. `ecfs.rs` keeps the
|
||||
//! trait impl, which delegates every method here, and the helpers several batches share.
|
||||
//! `scripts/codemods/split_ecfs_legacy_edge.py` produced this split from the unsplit file.
|
||||
|
||||
mod bucket_access;
|
||||
mod bucket_configs;
|
||||
mod bucket_core;
|
||||
mod bucket_replication_lifecycle;
|
||||
mod listing_multipart;
|
||||
mod object_meta;
|
||||
mod object_misc;
|
||||
mod object_read;
|
||||
mod object_write;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,976 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of object ACL, tagging, legal hold, retention and torrent: batch B1 of the
|
||||
//! gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::app::storage_api::object_usecase::bucket::replication::{
|
||||
ReplicateDecision, get_read_proxy_targets, must_replicate_metadata, schedule_metadata_replication,
|
||||
};
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::access::{apply_bucket_generation_guard, has_bypass_governance_header, replication_proxy_allowed};
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::OperationHelper;
|
||||
use crate::storage::options::get_opts;
|
||||
use crate::storage::s3_api::{self, acl};
|
||||
use crate::storage::storage_api::ecfs_consumer::StorageObjectOptions as ObjectOptions;
|
||||
use crate::storage::storage_api::ecfs_consumer::contract::{
|
||||
bucket::{BucketOperations, BucketOptions},
|
||||
object::ObjectLockRetentionOptions,
|
||||
};
|
||||
use crate::storage::storage_api::ecfs_consumer::object_lock::{
|
||||
parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled,
|
||||
};
|
||||
use crate::storage::{
|
||||
BucketVersioningSys, check_retention_for_modification, decode_tags, encode_tags, is_err_object_not_found,
|
||||
is_err_version_not_found, record_replication_proxy,
|
||||
};
|
||||
use crate::table_catalog;
|
||||
use metrics::{counter, histogram};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER;
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_REPLICATION_GENERATION, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, insert_str,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tracing::{debug, error, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
const LOG_SUBSYSTEM_TAGGING: &str = "tagging";
|
||||
|
||||
async fn reject_veeam_sosapi_mutation(fs: &FS, bucket: &str, key: &str) -> S3Result<()> {
|
||||
s3_api::reject_veeam_sosapi_mutation(|| fs.server_ctx.object_store(), bucket, key).await
|
||||
}
|
||||
|
||||
async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> {
|
||||
table_catalog::validate_bucket_object_mutation(bucket, key)
|
||||
.await
|
||||
.map_err(|_| s3_error!(InvalidRequest, "{}", table_catalog::RESERVED_CATALOG_OBJECT_MESSAGE))
|
||||
}
|
||||
|
||||
impl FS {
|
||||
/// Not-found classifier for proxied SDK tagging calls: a raw 404 covers
|
||||
/// NoSuchKey and NoSuchVersion alike; the caller silently tries the next
|
||||
/// replication target.
|
||||
fn proxy_sdk_error_is_not_found<E>(err: &aws_sdk_s3::error::SdkError<E>) -> bool {
|
||||
err.raw_response().is_some_and(|resp| resp.status().as_u16() == 404)
|
||||
}
|
||||
|
||||
/// Selector options for a tagging proxy. Reuses `get_opts` so the
|
||||
/// anti-loop `source-proxy-request` header family and the bucket's
|
||||
/// version-suspension state gate proxying exactly like GET/HEAD.
|
||||
async fn tagging_proxy_opts(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
headers: &http::HeaderMap,
|
||||
) -> Option<ObjectOptions> {
|
||||
get_opts(bucket, object, version_id, None, headers).await.ok()
|
||||
}
|
||||
|
||||
/// Serve a GetObjectTagging for an object missing locally by proxying to
|
||||
/// the bucket's replication targets (MinIO `proxyGetTaggingToRepTarget`,
|
||||
/// backlog#1675 P1-5). None means no target had the object.
|
||||
async fn proxy_get_object_tagging(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
headers: &http::HeaderMap,
|
||||
) -> Option<GetObjectTaggingOutput> {
|
||||
let opts = Self::tagging_proxy_opts(bucket, object, version_id, headers).await?;
|
||||
let targets = get_read_proxy_targets(bucket, object, &opts).await;
|
||||
if targets.is_empty() {
|
||||
return None;
|
||||
}
|
||||
for target in targets {
|
||||
match target
|
||||
.get_object_tagging(&target.bucket, object, opts.version_id.clone())
|
||||
.await
|
||||
{
|
||||
Ok(remote) => {
|
||||
// MinIO-aligned accounting: one total per proxy attempt,
|
||||
// one failed when no target served it.
|
||||
record_replication_proxy(bucket, "GetObjectTagging", false).await;
|
||||
return Some(GetObjectTaggingOutput {
|
||||
tag_set: remote
|
||||
.tag_set
|
||||
.into_iter()
|
||||
.map(|tag| Tag {
|
||||
key: Some(tag.key),
|
||||
value: Some(tag.value),
|
||||
})
|
||||
.collect(),
|
||||
version_id: remote.version_id,
|
||||
});
|
||||
}
|
||||
Err(err) if Self::proxy_sdk_error_is_not_found(&err) => {
|
||||
debug!(bucket, object, arn = %target.arn, "tagging proxy: target does not have the object");
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(bucket, object, arn = %target.arn, error = %err, "tagging proxy: GetObjectTagging against replication target failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
record_replication_proxy(bucket, "GetObjectTagging", true).await;
|
||||
None
|
||||
}
|
||||
|
||||
/// Apply a PutObjectTagging for an object missing locally on a
|
||||
/// replication target (MinIO `proxyTaggingToRepTarget`).
|
||||
async fn proxy_put_object_tagging(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
headers: &http::HeaderMap,
|
||||
tag_set: &TagSet,
|
||||
) -> Option<()> {
|
||||
let opts = Self::tagging_proxy_opts(bucket, object, version_id, headers).await?;
|
||||
let mut tagging = aws_sdk_s3::types::Tagging::builder();
|
||||
for tag in tag_set {
|
||||
let sdk_tag = aws_sdk_s3::types::Tag::builder()
|
||||
.key(tag.key.clone().unwrap_or_default())
|
||||
.value(tag.value.clone().unwrap_or_default())
|
||||
.build()
|
||||
.ok()?;
|
||||
tagging = tagging.tag_set(sdk_tag);
|
||||
}
|
||||
let tagging = tagging.build().ok()?;
|
||||
let targets = get_read_proxy_targets(bucket, object, &opts).await;
|
||||
if targets.is_empty() {
|
||||
return None;
|
||||
}
|
||||
for target in targets {
|
||||
match target
|
||||
.put_object_tagging(&target.bucket, object, opts.version_id.clone(), tagging.clone())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
// MinIO-aligned accounting: one total per proxy attempt,
|
||||
// one failed when no target served it.
|
||||
record_replication_proxy(bucket, "PutObjectTagging", false).await;
|
||||
return Some(());
|
||||
}
|
||||
Err(err) if Self::proxy_sdk_error_is_not_found(&err) => {
|
||||
debug!(bucket, object, arn = %target.arn, "tagging proxy: target does not have the object");
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(bucket, object, arn = %target.arn, error = %err, "tagging proxy: PutObjectTagging against replication target failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
record_replication_proxy(bucket, "PutObjectTagging", true).await;
|
||||
None
|
||||
}
|
||||
|
||||
/// Apply a DeleteObjectTagging for an object missing locally on a
|
||||
/// replication target (MinIO `proxyTaggingToRepTarget`).
|
||||
async fn proxy_delete_object_tagging(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
headers: &http::HeaderMap,
|
||||
) -> Option<()> {
|
||||
let opts = Self::tagging_proxy_opts(bucket, object, version_id, headers).await?;
|
||||
let targets = get_read_proxy_targets(bucket, object, &opts).await;
|
||||
if targets.is_empty() {
|
||||
return None;
|
||||
}
|
||||
for target in targets {
|
||||
match target
|
||||
.delete_object_tagging(&target.bucket, object, opts.version_id.clone())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
// MinIO-aligned accounting: one total per proxy attempt,
|
||||
// one failed when no target served it.
|
||||
record_replication_proxy(bucket, "DeleteObjectTagging", false).await;
|
||||
return Some(());
|
||||
}
|
||||
Err(err) if Self::proxy_sdk_error_is_not_found(&err) => {
|
||||
debug!(bucket, object, arn = %target.arn, "tagging proxy: target does not have the object");
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(bucket, object, arn = %target.arn, error = %err, "tagging proxy: DeleteObjectTagging against replication target failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
record_replication_proxy(bucket, "DeleteObjectTagging", true).await;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "delete_object_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_object_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectTaggingInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectTaggingOutput>> {
|
||||
let start_time = std::time::Instant::now();
|
||||
let mut helper = OperationHelper::new(&req, EventName::ObjectTaggingDelete, S3Operation::DeleteObjectTagging);
|
||||
let DeleteObjectTaggingInput {
|
||||
bucket,
|
||||
key: object,
|
||||
version_id,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &object).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &object).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_store_uninitialized",
|
||||
operation = "delete",
|
||||
"Object tagging operation failed because storage is not initialized"
|
||||
);
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let mut opts = get_opts(&bucket, &object, version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let existing_object_info = match store.get_object_info(&bucket, &object, &opts).await {
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
// Replication lag window: apply the tagging delete on a
|
||||
// replication target that already has the object
|
||||
// (backlog#1675 P1-5). No local object exists, so no bucket
|
||||
// notification event is emitted for the proxied write.
|
||||
if (is_err_object_not_found(&e) || is_err_version_not_found(&e))
|
||||
&& replication_proxy_allowed(&req.extensions)
|
||||
&& Self::proxy_delete_object_tagging(&bucket, &object, version_id.clone(), &req.headers)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
counter!("rustfs_delete_object_tagging_success").increment(1);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "delete")
|
||||
.record(duration.as_secs_f64());
|
||||
return Ok(S3Response::new(DeleteObjectTaggingOutput { version_id }));
|
||||
}
|
||||
return Err(ApiError::from(e).into());
|
||||
}
|
||||
};
|
||||
let dsc = must_replicate_metadata(
|
||||
&bucket,
|
||||
&object,
|
||||
&existing_object_info.user_defined,
|
||||
String::new(),
|
||||
existing_object_info.replication_status.clone(),
|
||||
opts.clone(),
|
||||
)
|
||||
.await;
|
||||
if dsc.replicate_any() {
|
||||
let mut eval_metadata = HashMap::new();
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
|
||||
insert_str(
|
||||
&mut eval_metadata,
|
||||
SUFFIX_TAGGING_TIMESTAMP,
|
||||
OffsetDateTime::now_utc().format(&Rfc3339).unwrap_or_default(),
|
||||
);
|
||||
opts.eval_metadata = Some(eval_metadata);
|
||||
}
|
||||
|
||||
let delete_tags_result = store.delete_object_tags(&bucket, &object, &opts).await;
|
||||
let object_info = delete_tags_result.map_err(|e| {
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_delete_failed",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
error = %e,
|
||||
"Failed to delete object tags"
|
||||
);
|
||||
ApiError::from(e)
|
||||
})?;
|
||||
|
||||
let event_object_info = Some(object_info.clone());
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(object_info, store.clone(), dsc).await;
|
||||
}
|
||||
|
||||
counter!("rustfs_delete_object_tagging_success").increment(1);
|
||||
|
||||
let event_version_id = version_id
|
||||
.as_deref()
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_string)
|
||||
.or_else(|| {
|
||||
event_object_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.version_id.map(|version_id| version_id.to_string()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if let Some(event_object_info) = event_object_info {
|
||||
helper = helper.object(event_object_info);
|
||||
}
|
||||
helper = helper.version_id(event_version_id);
|
||||
|
||||
let result = Ok(S3Response::new(DeleteObjectTaggingOutput { version_id }));
|
||||
let _ = helper.complete(&result);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&object,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata,
|
||||
);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "delete").record(duration.as_secs_f64());
|
||||
result
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_object_acl_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectAclInput>,
|
||||
) -> S3Result<S3Response<GetObjectAclOutput>> {
|
||||
record_s3_op(S3Operation::GetObjectAcl);
|
||||
let GetObjectAclInput {
|
||||
bucket, key, version_id, ..
|
||||
} = req.input;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
store.get_object_info(&bucket, &key, &opts).await.map_err(ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(acl::build_get_object_acl_output()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_object_legal_hold_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectLegalHoldInput>,
|
||||
) -> S3Result<S3Response<GetObjectLegalHoldOutput>> {
|
||||
let mut helper =
|
||||
OperationHelper::new(&req, EventName::ObjectAccessedGetLegalHold, S3Operation::GetObjectLegalHold).suppress_event();
|
||||
let GetObjectLegalHoldInput {
|
||||
bucket, key, version_id, ..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let _ = store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
validate_bucket_object_lock_enabled(&bucket).await?;
|
||||
|
||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| {
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
event = "object_info_load_failed",
|
||||
bucket = %bucket,
|
||||
object = %key,
|
||||
error = %e,
|
||||
"Failed to load object info"
|
||||
);
|
||||
s3_error!(InternalError, "{}", e.to_string())
|
||||
})?;
|
||||
|
||||
let legal_hold = object_info
|
||||
.user_defined
|
||||
.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER)
|
||||
.map(|v| v.as_str().to_string());
|
||||
|
||||
let status = if let Some(v) = legal_hold {
|
||||
v
|
||||
} else {
|
||||
ObjectLockLegalHoldStatus::OFF.to_string()
|
||||
};
|
||||
|
||||
let output = GetObjectLegalHoldOutput {
|
||||
legal_hold: Some(ObjectLockLegalHold {
|
||||
status: Some(ObjectLockLegalHoldStatus::from(status)),
|
||||
}),
|
||||
};
|
||||
|
||||
let version_id = req.input.version_id.clone().unwrap_or_else(|| Uuid::new_v4().to_string());
|
||||
helper = helper.object(object_info).version_id(version_id);
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_object_retention_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectRetentionInput>,
|
||||
) -> S3Result<S3Response<GetObjectRetentionOutput>> {
|
||||
let mut helper =
|
||||
OperationHelper::new(&req, EventName::ObjectAccessedGetRetention, S3Operation::GetObjectRetention).suppress_event();
|
||||
let GetObjectRetentionInput {
|
||||
bucket, key, version_id, ..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
validate_bucket_object_lock_enabled(&bucket).await?;
|
||||
|
||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| {
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
event = "object_info_load_failed",
|
||||
bucket = %bucket,
|
||||
object = %key,
|
||||
error = %e,
|
||||
"Failed to load object info"
|
||||
);
|
||||
s3_error!(InternalError, "{}", e.to_string())
|
||||
})?;
|
||||
|
||||
let output = GetObjectRetentionOutput {
|
||||
retention: Some(object_lock_retention_from_metadata(&object_info.user_defined)?),
|
||||
};
|
||||
let version_id = req.input.version_id.clone().unwrap_or_default();
|
||||
helper = helper.object(object_info).version_id(version_id);
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(name = "get_object_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_object_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectTaggingInput>,
|
||||
) -> S3Result<S3Response<GetObjectTaggingOutput>> {
|
||||
use crate::storage::storage_api::ecstore_bucket::versioning::VersioningApi as _;
|
||||
|
||||
record_s3_op(S3Operation::GetObjectTagging);
|
||||
let start_time = std::time::Instant::now();
|
||||
let bucket = req.input.bucket.as_str();
|
||||
let object = req.input.key.as_str();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, bucket, object).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_store_uninitialized",
|
||||
operation = "get",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
"Object tagging operation failed because storage is not initialized"
|
||||
);
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let version_id = parse_object_version_id(req.input.version_id.clone())?.map(Into::into);
|
||||
let versioning = BucketVersioningSys::get(bucket).await.map_err(ApiError::from)?;
|
||||
let opts = ObjectOptions {
|
||||
version_id,
|
||||
versioned: versioning.prefix_enabled(object),
|
||||
version_suspended: versioning.prefix_suspended(object),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Tags and response identity must come from the same locked metadata snapshot.
|
||||
let info = match store.get_object_info(bucket, object, &opts).await {
|
||||
Ok(info) if info.delete_marker => {
|
||||
return Err(S3Error::new(if opts.version_id.is_some() {
|
||||
S3ErrorCode::MethodNotAllowed
|
||||
} else {
|
||||
S3ErrorCode::NoSuchKey
|
||||
}));
|
||||
}
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
// Replication lag window: the object may exist on a
|
||||
// replication target even though it is missing locally —
|
||||
// proxy the tagging read there (backlog#1675 P1-5).
|
||||
if (is_err_object_not_found(&e) || is_err_version_not_found(&e))
|
||||
&& replication_proxy_allowed(&req.extensions)
|
||||
&& let Some(output) =
|
||||
Self::proxy_get_object_tagging(bucket, object, req.input.version_id.clone(), &req.headers).await
|
||||
{
|
||||
counter!("rustfs_get_object_tagging_success").increment(1);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "get")
|
||||
.record(duration.as_secs_f64());
|
||||
return Ok(S3Response::new(output));
|
||||
}
|
||||
if is_err_object_not_found(&e) {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_not_found",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
error = %e,
|
||||
"Object tags not found"
|
||||
);
|
||||
return Err(S3Error::new(if opts.version_id.is_some() {
|
||||
S3ErrorCode::NoSuchVersion
|
||||
} else {
|
||||
S3ErrorCode::NoSuchKey
|
||||
}));
|
||||
}
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_get_failed",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
error = %e,
|
||||
"Failed to load object tags"
|
||||
);
|
||||
return Err(ApiError::from(e).into());
|
||||
}
|
||||
};
|
||||
|
||||
let tag_set = decode_tags(info.user_tags.as_str());
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_decoded",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
tag_count = tag_set.len(),
|
||||
"Decoded object tags"
|
||||
);
|
||||
|
||||
counter!("rustfs_get_object_tagging_success").increment(1);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "get").record(duration.as_secs_f64());
|
||||
Ok(S3Response::new(GetObjectTaggingOutput {
|
||||
tag_set,
|
||||
version_id: s3_api::s3_response_version_id(info.version_id),
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "get_object_torrent", level = "debug", skip(self, _req))]
|
||||
pub(in crate::storage::ecfs) async fn get_object_torrent_edge(
|
||||
&self,
|
||||
_req: S3Request<GetObjectTorrentInput>,
|
||||
) -> S3Result<S3Response<GetObjectTorrentOutput>> {
|
||||
// Torrent functionality is not implemented in RustFS
|
||||
// Per S3 API test expectations, return 404 NoSuchKey (not 501 Not Implemented)
|
||||
// This allows clients to gracefully handle the absence of torrent support
|
||||
record_s3_op(S3Operation::GetObjectTorrent);
|
||||
Err(S3Error::new(S3ErrorCode::NoSuchKey))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_object_acl_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectAclInput>,
|
||||
) -> S3Result<S3Response<PutObjectAclOutput>> {
|
||||
let mut helper = OperationHelper::new(&req, EventName::ObjectAclPut, S3Operation::PutObjectAcl);
|
||||
let bucket = &req.input.bucket;
|
||||
let key = &req.input.key;
|
||||
let version_id = req.input.version_id.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, bucket, key).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let opts: ObjectOptions = get_opts(bucket, key, version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let object_info = store.get_object_info(bucket, key, &opts).await.map_err(ApiError::from)?;
|
||||
|
||||
if req.input.access_control_policy.is_some() {
|
||||
return Err(s3_error!(
|
||||
NotImplemented,
|
||||
"ACL XML grants are not supported; use canned ACL headers or omit ACL"
|
||||
));
|
||||
}
|
||||
|
||||
let event_version_id = version_id
|
||||
.or_else(|| object_info.version_id.map(|version_id| version_id.to_string()))
|
||||
.unwrap_or_default();
|
||||
helper = helper.object(object_info).version_id(event_version_id);
|
||||
|
||||
let result = Ok(S3Response::new(PutObjectAclOutput::default()));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_object_legal_hold_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectLegalHoldInput>,
|
||||
) -> S3Result<S3Response<PutObjectLegalHoldOutput>> {
|
||||
let mut helper =
|
||||
OperationHelper::new(&req, EventName::ObjectCreatedPutLegalHold, S3Operation::PutObjectLegalHold).suppress_event();
|
||||
let PutObjectLegalHoldInput {
|
||||
bucket,
|
||||
key,
|
||||
legal_hold,
|
||||
version_id,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &key).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
if let Err(err) = validate_bucket_object_lock_enabled(&bucket).await {
|
||||
let _ = store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let mut popts = ObjectOptions {
|
||||
mod_time: opts.mod_time,
|
||||
version_id: opts.version_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut popts)?;
|
||||
|
||||
// PutObjectLegalHold only rewrites metadata, so replication is not scheduled by the
|
||||
// object PUT path. Schedule it explicitly, otherwise the legal hold never reaches the
|
||||
// replica and the peer copy remains deletable.
|
||||
//
|
||||
// Mirror the PUT path ordering (see object_usecase::put_object): compute the decision
|
||||
// once BEFORE the commit and persist the pending marker with it, so a restart between
|
||||
// the commit and the worker write-back leaves a Pending status on disk for the scanner
|
||||
// to re-drive. Scheduling without that marker would lose the task silently.
|
||||
let dsc = if crate::app::storage_api::bucket::metadata_sys::get_replication_config(&bucket)
|
||||
.await
|
||||
.is_ok()
|
||||
&& let Ok(info) = store.get_object_info(&bucket, &key, &opts).await
|
||||
{
|
||||
must_replicate_metadata(
|
||||
&bucket,
|
||||
&key,
|
||||
&info.user_defined,
|
||||
info.user_tags.as_ref().clone(),
|
||||
popts.delete_marker_replication_status(),
|
||||
popts.clone(),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
ReplicateDecision::new()
|
||||
};
|
||||
|
||||
let mut eval_metadata = parse_object_lock_legal_hold(legal_hold)?;
|
||||
if dsc.replicate_any() {
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
|
||||
}
|
||||
popts.eval_metadata = Some(eval_metadata);
|
||||
|
||||
let info = store.put_object_metadata(&bucket, &key, &popts).await.map_err(|e| {
|
||||
error!("put_object_metadata failed, {}", e.to_string());
|
||||
s3_error!(InternalError, "{}", e.to_string())
|
||||
})?;
|
||||
|
||||
// The current target transport carries the updated metadata in a full-object PUT.
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(info.clone(), store.clone(), dsc).await;
|
||||
}
|
||||
|
||||
let output = PutObjectLegalHoldOutput {
|
||||
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
};
|
||||
let version_id = req.input.version_id.clone().unwrap_or_default();
|
||||
helper = helper.object(info).version_id(version_id);
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata,
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_object_retention_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectRetentionInput>,
|
||||
) -> S3Result<S3Response<PutObjectRetentionOutput>> {
|
||||
let mut helper =
|
||||
OperationHelper::new(&req, EventName::ObjectCreatedPutRetention, S3Operation::PutObjectRetention).suppress_event();
|
||||
let PutObjectRetentionInput {
|
||||
bucket,
|
||||
key,
|
||||
retention,
|
||||
version_id,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &key).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
validate_bucket_object_lock_enabled(&bucket).await?;
|
||||
|
||||
let new_retain_until = retention
|
||||
.as_ref()
|
||||
.and_then(|r| r.retain_until_date.as_ref())
|
||||
.map(|d| OffsetDateTime::from(d.clone()));
|
||||
let new_mode = retention
|
||||
.as_ref()
|
||||
.and_then(|r| r.mode.as_ref())
|
||||
.map(|mode| mode.as_str().to_string());
|
||||
|
||||
let bypass_governance = has_bypass_governance_header(&req.headers);
|
||||
// Keep the early check for existing response behavior; put_object_metadata
|
||||
// repeats the same check after taking the metadata write lock.
|
||||
let check_opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let existing_obj_info = store.get_object_info(&bucket, &key, &check_opts).await.ok();
|
||||
|
||||
if let Some(existing_obj_info) = existing_obj_info.as_ref()
|
||||
&& let Some(block_reason) = check_retention_for_modification(
|
||||
&existing_obj_info.user_defined,
|
||||
new_mode.as_deref(),
|
||||
new_retain_until,
|
||||
bypass_governance,
|
||||
)
|
||||
{
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, block_reason.error_message()));
|
||||
}
|
||||
|
||||
let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
|
||||
opts.object_lock_retention = Some(ObjectLockRetentionOptions {
|
||||
mode: new_mode,
|
||||
retain_until: new_retain_until,
|
||||
bypass_governance,
|
||||
});
|
||||
|
||||
// PutObjectRetention only rewrites metadata, so the object PUT path that normally
|
||||
// computes a replication decision and schedules replication never runs for it. Without
|
||||
// scheduling here the peer keeps the previous, unprotected lock state and a
|
||||
// WORM-protected object stays deletable on the replica.
|
||||
//
|
||||
// Mirror the PUT path ordering (see object_usecase::put_object): compute the decision
|
||||
// once BEFORE the commit and persist the pending marker with it, so a restart between
|
||||
// the commit and the worker write-back leaves a Pending status on disk for the scanner
|
||||
// to re-drive. Scheduling without that marker would lose the task silently.
|
||||
let dsc = match existing_obj_info.as_ref() {
|
||||
Some(info) => {
|
||||
must_replicate_metadata(
|
||||
&bucket,
|
||||
&key,
|
||||
&info.user_defined,
|
||||
info.user_tags.as_ref().clone(),
|
||||
opts.delete_marker_replication_status(),
|
||||
opts.clone(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => ReplicateDecision::new(),
|
||||
};
|
||||
|
||||
let mut eval_metadata = parse_object_lock_retention(retention)?;
|
||||
if dsc.replicate_any() {
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
|
||||
}
|
||||
opts.eval_metadata = Some(eval_metadata);
|
||||
|
||||
let object_info = store.put_object_metadata(&bucket, &key, &opts).await.map_err(|e| {
|
||||
error!("put_object_metadata failed, {}", e.to_string());
|
||||
S3Error::from(ApiError::from(e))
|
||||
})?;
|
||||
|
||||
// The current target transport carries the updated metadata in a full-object PUT.
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(object_info.clone(), store.clone(), dsc).await;
|
||||
}
|
||||
|
||||
let output = PutObjectRetentionOutput {
|
||||
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
};
|
||||
|
||||
let version_id = req.input.version_id.clone().unwrap_or_else(|| Uuid::new_v4().to_string());
|
||||
helper = helper.object(object_info).version_id(version_id);
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata,
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(name = "put_object_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_object_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectTaggingInput>,
|
||||
) -> S3Result<S3Response<PutObjectTaggingOutput>> {
|
||||
let start_time = std::time::Instant::now();
|
||||
let mut helper = OperationHelper::new(&req, EventName::ObjectTaggingPut, S3Operation::PutObjectTagging);
|
||||
let PutObjectTaggingInput {
|
||||
bucket,
|
||||
key: object,
|
||||
tagging,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &object).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &object).await?;
|
||||
|
||||
crate::storage::s3_api::tagging::validate_object_tag_set(&tagging.tag_set)?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let tags = encode_tags(tagging.tag_set.clone());
|
||||
debug!("Encoded tags: {}", tags);
|
||||
|
||||
let version_id = req.input.version_id.clone();
|
||||
let mut opts = get_opts(&bucket, &object, version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let existing_object_info = match store.get_object_info(&bucket, &object, &opts).await {
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
// Replication lag window: apply the tagging update on a
|
||||
// replication target that already has the object
|
||||
// (backlog#1675 P1-5). No local object exists, so no bucket
|
||||
// notification event is emitted for the proxied write.
|
||||
if (is_err_object_not_found(&e) || is_err_version_not_found(&e))
|
||||
&& replication_proxy_allowed(&req.extensions)
|
||||
&& Self::proxy_put_object_tagging(&bucket, &object, version_id.clone(), &req.headers, &tagging.tag_set)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
counter!("rustfs_put_object_tagging_success").increment(1);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "put")
|
||||
.record(duration.as_secs_f64());
|
||||
return Ok(S3Response::new(PutObjectTaggingOutput {
|
||||
version_id: req.input.version_id.clone(),
|
||||
}));
|
||||
}
|
||||
return Err(ApiError::from(e).into());
|
||||
}
|
||||
};
|
||||
let dsc = must_replicate_metadata(
|
||||
&bucket,
|
||||
&object,
|
||||
&existing_object_info.user_defined,
|
||||
tags.clone(),
|
||||
existing_object_info.replication_status.clone(),
|
||||
opts.clone(),
|
||||
)
|
||||
.await;
|
||||
if dsc.replicate_any() {
|
||||
let mut eval_metadata = HashMap::new();
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
|
||||
insert_str(
|
||||
&mut eval_metadata,
|
||||
SUFFIX_TAGGING_TIMESTAMP,
|
||||
OffsetDateTime::now_utc().format(&Rfc3339).unwrap_or_default(),
|
||||
);
|
||||
opts.eval_metadata = Some(eval_metadata);
|
||||
}
|
||||
|
||||
let put_tags_result = store.put_object_tags(&bucket, &object, &tags, &opts).await;
|
||||
let object_info = put_tags_result.map_err(|e| {
|
||||
error!("Failed to put object tags: {}", e);
|
||||
counter!("rustfs_put_object_tagging_failure").increment(1);
|
||||
ApiError::from(e)
|
||||
})?;
|
||||
|
||||
let event_object_info = Some(object_info.clone());
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(object_info, store.clone(), dsc).await;
|
||||
}
|
||||
|
||||
counter!("rustfs_put_object_tagging_success").increment(1);
|
||||
|
||||
let event_version_id = req
|
||||
.input
|
||||
.version_id
|
||||
.as_deref()
|
||||
.filter(|version_id| !version_id.is_empty())
|
||||
.map(str::to_string)
|
||||
.or_else(|| {
|
||||
event_object_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.version_id.map(|version_id| version_id.to_string()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if let Some(event_object_info) = event_object_info {
|
||||
helper = helper.object(event_object_info);
|
||||
}
|
||||
helper = helper.version_id(event_version_id);
|
||||
|
||||
let result = Ok(S3Response::new(PutObjectTaggingOutput {
|
||||
version_id: req.input.version_id.clone(),
|
||||
}));
|
||||
let _ = helper.complete(&result);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&object,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata,
|
||||
);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "put").record(duration.as_secs_f64());
|
||||
result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of CopyObject, UploadPartCopy, CompleteMultipartUpload, DeleteObject,
|
||||
//! DeleteObjects, RestoreObject and SelectObjectContent: batch E of the gateway migration
|
||||
//! (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "complete_multipart_upload", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn complete_multipart_upload_edge(
|
||||
&self,
|
||||
req: S3Request<CompleteMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CompleteMultipartUploadOutput>> {
|
||||
crate::hp_guard!("S3::complete_multipart_upload");
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
Box::pin(usecase.execute_complete_multipart_upload(req)).await
|
||||
}
|
||||
|
||||
/// Copy an object from one location to another
|
||||
#[instrument(name = "copy_object", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn copy_object_edge(
|
||||
&self,
|
||||
req: S3Request<CopyObjectInput>,
|
||||
) -> S3Result<S3Response<CopyObjectOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_copy_object(req).await
|
||||
}
|
||||
|
||||
/// Delete an object
|
||||
#[instrument(name = "delete_object", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_object_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||
crate::hp_guard!("S3::delete_object");
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<crate::storage::request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.clone());
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
let result = Box::pin(usecase.execute_delete_object(req)).await;
|
||||
if let Err(error) = &result {
|
||||
log_object_internal_error("DeleteObject", request_id.as_deref(), error);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Delete multiple objects
|
||||
#[instrument(name = "delete_objects", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_objects_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectsInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectsOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_delete_objects(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn restore_object_edge(
|
||||
&self,
|
||||
req: S3Request<RestoreObjectInput>,
|
||||
) -> S3Result<S3Response<RestoreObjectOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_restore_object(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn select_object_content_edge(
|
||||
&self,
|
||||
req: S3Request<SelectObjectContentInput>,
|
||||
) -> S3Result<S3Response<SelectObjectContentOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_select_object_content(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "upload_part_copy", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn upload_part_copy_edge(
|
||||
&self,
|
||||
req: S3Request<UploadPartCopyInput>,
|
||||
) -> S3Result<S3Response<UploadPartCopyOutput>> {
|
||||
record_s3_op(S3Operation::UploadPartCopy);
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::UploadPartCopy, |req| Box::pin(usecase.execute_upload_part_copy(req))).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of GetObject, HeadObject and GetObjectAttributes: batch C of the gateway
|
||||
//! migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::s3_api;
|
||||
use tracing::instrument;
|
||||
|
||||
impl FS {
|
||||
/// Get bucket notification
|
||||
#[instrument(
|
||||
name = "get_object",
|
||||
level = "debug",
|
||||
skip(self, req),
|
||||
fields(start_time=?time::OffsetDateTime::now_utc())
|
||||
)]
|
||||
pub(in crate::storage::ecfs) async fn get_object_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectInput>,
|
||||
) -> S3Result<S3Response<GetObjectOutput>> {
|
||||
crate::hp_guard!("S3::get_object");
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<crate::storage::request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.clone());
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
let result = usecase.execute_get_object(req).await;
|
||||
if let Err(error) = &result {
|
||||
log_object_internal_error("GetObject", request_id.as_deref(), error);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_object_attributes_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectAttributesInput>,
|
||||
) -> S3Result<S3Response<GetObjectAttributesOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_get_object_attributes(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "head_object", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn head_object_edge(
|
||||
&self,
|
||||
req: S3Request<HeadObjectInput>,
|
||||
) -> S3Result<S3Response<HeadObjectOutput>> {
|
||||
crate::hp_guard!("S3::head_object");
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_head_object(req).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of PutObject and UploadPart (PostObject has no trait method of its own and
|
||||
//! reaches PutObject): batch D of the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "put_object", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_object_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectInput>,
|
||||
) -> S3Result<S3Response<PutObjectOutput>> {
|
||||
crate::hp_guard!("S3::put_object");
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_put_object(self, req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "upload_part", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn upload_part_edge(
|
||||
&self,
|
||||
req: S3Request<UploadPartInput>,
|
||||
) -> S3Result<S3Response<UploadPartOutput>> {
|
||||
crate::hp_guard!("S3::upload_part");
|
||||
record_s3_op(S3Operation::UploadPart);
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::UploadPart, |req| usecase.execute_upload_part(req)).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tests of the `bucket_access` legacy-edge handlers (batch A2), moved unchanged from `ecfs.rs` and
|
||||
//! `ecfs_test.rs`.
|
||||
|
||||
use super::{build_request, store_uninitialized_premise_holds};
|
||||
use crate::storage::ecfs::*;
|
||||
use http::Method;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_bucket_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetBucketAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_bucket_acl(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_bucket_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutBucketAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_bucket_acl(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tests of the `bucket_replication_lifecycle` legacy-edge handlers (batch A3), moved unchanged
|
||||
//! from `ecfs.rs` and `ecfs_test.rs`.
|
||||
|
||||
use super::super::bucket_replication_lifecycle::{
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED, SITE_REPLICATION_GATE_FORCE_ENABLED, SITE_REPLICATION_GATE_TEST_OVERRIDE,
|
||||
};
|
||||
use super::{build_request, store_uninitialized_premise_holds};
|
||||
use crate::storage::access::ReqInfo;
|
||||
use crate::storage::ecfs::*;
|
||||
use http::Method;
|
||||
use http::StatusCode;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
fn replication_config_edit_request<T>(input: T, is_owner: bool) -> S3Request<T> {
|
||||
let mut req = S3Request {
|
||||
input,
|
||||
method: Method::PUT,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
req.extensions.insert(ReqInfo {
|
||||
is_owner,
|
||||
..Default::default()
|
||||
});
|
||||
req
|
||||
}
|
||||
|
||||
fn put_bucket_replication_input() -> PutBucketReplicationInput {
|
||||
PutBucketReplicationInput {
|
||||
bucket: "test-bucket".to_string(),
|
||||
checksum_algorithm: None,
|
||||
content_md5: None,
|
||||
expected_bucket_owner: None,
|
||||
replication_configuration: ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: Vec::new(),
|
||||
},
|
||||
token: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn delete_bucket_replication_input() -> DeleteBucketReplicationInput {
|
||||
DeleteBucketReplicationInput {
|
||||
bucket: "test-bucket".to_string(),
|
||||
expected_bucket_owner: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_replication_deny_edit(err: &S3Error) {
|
||||
match err.code() {
|
||||
S3ErrorCode::Custom(code) => assert_eq!(code, "XMinioReplicationDenyEdit"),
|
||||
other => panic!("expected XMinioReplicationDenyEdit, got {other:?}"),
|
||||
}
|
||||
assert_eq!(err.status_code(), Some(StatusCode::BAD_REQUEST));
|
||||
}
|
||||
|
||||
/// Single test on purpose: the branches share the process-wide gate
|
||||
/// override, and parallel tests would race it.
|
||||
#[tokio::test]
|
||||
async fn replication_config_edit_gate_denies_only_non_owner_under_site_replication() {
|
||||
let fs = FS::new();
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_ENABLED, Ordering::SeqCst);
|
||||
|
||||
// Non-owner PUT/DELETE through the real S3 handlers: denied by the
|
||||
// gate before the usecase (and thus the store) is ever touched.
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner PutBucketReplication must be denied while site replication is enabled");
|
||||
assert_replication_deny_edit(&err);
|
||||
let err = fs
|
||||
.delete_bucket_replication(replication_config_edit_request(delete_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner DeleteBucketReplication must be denied while site replication is enabled");
|
||||
assert_replication_deny_edit(&err);
|
||||
|
||||
// Owner passes the gate (the usecase's empty-rules structure error
|
||||
// proves the request reached the usecase instead of the deny path).
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), true))
|
||||
.await
|
||||
.expect_err("owner request should pass the gate and fail later on config validation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
// Without site replication the policy check alone still governs the edit.
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_DISABLED, Ordering::SeqCst);
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner request should pass the gate and fail later on config validation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(0, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectLockConfigurationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.object_lock_configuration(Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs
|
||||
.put_object_lock_configuration(build_request(input, Method::PUT))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tests of the `listing_multipart` legacy-edge handlers (batch B2), moved unchanged from `ecfs.rs`
|
||||
//! and `ecfs_test.rs`.
|
||||
|
||||
use super::build_request;
|
||||
use crate::storage::StorageError;
|
||||
use crate::storage::ecfs::*;
|
||||
use http::{Method, StatusCode};
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_apis_report_storage_read_quorum_as_unavailable() {
|
||||
crate::app::gating_test_env::run_large_stack_test("list-quorum-surface", || async {
|
||||
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let bucket = format!("list-quorum-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create listing bucket");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let saved_disks = set.disks.read().await.clone();
|
||||
{
|
||||
let mut disks = set.disks.write().await;
|
||||
for disk in disks.iter_mut().skip(1) {
|
||||
*disk = None;
|
||||
}
|
||||
}
|
||||
let fs = FS::new();
|
||||
let errors = [
|
||||
S3::list_objects(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectsInput {
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("V1 cannot list below storage read quorum"),
|
||||
S3::list_objects_v2(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectsV2Input {
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("V2 cannot list below storage read quorum"),
|
||||
S3::list_object_versions(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectVersionsInput {
|
||||
bucket,
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("versions cannot list below storage read quorum"),
|
||||
];
|
||||
*set.disks.write().await = saved_disks;
|
||||
for error in errors {
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
|
||||
let source = error.source().expect("retain storage diagnostics");
|
||||
assert!(
|
||||
matches!(
|
||||
source.downcast_ref::<StorageError>(),
|
||||
Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
|
||||
),
|
||||
"the production LIST boundary must expose a typed read-quorum failure: {source}"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tests of the legacy-edge handlers, one module per migration batch, and the fixtures they
|
||||
//! share.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use http::{Extensions, HeaderMap, Method, Uri};
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
input,
|
||||
method,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Premise guard for the `*_returns_internal_error_when_store_uninitialized`
|
||||
/// tests (backlog#1830): they assert the error path taken while the global
|
||||
/// object layer is absent. Under nextest — the authoritative runner — every
|
||||
/// test owns its process, so the premise always holds and the assertion
|
||||
/// always runs. Under the documented shared-process `cargo test` fallback a
|
||||
/// sibling test may have initialized the store first; the premise is then
|
||||
/// unattainable, so the test skips instead of asserting against a scenario
|
||||
/// it does not describe.
|
||||
fn store_uninitialized_premise_holds() -> bool {
|
||||
if crate::runtime_sources::current_object_store_handle().is_some() {
|
||||
eprintln!("skipping store-uninitialized assertion: a sibling test already initialized the global object layer");
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
mod bucket_access;
|
||||
mod bucket_replication_lifecycle;
|
||||
mod listing_multipart;
|
||||
mod object_meta;
|
||||
@@ -0,0 +1,208 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tests of the `object_meta` legacy-edge handlers (batch B1), moved unchanged from `ecfs.rs` and
|
||||
//! `ecfs_test.rs`.
|
||||
|
||||
use super::{build_request, store_uninitialized_premise_holds};
|
||||
use crate::storage::ecfs::*;
|
||||
use http::Method;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_acl(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_legal_hold_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectLegalHoldInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_legal_hold(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_retention_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetObjectRetentionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_retention(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_legal_hold_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectLegalHoldInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_legal_hold(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_acl(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_retention_returns_internal_error_when_store_uninitialized() {
|
||||
let input = PutObjectRetentionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_retention(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_tagging(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_rejects_too_many_tags() {
|
||||
let tag_set = (0..11)
|
||||
.map(|index| Tag {
|
||||
key: Some(format!("k{index}")),
|
||||
value: Some(format!("v{index}")),
|
||||
})
|
||||
.collect();
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging { tag_set })
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Cannot have more than 10 tags per object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_rejects_empty_tag_key_before_store_lookup() {
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some(String::new()),
|
||||
value: Some("v1".to_string()),
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Tag key cannot be empty"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some("k".to_string()),
|
||||
value: Some("v".to_string()),
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delete_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = DeleteObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs
|
||||
.delete_object_tagging(build_request(input, Method::DELETE))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
@@ -14476,34 +14476,165 @@ fn object_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
let source = include_str!("../app/multipart_usecase.rs");
|
||||
const MULTIPART_KEY_GUARD: &str = "validate_table_catalog_object_mutation(&bucket, &key).await?;";
|
||||
const OBJECT_METADATA_OBJECT_GUARD: &str = "validate_table_catalog_object_mutation(&bucket, &object).await?;";
|
||||
const OBJECT_METADATA_KEY_GUARD: &str = MULTIPART_KEY_GUARD;
|
||||
|
||||
fn assert_multipart_reserved_prefix_guard_calls(source: &crate::guard_sources::SourceSet) {
|
||||
assert_eq!(
|
||||
source
|
||||
.matches("validate_table_catalog_object_mutation(&bucket, &key).await?;")
|
||||
.count(),
|
||||
4
|
||||
source.count(MULTIPART_KEY_GUARD),
|
||||
4,
|
||||
"reserved-prefix guard calls across {}",
|
||||
source.describe()
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_object_metadata_reserved_prefix_guard_calls(source: &crate::guard_sources::SourceSet) {
|
||||
assert_eq!(
|
||||
source.count(OBJECT_METADATA_OBJECT_GUARD),
|
||||
2,
|
||||
"reserved-prefix guard calls across {}",
|
||||
source.describe()
|
||||
);
|
||||
assert_eq!(
|
||||
source.count(OBJECT_METADATA_KEY_GUARD),
|
||||
2,
|
||||
"reserved-prefix guard calls across {}",
|
||||
source.describe()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_metadata_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
let source = include_str!("../storage/ecfs.rs");
|
||||
fn multipart_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
assert_multipart_reserved_prefix_guard_calls(&crate::guard_sources::SourceSet::production(
|
||||
"src/app/multipart_usecase.rs",
|
||||
"src/app/multipart_usecase",
|
||||
));
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
source
|
||||
.matches("validate_table_catalog_object_mutation(&bucket, &object).await?;")
|
||||
.count(),
|
||||
2
|
||||
#[test]
|
||||
fn object_metadata_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
assert_object_metadata_reserved_prefix_guard_calls(&crate::guard_sources::SourceSet::production(
|
||||
"src/storage/ecfs.rs",
|
||||
"src/storage/legacy_edge",
|
||||
));
|
||||
}
|
||||
|
||||
/// `count` production calls of `guard`, one per method, for count-guard fixtures.
|
||||
fn guarded_methods(guard: &str, count: usize) -> String {
|
||||
let methods: String = (0..count)
|
||||
.map(|index| format!(" async fn guarded_{index}(&self) {{\n {guard}\n }}\n"))
|
||||
.collect();
|
||||
format!("impl Usecase {{\n{methods}}}\n")
|
||||
}
|
||||
|
||||
fn guard_fixture(files: &[(&str, &str)], file: &str, split_dir: &str) -> crate::guard_sources::SourceSet {
|
||||
let root = crate::guard_sources::fixture_root(files);
|
||||
crate::guard_sources::SourceSet::production_under(root.path(), file, split_dir)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_count_guard_fails_when_a_dropped_production_call_is_replaced_by_test_mentions() {
|
||||
let parent = format!(
|
||||
"{}\n#[cfg(test)]\nmod tests {{\n fn mentions() {{\n {MULTIPART_KEY_GUARD}\n }}\n}}\n",
|
||||
guarded_methods(MULTIPART_KEY_GUARD, 2)
|
||||
);
|
||||
let source = guard_fixture(
|
||||
&[
|
||||
("src/app/multipart_usecase.rs", parent.as_str()),
|
||||
("src/app/multipart_usecase/complete.rs", guarded_methods(MULTIPART_KEY_GUARD, 1).as_str()),
|
||||
],
|
||||
"src/app/multipart_usecase.rs",
|
||||
"src/app/multipart_usecase",
|
||||
);
|
||||
assert_eq!(
|
||||
source
|
||||
.matches("validate_table_catalog_object_mutation(&bucket, &key).await?;")
|
||||
.count(),
|
||||
2
|
||||
source.count(MULTIPART_KEY_GUARD),
|
||||
3,
|
||||
"fixture: three production calls and one test mention"
|
||||
);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_multipart_reserved_prefix_guard_calls(&source));
|
||||
|
||||
assert!(message.contains("reserved-prefix guard calls"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_count_guard_passes_when_calls_move_into_the_split_directory() {
|
||||
let source = guard_fixture(
|
||||
&[
|
||||
("src/app/multipart_usecase.rs", guarded_methods(MULTIPART_KEY_GUARD, 1).as_str()),
|
||||
("src/app/multipart_usecase/complete.rs", guarded_methods(MULTIPART_KEY_GUARD, 3).as_str()),
|
||||
],
|
||||
"src/app/multipart_usecase.rs",
|
||||
"src/app/multipart_usecase",
|
||||
);
|
||||
|
||||
assert_multipart_reserved_prefix_guard_calls(&source);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_metadata_key_count_guard_fails_when_a_moved_call_is_broken() {
|
||||
let moved_and_broken = guarded_methods(MULTIPART_KEY_GUARD, 1).replace("&key)", "key.as_str())");
|
||||
let parent = format!(
|
||||
"{}{}",
|
||||
guarded_methods(OBJECT_METADATA_OBJECT_GUARD, 2),
|
||||
guarded_methods(OBJECT_METADATA_KEY_GUARD, 1)
|
||||
);
|
||||
let source = guard_fixture(
|
||||
&[
|
||||
("src/storage/ecfs.rs", parent.as_str()),
|
||||
("src/storage/legacy_edge/object_write.rs", moved_and_broken.as_str()),
|
||||
],
|
||||
"src/storage/ecfs.rs",
|
||||
"src/storage/legacy_edge",
|
||||
);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_object_metadata_reserved_prefix_guard_calls(&source));
|
||||
|
||||
assert!(message.contains("src/storage/legacy_edge/object_write.rs"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_metadata_key_count_guard_passes_when_a_call_moves_intact() {
|
||||
let parent = format!(
|
||||
"{}{}",
|
||||
guarded_methods(OBJECT_METADATA_OBJECT_GUARD, 2),
|
||||
guarded_methods(OBJECT_METADATA_KEY_GUARD, 1)
|
||||
);
|
||||
let source = guard_fixture(
|
||||
&[
|
||||
("src/storage/ecfs.rs", parent.as_str()),
|
||||
(
|
||||
"src/storage/legacy_edge/object_write.rs",
|
||||
guarded_methods(OBJECT_METADATA_KEY_GUARD, 1).as_str(),
|
||||
),
|
||||
],
|
||||
"src/storage/ecfs.rs",
|
||||
"src/storage/legacy_edge",
|
||||
);
|
||||
|
||||
assert_object_metadata_reserved_prefix_guard_calls(&source);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_metadata_count_guard_fails_when_a_legacy_edge_test_replaces_a_production_call() {
|
||||
let parent = format!(
|
||||
"{}{}",
|
||||
guarded_methods(OBJECT_METADATA_OBJECT_GUARD, 1),
|
||||
guarded_methods(OBJECT_METADATA_KEY_GUARD, 2)
|
||||
);
|
||||
let source = guard_fixture(
|
||||
&[
|
||||
("src/storage/ecfs.rs", parent.as_str()),
|
||||
("src/storage/legacy_edge/tests/object_meta.rs", OBJECT_METADATA_OBJECT_GUARD),
|
||||
],
|
||||
"src/storage/ecfs.rs",
|
||||
"src/storage/legacy_edge",
|
||||
);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_object_metadata_reserved_prefix_guard_calls(&source));
|
||||
|
||||
assert!(message.contains("reserved-prefix guard calls"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -38,6 +38,7 @@ their issue closes.
|
||||
| `check_no_tokio_io_uring.sh` | ci-gate | Keeps tokio's io-uring backend disabled | ci.yml Quick Checks |
|
||||
| `check_s3s_footprint.sh` | ci-gate | Exact, lower-only baseline on the direct s3s surface ahead of the gateway migration (reads `.config/s3s-footprint-baseline.txt`; `s3_error_lines` leaves out lines whose file imports the macro from `rustfs-s3-types`, rule in the script header); `--mode allowlist` fails on any Rust file referencing s3s outside `.config/s3s-edge-allowlist.txt`, `--dry-run` lists them | ci.yml Quick Checks; `make pre-commit`; `make script-tests` (`--self-test`) |
|
||||
| `check_test_count_ratchet.sh` | ci-gate | Per-crate test-count floor and `#[ignore]` ceiling against `.config/test-count-baseline.txt`; no crate may fall below its tests row or rise above its ignored row (extra tests and new crates pass with a note) | ci.yml Quick Checks; `make pre-commit` |
|
||||
| `guard_source_set.sh` | ci-gate (library) | Sourced helpers for text guards that must survive a mechanical file split (rustfs/backlog#2765): `guard_source_set` resolves a required file plus the split directory (a missing required file fails), `module_relative_rg` rewrites a `super::` rule for each file's module depth | Sourced by `check_architecture_migration_rules.sh` and `check_logging_guardrails.sh`; contract tests `test_guard_source_set.sh`; end-to-end split probe `test_split_guard_rules.sh` (both `make script-tests`) |
|
||||
| `check_unsafe_code_allowances.sh` | ci-gate | Unsafe-code allowance ledger guard | ci.yml Quick Checks |
|
||||
| `layer-dependency-baseline.txt` | ci-gate (data) | Committed baseline consumed by `check_layer_dependencies.sh` | [Architecture guard troubleshooting](../docs/operations/architecture-guard-troubleshooting.md) |
|
||||
| `static.sh` | ci-gate | Static-build helper executed inside image builds | `Dockerfile.source`, `Dockerfile.decommission-local` |
|
||||
@@ -57,6 +58,9 @@ their issue closes.
|
||||
| `test_object_generation_protocol_model.py` | dev-tool | Bounded single-slot, four-voter promise/accept design model and recovery/retirement examples; no production runtime evidence | `python3 scripts/test_object_generation_protocol_model.py`; [Generation contract](../docs/architecture/unified-object-generation.md#executable-design-model-and-acceptance-mapping) |
|
||||
| `junit_diff.py` | dev-tool | Case-by-case pass/fail/skip diff of two JUnit XML reports (reference vs candidate stack); exit 1 on any difference or stale `--allow` entry, 2 on unreadable input | `python3 scripts/junit_diff.py a.xml b.xml`; backlog T3.1 |
|
||||
| `test_junit_diff.py` | ci-gate | Contract tests and fixtures (`scripts/tests/junit_diff/`) for `junit_diff.py` | `make script-tests` |
|
||||
| `s3-tests/reference_parity.py` | ci-gate | Plans the reference parity targets, lists a run's failures for the single rerun, and compares every candidate's s3-tests JUnit against the pinned reference per node id with a tolerated-difference file | `.github/workflows/reference-parity.yml`; backlog T3.2 |
|
||||
| `s3-tests/test_reference_parity.py` | ci-gate | Contract tests for `reference_parity.py` | `make script-tests` |
|
||||
| `test_reference_binary.py` | ci-gate | Contract tests for `.github/actions/reference-binary/reference_binary.py` (pin resolution and verified unpacking) | `make script-tests` |
|
||||
| `e2e-run.sh` | ci-gate | Boots a rustfs server and runs the `s3s-e2e` black-box conformance tool against it | ci.yml `e2e-tests` jobs; `docs/testing/README.md` |
|
||||
| `run_ecstore_validation_suite.sh` | dev-tool | ecstore black-box validation suite (`quick`/`full`/`destructive`/`fuzz` profiles) | `docs/testing/README.md`, `docs/testing/ecstore-validation-suite-design.md` |
|
||||
| `run_e2e_tests.sh` | dev-tool | Local `e2e_test` crate runner (starts a server, applies filters, cleans up) | `crates/e2e_test/README.md` |
|
||||
|
||||
@@ -118,6 +118,45 @@ require_source_contains "docs/architecture/ecstore-api-facade-inventory.md" "Do
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
# Rules about one of the large files below read that file plus the sibling
|
||||
# directory a mechanical split moves its code into (rustfs/backlog#2765), so a
|
||||
# split neither breaks nor silently blinds them. The *_SPLIT_SOURCES arrays hold
|
||||
# only the directory part, for rules that select the file itself by glob.
|
||||
# shellcheck source=scripts/guard_source_set.sh
|
||||
source "${ROOT_DIR}/scripts/guard_source_set.sh"
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/storage/ecfs.rs rustfs/src/storage/legacy_edge ||
|
||||
report_failure "ECFS guard source set is unreadable"
|
||||
ECFS_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/app/bucket_usecase.rs rustfs/src/app/bucket_usecase ||
|
||||
report_failure "bucket usecase guard source set is unreadable"
|
||||
BUCKET_USECASE_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
BUCKET_USECASE_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/app/multipart_usecase.rs rustfs/src/app/multipart_usecase ||
|
||||
report_failure "multipart usecase guard source set is unreadable"
|
||||
MULTIPART_USECASE_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
MULTIPART_USECASE_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/lifecycle_transition_api_test ||
|
||||
report_failure "lifecycle transition API test guard source set is unreadable"
|
||||
LIFECYCLE_TRANSITION_TEST_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/storage/ecfs_test.rs rustfs/src/storage/legacy_edge/tests ||
|
||||
report_failure "ECFS test guard source set is unreadable"
|
||||
ECFS_TEST_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
# Every .rs file under rustfs/src/app, for depth-aware rules over the whole app tree.
|
||||
APP_RUNTIME_RULE_SOURCES=()
|
||||
while IFS= read -r app_source; do
|
||||
APP_RUNTIME_RULE_SOURCES+=("$app_source")
|
||||
done < <(cd "$ROOT_DIR" && rg --files rustfs/src/app --glob '*.rs' --glob '!rustfs/src/app/storage_api.rs' | LC_ALL=C sort)
|
||||
APP_OBJECT_SOURCES=()
|
||||
while IFS= read -r app_source; do
|
||||
APP_OBJECT_SOURCES+=("$app_source")
|
||||
done < <(cd "$ROOT_DIR" && rg --files rustfs/src/app/object --glob '*.rs' | LC_ALL=C sort)
|
||||
APP_USECASE_RULE_SOURCES=("${BUCKET_USECASE_GUARD_SOURCES[@]}" rustfs/src/app/object_usecase.rs "${APP_OBJECT_SOURCES[@]}"
|
||||
"${MULTIPART_USECASE_GUARD_SOURCES[@]}" "${LIFECYCLE_TRANSITION_TEST_GUARD_SOURCES[@]}")
|
||||
if [[ ${#APP_RUNTIME_RULE_SOURCES[@]} -eq 0 || ${#APP_OBJECT_SOURCES[@]} -eq 0 ]]; then
|
||||
report_failure "app source listing for depth-aware import rules is empty"
|
||||
fi
|
||||
|
||||
PR_TYPES_FILE="${TMP_DIR}/pr_types.txt"
|
||||
PR_TYPE_HITS_FILE="${TMP_DIR}/pr_type_hits.txt"
|
||||
SOURCE_MARKERS_FILE="${TMP_DIR}/source_markers.txt"
|
||||
@@ -2157,7 +2196,7 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::app::context::(?:\{[^}]*\b(?:resolve_object_store_handle|resolve_buffer_config|resolve_internode_metrics|resolve_local_node_name|resolve_action_credentials|resolve_notify_interface|resolve_performance_metrics|resolve_encryption_service|resolve_region|resolve_ready_iam_handle)\b|(?:resolve_object_store_handle|resolve_buffer_config|resolve_internode_metrics|resolve_local_node_name|resolve_action_credentials|resolve_notify_interface|resolve_performance_metrics|resolve_encryption_service|resolve_region|resolve_ready_iam_handle)\b)' \
|
||||
rustfs/src/storage/access.rs \
|
||||
rustfs/src/storage/ecfs.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
rustfs/src/storage/ecfs_extend.rs \
|
||||
rustfs/src/storage/helper.rs \
|
||||
rustfs/src/storage/concurrency/manager.rs \
|
||||
@@ -2175,7 +2214,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::app::(?:bucket_usecase|multipart_usecase|object_usecase|object\b)|Default(?:Bucket|Multipart|Object)Usecase::from_global\(\)' \
|
||||
rustfs/src/storage/ecfs.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
--glob '*.rs' || true
|
||||
) >"$RUSTFS_STORAGE_ECFS_USECASE_BYPASS_HITS_FILE"
|
||||
|
||||
@@ -2228,7 +2267,10 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'use crate::storage::\*;' rustfs/src/app --glob '*_usecase.rs' --glob 'object/*.rs' || true
|
||||
# A glob containing a slash is anchored at the search root, so object files
|
||||
# need the leading **/ to match under rustfs/src/app.
|
||||
rg -n --with-filename 'use crate::storage::\*;' rustfs/src/app --glob '*_usecase.rs' --glob '**/object/*.rs' \
|
||||
${BUCKET_USECASE_SPLIT_SOURCES[@]+"${BUCKET_USECASE_SPLIT_SOURCES[@]}"} ${MULTIPART_USECASE_SPLIT_SOURCES[@]+"${MULTIPART_USECASE_SPLIT_SOURCES[@]}"} || true
|
||||
) >"$RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE" ]]; then
|
||||
@@ -2246,7 +2288,8 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::storage::s3_api::|use crate::storage::s3_api|super::s3_api::|use super::s3_api' rustfs/src/app --glob '*_usecase.rs' --glob 'object/*.rs' || true
|
||||
rg -n --with-filename 'crate::storage::s3_api::|use crate::storage::s3_api|super::s3_api::|use super::s3_api' rustfs/src/app --glob '*_usecase.rs' --glob '**/object/*.rs' \
|
||||
${BUCKET_USECASE_SPLIT_SOURCES[@]+"${BUCKET_USECASE_SPLIT_SOURCES[@]}"} ${MULTIPART_USECASE_SPLIT_SOURCES[@]+"${MULTIPART_USECASE_SPLIT_SOURCES[@]}"} || true
|
||||
) >"$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE" ]]; then
|
||||
@@ -2258,13 +2301,19 @@ fi
|
||||
{
|
||||
rg -n --with-filename \
|
||||
'(use crate::storage::(access|helper|options|request_context|sse|timeout_wrapper|head_prefix|concurrency|ecfs)|crate::storage::sse::EncryptionKeyKind|use crate::storage::\{|use crate::storage::[A-Z])' \
|
||||
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs rustfs/src/app/object || true
|
||||
rg -n --with-filename \
|
||||
'use super::(?:\{[^}]*\b(?:DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|StorageError|DiskError|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b|(?:object_api_utils::to_s3s_etag|storageclass|StorageError|DiskError|DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b)' \
|
||||
rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs || true
|
||||
rg -n --with-filename \
|
||||
'use super::(?:\{[^}]*\b(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b|(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b)|super::(?:lifecycle|metadata_sys|object_lock|quota|replication|tagging|target|utils|versioning_sys|transition_api)::|super::super::(?:metadata_sys|lifecycle|target)::' \
|
||||
rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/capacity_dirty_scope_test.rs rustfs/src/app/context.rs rustfs/src/app/context/handles.rs rustfs/src/app/context/interfaces.rs rustfs/src/app/context/runtime_sources.rs || true
|
||||
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs rustfs/src/app/object \
|
||||
${BUCKET_USECASE_SPLIT_SOURCES[@]+"${BUCKET_USECASE_SPLIT_SOURCES[@]}"} ${MULTIPART_USECASE_SPLIT_SOURCES[@]+"${MULTIPART_USECASE_SPLIT_SOURCES[@]}"} || true
|
||||
# The super:: chain that names crate::app depends on each file's depth, so
|
||||
# these rules are rewritten per depth (scripts/guard_source_set.sh).
|
||||
module_relative_rg rustfs/src/app \
|
||||
'(?:use @UP@|use crate::app::)(?:\{[^}]*\b(?:DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|StorageError|DiskError|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b|(?:object_api_utils::to_s3s_etag|storageclass|StorageError|DiskError|DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b)' \
|
||||
-n --with-filename -- "${APP_USECASE_RULE_SOURCES[@]}" ||
|
||||
echo "storage helper import rule could not search its sources"
|
||||
module_relative_rg rustfs/src/app \
|
||||
'(?:use @UP@|use crate::app::)(?:\{[^}]*\b(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b|(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b)|(?:@UP@|crate::app::)(?:lifecycle|metadata_sys|object_lock|quota|replication|tagging|target|utils|versioning_sys|transition_api)::|@UP@super::(?:metadata_sys|lifecycle|target)::' \
|
||||
-n --with-filename -- "${APP_USECASE_RULE_SOURCES[@]}" rustfs/src/app/capacity_dirty_scope_test.rs \
|
||||
rustfs/src/app/context.rs rustfs/src/app/context/handles.rs rustfs/src/app/context/interfaces.rs rustfs/src/app/context/runtime_sources.rs ||
|
||||
echo "storage owner import rule could not search its sources"
|
||||
}
|
||||
) >"$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE"
|
||||
|
||||
@@ -2277,6 +2326,7 @@ fi
|
||||
rg -n --with-filename \
|
||||
'(use crate::storage::(access|request_context|ecfs)|use crate::storage::\{[^}]*Storage(ObjectInfo|ObjectOptions|PutObjReader)|crate::storage::StorageObjectOptions)' \
|
||||
rustfs/src/app/*_test.rs \
|
||||
${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]+"${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]}"} \
|
||||
rustfs/src/admin/router.rs \
|
||||
rustfs/src/admin/console.rs \
|
||||
rustfs/src/admin/handlers/heal.rs \
|
||||
@@ -2291,16 +2341,14 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
{
|
||||
rg -n --with-filename \
|
||||
'use super::(?:\{[^}]*\b(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|AppWarmBackend|WarmBackendGetOpts|TierConfig|TierType|PoolStatus|PoolDecommissionInfo|RebalStatus)\b|(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|AppWarmBackend|WarmBackendGetOpts|TierConfig|TierType|PoolStatus|PoolDecommissionInfo|RebalStatus)\b)' \
|
||||
rustfs/src/app \
|
||||
--glob '*.rs' \
|
||||
--glob '!rustfs/src/app/storage_api.rs' || true
|
||||
rg -n --with-filename \
|
||||
'super::(?:get_server_info|get_total_usable_capacity|get_total_usable_capacity_free|apply_bucket_usage_memory_overlay|load_data_usage_from_backend|record_bucket_object_delete_memory|record_bucket_object_write_memory|remove_bucket_usage_from_backend|get_global_|global_rustfs_port|new_object_layer_fn|set_object_store_resolver|set_global_storage_class|collect_scanner_metrics_report|init_local_disks)|super::super::(?:get_server_info|get_total_usable_capacity|get_total_usable_capacity_free|apply_bucket_usage_memory_overlay|load_data_usage_from_backend|record_bucket_object_delete_memory|record_bucket_object_write_memory|remove_bucket_usage_from_backend|get_global_|global_rustfs_port|new_object_layer_fn|set_object_store_resolver|set_global_storage_class|collect_scanner_metrics_report|init_local_disks)' \
|
||||
rustfs/src/app \
|
||||
--glob '*.rs' \
|
||||
--glob '!rustfs/src/app/storage_api.rs' || true
|
||||
module_relative_rg rustfs/src/app \
|
||||
'use @UP@(?:\{[^}]*\b(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|AppWarmBackend|WarmBackendGetOpts|TierConfig|TierType|PoolStatus|PoolDecommissionInfo|RebalStatus)\b|(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|AppWarmBackend|WarmBackendGetOpts|TierConfig|TierType|PoolStatus|PoolDecommissionInfo|RebalStatus)\b)' \
|
||||
-n --with-filename -- "${APP_RUNTIME_RULE_SOURCES[@]}" ||
|
||||
echo "runtime facade import rule could not search its sources"
|
||||
module_relative_rg rustfs/src/app \
|
||||
'@UP@(?:get_server_info|get_total_usable_capacity|get_total_usable_capacity_free|apply_bucket_usage_memory_overlay|load_data_usage_from_backend|record_bucket_object_delete_memory|record_bucket_object_write_memory|remove_bucket_usage_from_backend|get_global_|global_rustfs_port|new_object_layer_fn|set_object_store_resolver|set_global_storage_class|collect_scanner_metrics_report|init_local_disks)|@UP@super::(?:get_server_info|get_total_usable_capacity|get_total_usable_capacity_free|apply_bucket_usage_memory_overlay|load_data_usage_from_backend|record_bucket_object_delete_memory|record_bucket_object_write_memory|remove_bucket_usage_from_backend|get_global_|global_rustfs_port|new_object_layer_fn|set_object_store_resolver|set_global_storage_class|collect_scanner_metrics_report|init_local_disks)' \
|
||||
-n --with-filename -- "${APP_RUNTIME_RULE_SOURCES[@]}" ||
|
||||
echo "runtime facade path rule could not search its sources"
|
||||
rg -n --with-filename \
|
||||
'crate::app::(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|get_global_|set_object_store_resolver)' \
|
||||
rustfs/src --glob '*.rs' || true
|
||||
@@ -2524,9 +2572,22 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n -U --with-filename \
|
||||
'use\s+(?:super::super|crate::storage)::\{[^}]*?(?:BucketMetadata|DEFAULT_READ_BUFFER_SIZE|bucket_metadata_sys_initialized|get_global_bucket_metadata_sys|set_bucket_metadata|apply_cors_headers|apply_default_lock_retention_metadata|check_preconditions|decode_tags_to_map|get_adaptive_buffer_size_with_profile|get_buffer_size_opt_in|is_etag_equal|matches_origin_pattern|parse_etag|parse_object_lock_legal_hold|parse_object_lock_retention|process_lambda_configurations|process_queue_configurations|process_topic_configurations|remove_object_lock_metadata_for_copy|remove_object_lock_retention_metadata|validate_bucket_object_lock_enabled|validate_list_object_unordered_with_delimiter)' \
|
||||
rustfs/src/storage/ecfs_test.rs || true
|
||||
# ecfs_test.rs holds its tests in an inline `mod tests`, so the storage root
|
||||
# is one super:: past the depth of the file itself. legacy_edge/tests/ is
|
||||
# where a split of ecfs.rs moves them, with or without that inline module,
|
||||
# so both depths are forbidden there.
|
||||
{
|
||||
module_relative_rg rustfs/src/storage \
|
||||
'use\s+(?:@UP@super::|crate::storage::)\{[^}]*?(?:BucketMetadata|DEFAULT_READ_BUFFER_SIZE|bucket_metadata_sys_initialized|get_global_bucket_metadata_sys|set_bucket_metadata|apply_cors_headers|apply_default_lock_retention_metadata|check_preconditions|decode_tags_to_map|get_adaptive_buffer_size_with_profile|get_buffer_size_opt_in|is_etag_equal|matches_origin_pattern|parse_etag|parse_object_lock_legal_hold|parse_object_lock_retention|process_lambda_configurations|process_queue_configurations|process_topic_configurations|remove_object_lock_metadata_for_copy|remove_object_lock_retention_metadata|validate_bucket_object_lock_enabled|validate_list_object_unordered_with_delimiter)' \
|
||||
-n -U --with-filename -- rustfs/src/storage/ecfs_test.rs ||
|
||||
echo "storage-owner test import rule could not search rustfs/src/storage/ecfs_test.rs"
|
||||
if [[ ${#ECFS_TEST_GUARD_SOURCES[@]} -gt 1 ]]; then
|
||||
module_relative_rg rustfs/src/storage \
|
||||
'use\s+(?:@UP@(?:super::)?|crate::storage::)\{[^}]*?(?:BucketMetadata|DEFAULT_READ_BUFFER_SIZE|bucket_metadata_sys_initialized|get_global_bucket_metadata_sys|set_bucket_metadata|apply_cors_headers|apply_default_lock_retention_metadata|check_preconditions|decode_tags_to_map|get_adaptive_buffer_size_with_profile|get_buffer_size_opt_in|is_etag_equal|matches_origin_pattern|parse_etag|parse_object_lock_legal_hold|parse_object_lock_retention|process_lambda_configurations|process_queue_configurations|process_topic_configurations|remove_object_lock_metadata_for_copy|remove_object_lock_retention_metadata|validate_bucket_object_lock_enabled|validate_list_object_unordered_with_delimiter)' \
|
||||
-n -U --with-filename -- "${ECFS_TEST_GUARD_SOURCES[@]:1}" ||
|
||||
echo "storage-owner test import rule could not search rustfs/src/storage/legacy_edge/tests"
|
||||
fi
|
||||
}
|
||||
) >"$RUSTFS_STORAGE_OWNER_TEST_ROOT_CONSUMER_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_STORAGE_OWNER_TEST_ROOT_CONSUMER_HITS_FILE" ]]; then
|
||||
@@ -4561,7 +4622,7 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::storage::core_storage_compat' \
|
||||
rustfs/src/storage/mod.rs \
|
||||
rustfs/src/storage/ecfs.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
rustfs/src/storage/ecfs_extend.rs \
|
||||
rustfs/src/storage/access.rs \
|
||||
rustfs/src/storage/head_prefix.rs \
|
||||
@@ -4577,11 +4638,12 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
{
|
||||
rg -n --with-filename 'crate::app::usecase_storage_compat' \
|
||||
rustfs/src/app/*.rs || true
|
||||
rustfs/src/app/*.rs ${BUCKET_USECASE_SPLIT_SOURCES[@]+"${BUCKET_USECASE_SPLIT_SOURCES[@]}"} ${MULTIPART_USECASE_SPLIT_SOURCES[@]+"${MULTIPART_USECASE_SPLIT_SOURCES[@]}"} \
|
||||
${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]+"${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]}"} || true
|
||||
rg -n --with-filename 'crate::admin::router_storage_compat' \
|
||||
rustfs/src/admin/router.rs || true
|
||||
rg -n --with-filename 'crate::storage::core_storage_compat' \
|
||||
rustfs/src/storage/ecfs_test.rs || true
|
||||
"${ECFS_TEST_GUARD_SOURCES[@]}" || true
|
||||
}
|
||||
) >"$RUSTFS_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE"
|
||||
|
||||
|
||||
@@ -22,7 +22,8 @@ classify_source_layer() {
|
||||
printf 'composition'
|
||||
elif [[ "$file" == rustfs/src/app/* ]]; then
|
||||
printf 'app'
|
||||
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
|
||||
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] ||
|
||||
[[ "$file" == rustfs/src/storage/legacy_edge/* ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
|
||||
[[ "$file" == rustfs/src/storage/s3_compat/* ]]; then
|
||||
printf 'interface'
|
||||
elif [[ "$file" == rustfs/src/* ]]; then
|
||||
@@ -55,6 +56,7 @@ classify_target_layer() {
|
||||
storage)
|
||||
storage_path="${path#storage::}"
|
||||
if [[ "$storage_path" == "ecfs" ]] || [[ "$storage_path" == ecfs::* ]] ||
|
||||
[[ "$storage_path" == "legacy_edge" ]] || [[ "$storage_path" == legacy_edge::* ]] ||
|
||||
[[ "$storage_path" == "s3_api" ]] || [[ "$storage_path" == s3_api::* ]] ||
|
||||
[[ "$storage_path" == "s3_compat" ]] || [[ "$storage_path" == s3_compat::* ]]; then
|
||||
printf 'interface'
|
||||
@@ -137,6 +139,60 @@ run_layer_model_self_tests() {
|
||||
assert_dependency_direction 'allowed' "$s3_compat_source" "$app_target"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$s3_compat_target"
|
||||
assert_dependency_direction 'reverse' "$storage_source" "$s3_compat_target"
|
||||
|
||||
# A mechanical split moves ecfs.rs code into storage/legacy_edge/ and the
|
||||
# usecase files' code into same-named directories; each keeps its layer.
|
||||
assert_source_layer 'interface' rustfs/src/storage/ecfs.rs
|
||||
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/bucket_core.rs
|
||||
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/tests/object_meta.rs
|
||||
assert_source_layer 'infra' rustfs/src/storage/legacy_edge.rs
|
||||
assert_source_layer 'infra' rustfs/src/storage/legacy_edge_helper.rs
|
||||
assert_source_layer 'app' rustfs/src/app/bucket_usecase/cors.rs
|
||||
assert_source_layer 'app' rustfs/src/app/multipart_usecase/complete.rs
|
||||
assert_target_layer 'interface' storage::legacy_edge
|
||||
assert_target_layer 'interface' storage::legacy_edge::object_meta
|
||||
assert_target_layer 'infra' storage::legacy_edge_helper
|
||||
assert_dependency_direction 'allowed' "$(classify_source_layer rustfs/src/storage/legacy_edge/bucket_core.rs)" "$app_target"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$(classify_target_layer storage::legacy_edge::bucket_core)"
|
||||
assert_dependency_direction 'reverse' "$storage_source" "$(classify_target_layer storage::legacy_edge)"
|
||||
assert_dedicated_test_file 'yes' rustfs/src/app/lifecycle_transition_api_test.rs
|
||||
assert_dedicated_test_file 'yes' rustfs/src/app/lifecycle_transition_api_test/put.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/app/lifecycle_transition_api_test_support.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/app/lifecycle_transition_api.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/app/bucket_usecase/cors.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/storage/legacy_edge/bucket_core.rs
|
||||
}
|
||||
|
||||
assert_source_layer() {
|
||||
local expected="$1" file="$2" actual
|
||||
|
||||
actual="$(classify_source_layer "$file")"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
printf 'Layer dependency guard self-test failed: %s classified as %s (expected %s)\n' "$file" "$actual" "$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_target_layer() {
|
||||
local expected="$1" path="$2" actual
|
||||
|
||||
actual="$(classify_target_layer "$path")"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
printf 'Layer dependency guard self-test failed: crate::%s classified as %s (expected %s)\n' "$path" "$actual" "$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_dedicated_test_file() {
|
||||
local expected="$1" file="$2" actual='no'
|
||||
|
||||
if is_dedicated_test_file "$file"; then
|
||||
actual='yes'
|
||||
fi
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
printf 'Layer dependency guard self-test failed: %s dedicated test file = %s (expected %s)\n' "$file" "$actual" "$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
normalize_import_group_item() {
|
||||
@@ -251,11 +307,20 @@ normalize_import_path() {
|
||||
printf '%s\n' "$path"
|
||||
}
|
||||
|
||||
# The guard is file-scoped: dedicated test modules are excluded, while inline
|
||||
# #[cfg(test)] imports remain subject to the source file's layer. The
|
||||
# lifecycle_transition_api_test/ directory is where a mechanical split of
|
||||
# lifecycle_transition_api_test.rs moves its tests (rustfs/backlog#2765).
|
||||
is_dedicated_test_file() {
|
||||
local file="$1"
|
||||
|
||||
[[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]] ||
|
||||
[[ "$file" == rustfs/src/app/lifecycle_transition_api_test/* ]]
|
||||
}
|
||||
|
||||
emit_crate_use_statements() {
|
||||
(cd "$ROOT_DIR" && rg --files -g '*.rs' rustfs/src | while IFS= read -r file; do
|
||||
# The guard is file-scoped: dedicated test modules are excluded, while
|
||||
# inline #[cfg(test)] imports remain subject to the source file's layer.
|
||||
if [[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]]; then
|
||||
if is_dedicated_test_file "$file"; then
|
||||
continue
|
||||
fi
|
||||
perl -0777 -ne '
|
||||
|
||||
@@ -1094,11 +1094,29 @@ trace_hot_spans=(
|
||||
"rustfs/src/app/object:execute_get_object"
|
||||
)
|
||||
|
||||
# A hot span in a file that is due for a mechanical split (rustfs/backlog#2765)
|
||||
# is also looked up in the directory the split moves its functions into; the
|
||||
# named file itself stays required.
|
||||
# shellcheck source=scripts/guard_source_set.sh
|
||||
source "${repo_root}/scripts/guard_source_set.sh"
|
||||
|
||||
for hot_span in "${trace_hot_spans[@]}"; do
|
||||
file="${hot_span%%:*}"
|
||||
function="${hot_span##*:}"
|
||||
hot_span_sources=("$file")
|
||||
split_dir=""
|
||||
case "$file" in
|
||||
rustfs/src/app/bucket_usecase.rs) split_dir="rustfs/src/app/bucket_usecase" ;;
|
||||
esac
|
||||
if [[ -n "$split_dir" ]]; then
|
||||
if ! guard_source_set "$repo_root" "$file" "$split_dir"; then
|
||||
echo "❌ logging guardrail violation: $file::$function guard source is missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
hot_span_sources=("${GUARD_SOURCES[@]}")
|
||||
fi
|
||||
trace_span_pattern="#\\[(tracing::)?instrument\\([^]]*level = \\\"trace\\\"[^]]*\\)\\]([[:space:]]+#\\[[^]]+\\])*[[:space:]]+(pub(\\([^)]*\\))?[[:space:]]+)?(super[[:space:]]+)?async fn ${function}\\b"
|
||||
if ! rg -U "$trace_span_pattern" "$file" >/dev/null; then
|
||||
if ! rg -U "$trace_span_pattern" "${hot_span_sources[@]}" >/dev/null; then
|
||||
echo "❌ logging guardrail violation: $file::$function must remain TRACE-only" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Source-set helper for text guards that must survive a mechanical file split.
|
||||
# Sourced by check_architecture_migration_rules.sh and check_logging_guardrails.sh;
|
||||
# contract tests: scripts/test_guard_source_set.sh.
|
||||
#
|
||||
# A guard that greps one named file stops seeing the code it guards once that
|
||||
# file is split into a sibling directory. guard_source_set resolves the set of
|
||||
# files such a guard is about instead:
|
||||
#
|
||||
# guard_source_set <root> <required-file> [<split-dir>...]
|
||||
#
|
||||
# On success it sets the global array GUARD_SOURCES to <required-file> followed
|
||||
# by every `.rs` file under each <split-dir> that exists, in byte order of path
|
||||
# (LC_ALL=C), all relative to <root>. An absent <split-dir> contributes nothing.
|
||||
# It returns 1 with a message on stderr when <required-file> is not a regular
|
||||
# file (a guard whose input is missing must fail, not pass) or when a
|
||||
# <split-dir> exists but is not a directory. On failure GUARD_SOURCES still
|
||||
# holds <required-file>, so a caller that records the failure and continues
|
||||
# hands the missing path to its search tool rather than nothing.
|
||||
|
||||
guard_source_set() {
|
||||
local root="$1" required="$2" split_dir path listing
|
||||
shift 2
|
||||
GUARD_SOURCES=("$required")
|
||||
|
||||
if [[ ! -f "${root}/${required}" ]]; then
|
||||
printf 'guard source %s is missing\n' "$required" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
for split_dir in "$@"; do
|
||||
if [[ ! -e "${root}/${split_dir}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ ! -d "${root}/${split_dir}" ]]; then
|
||||
printf 'guard split path %s is not a directory\n' "$split_dir" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! listing="$(set -o pipefail; cd "$root" && find "$split_dir" -type f -name '*.rs' | LC_ALL=C sort)"; then
|
||||
printf 'guard split directory %s cannot be listed\n' "$split_dir" >&2
|
||||
return 1
|
||||
fi
|
||||
while IFS= read -r path; do
|
||||
if [[ -n "$path" ]]; then
|
||||
GUARD_SOURCES+=("$path")
|
||||
fi
|
||||
done <<<"$listing"
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# A rule that forbids reaching an owner module through `super::` is only right
|
||||
# at the depth it was written for: in rustfs/src/app/bucket_usecase.rs `super`
|
||||
# is `crate::app`, but in rustfs/src/app/bucket_usecase/cors.rs `super` is the
|
||||
# bucket usecase itself and `crate::app` is `super::super`. module_relative_rg
|
||||
# rewrites a pattern for each file's depth below the owner module directory:
|
||||
#
|
||||
# module_relative_rg <module-dir> <template> [<rg-option>...] -- <file>...
|
||||
#
|
||||
# Every `@UP@` in <template> becomes the `super::` chain that names the owner
|
||||
# module from that file's own scope: one `super::` for <module-dir>/x.rs and
|
||||
# <module-dir>/x/mod.rs, two for <module-dir>/x/y.rs, and so on. A file at
|
||||
# depth 0 (<module-dir>/mod.rs) keeps one `super::`, the form the rule had
|
||||
# before it was depth-aware. Files are searched in groups of equal depth, so
|
||||
# rg's output stays `path:line:text`; rg's own errors (exit status 2) fail the
|
||||
# call, and "no match" (status 1) does not.
|
||||
module_relative_depth() {
|
||||
local module_dir="$1" file="$2" relative slashes depth
|
||||
relative="${file#"$module_dir"/}"
|
||||
slashes="${relative//[!\/]/}"
|
||||
depth=$((${#slashes} + 1))
|
||||
if [[ "${relative##*/}" == mod.rs ]]; then
|
||||
depth=$((depth - 1))
|
||||
fi
|
||||
printf '%s\n' "$depth"
|
||||
}
|
||||
|
||||
module_relative_rg() {
|
||||
local module_dir="$1" template="$2" file depth max_depth=0 up pattern status index count
|
||||
shift 2
|
||||
local -a options=() files=() depths=() group=()
|
||||
while [[ $# -gt 0 && "$1" != "--" ]]; do
|
||||
options+=("$1")
|
||||
shift
|
||||
done
|
||||
if [[ $# -eq 0 ]]; then
|
||||
printf 'module_relative_rg: missing -- before the file list\n' >&2
|
||||
return 2
|
||||
fi
|
||||
shift
|
||||
for file in "$@"; do
|
||||
if [[ "$file" != "$module_dir"/* ]]; then
|
||||
printf 'module_relative_rg: %s is not under %s\n' "$file" "$module_dir" >&2
|
||||
return 2
|
||||
fi
|
||||
depth="$(module_relative_depth "$module_dir" "$file")"
|
||||
files+=("$file")
|
||||
depths+=("$depth")
|
||||
if ((depth > max_depth)); then
|
||||
max_depth=$depth
|
||||
fi
|
||||
done
|
||||
count=${#files[@]}
|
||||
for ((depth = 0; depth <= max_depth; depth++)); do
|
||||
group=()
|
||||
for ((index = 0; index < count; index++)); do
|
||||
if [[ "${depths[index]}" == "$depth" ]]; then
|
||||
group+=("${files[index]}")
|
||||
fi
|
||||
done
|
||||
if [[ ${#group[@]} -eq 0 ]]; then
|
||||
continue
|
||||
fi
|
||||
up="super::"
|
||||
for ((index = 1; index < depth; index++)); do
|
||||
up+="super::"
|
||||
done
|
||||
pattern="${template//@UP@/$up}"
|
||||
status=0
|
||||
rg ${options[@]+"${options[@]}"} "$pattern" "${group[@]}" || status=$?
|
||||
if [[ $status -gt 1 ]]; then
|
||||
return "$status"
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
@@ -509,7 +509,9 @@ pass/fail table in the job summary.
|
||||
incomplete parameterized cases.
|
||||
- `api_coverage.py` — quantifies S3 API surface coverage by comparing the
|
||||
s3s `S3` trait (at the revision pinned in Cargo.toml) against the methods
|
||||
RustFS overrides in `impl S3 for FS`:
|
||||
RustFS overrides in `impl S3 for FS`, found exactly once across
|
||||
`rustfs/src/storage/ecfs.rs` and `rustfs/src/storage/legacy_edge/`
|
||||
(contract tests: `test_api_coverage.py`, run by `make script-tests`):
|
||||
|
||||
```bash
|
||||
python3 scripts/s3-tests/api_coverage.py # summary + missing ops
|
||||
|
||||
@@ -17,7 +17,8 @@
|
||||
|
||||
Compares the operations defined by the s3s `S3` trait (at the revision pinned
|
||||
in the workspace Cargo.toml) against the methods RustFS overrides in
|
||||
`impl S3 for FS` (rustfs/src/storage/ecfs.rs). Trait methods that are not
|
||||
`impl S3 for FS` (rustfs/src/storage/ecfs.rs, or the rustfs/src/storage/legacy_edge/
|
||||
directory a mechanical split of that file moves code into). Trait methods that are not
|
||||
overridden fall back to the s3s default implementation, which returns
|
||||
NotImplemented — so "overridden" is a faithful proxy for "implemented".
|
||||
|
||||
@@ -36,6 +37,8 @@ import sys
|
||||
import tempfile
|
||||
|
||||
ECFS_PATH = "rustfs/src/storage/ecfs.rs"
|
||||
ECFS_SPLIT_DIR = "rustfs/src/storage/legacy_edge"
|
||||
IMPL_MARKER = "impl S3 for FS {"
|
||||
TRAIT_RELPATH = "crates/s3s/src/s3_trait.rs"
|
||||
|
||||
|
||||
@@ -97,14 +100,28 @@ def trait_operations(trait_source: pathlib.Path) -> dict[str, str]:
|
||||
return ops
|
||||
|
||||
|
||||
def implemented_operations(ecfs_source: pathlib.Path) -> set[str]:
|
||||
src = ecfs_source.read_text(encoding="utf-8")
|
||||
marker = "impl S3 for FS {"
|
||||
start = src.find(marker)
|
||||
if start < 0:
|
||||
raise RuntimeError(f"`{marker}` not found in {ecfs_source}")
|
||||
block = extract_block(src, start)
|
||||
return set(re.findall(r"async fn (\w+)", block))
|
||||
def ecfs_sources(root: pathlib.Path) -> list[pathlib.Path]:
|
||||
"""Return ecfs.rs (required) followed by every .rs file under the split directory, in path order."""
|
||||
required = root / ECFS_PATH
|
||||
if not required.is_file():
|
||||
raise RuntimeError(f"{required} is missing")
|
||||
split_dir = root / ECFS_SPLIT_DIR
|
||||
if not split_dir.exists():
|
||||
return [required]
|
||||
if not split_dir.is_dir():
|
||||
raise RuntimeError(f"{split_dir} is not a directory")
|
||||
return [required, *sorted(split_dir.rglob("*.rs"), key=lambda path: path.relative_to(root).as_posix())]
|
||||
|
||||
|
||||
def implemented_operations(root: pathlib.Path) -> set[str]:
|
||||
"""Return the methods of the one `impl S3 for FS` block across the ECFS source set."""
|
||||
blocks = []
|
||||
for source in ecfs_sources(root):
|
||||
src = source.read_text(encoding="utf-8")
|
||||
blocks += [extract_block(src, match.start()) for match in re.finditer(re.escape(IMPL_MARKER), src)]
|
||||
if len(blocks) != 1:
|
||||
raise RuntimeError(f"`{IMPL_MARKER}` must occur exactly once under {ECFS_PATH} and {ECFS_SPLIT_DIR}, found {len(blocks)}")
|
||||
return set(re.findall(r"async fn (\w+)", blocks[0]))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -122,7 +139,7 @@ def main() -> int:
|
||||
trait_source = args.s3s_trait or locate_trait_source(git_url, rev)
|
||||
|
||||
all_ops = trait_operations(trait_source)
|
||||
impl_ops = implemented_operations(root / ECFS_PATH)
|
||||
impl_ops = implemented_operations(root)
|
||||
|
||||
stale = sorted(impl_ops - all_ops.keys())
|
||||
implemented = sorted(op for op in all_ops if op in impl_ops)
|
||||
|
||||
Executable
+329
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Plan and judge the s3-tests reference parity lane.
|
||||
|
||||
Subcommands:
|
||||
targets validate the target list and print it as a JSON matrix
|
||||
export-env validate one target's extra environment and print KEY=VALUE lines
|
||||
failed print the failed test names of one run, in implemented_tests.txt form
|
||||
compare compare every candidate against the reference, per pytest node id
|
||||
|
||||
`compare` reads <reports>/<target>/selected-nodeids.txt, junit.xml and the
|
||||
optional rerun-junit.xml of every target. Each node id is reduced to pass /
|
||||
fail / skip / missing; a case that failed and then passed its single rerun is a
|
||||
pass and is listed as flaky. A tolerated entry must match a real difference.
|
||||
|
||||
Exit codes: 0 no untolerated difference and no stale entry; 1 a difference, a
|
||||
stale tolerated entry or an incomplete run; 2 missing or malformed input.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import junit_diff # noqa: E402 (sibling script, not a package)
|
||||
|
||||
PASS, FAIL, SKIP, MISSING = junit_diff.PASS, junit_diff.FAIL, junit_diff.SKIP, junit_diff.MISSING
|
||||
NAME = re.compile(r"[a-z][a-z0-9-]{0,39}")
|
||||
ENV_KEY = re.compile(r"RUSTFS_[A-Z0-9_]+")
|
||||
ENV_VALUE = re.compile(r"[A-Za-z0-9._,:/=+-]*")
|
||||
ISSUE = re.compile(r"https://github\.com/rustfs/[A-Za-z0-9_.-]+/issues/[1-9][0-9]*")
|
||||
NODEID = re.compile(r"[A-Za-z0-9_/]+\.py(::[A-Za-z0-9_]+)+(\[[^\]\s]*\])?")
|
||||
TEST_FILE_PREFIX = "s3tests/functional/test_s3.py::"
|
||||
BINARIES = ("reference", "candidate")
|
||||
|
||||
|
||||
class InputError(Exception):
|
||||
"""Input that cannot be trusted; maps to exit code 2."""
|
||||
|
||||
|
||||
# --- target list -----------------------------------------------------------
|
||||
|
||||
def load_targets(path):
|
||||
try:
|
||||
data = tomllib.loads(Path(path).read_text(encoding="utf-8"))
|
||||
except FileNotFoundError as exc:
|
||||
raise InputError(f"{path}: target list is missing") from exc
|
||||
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
|
||||
raise InputError(f"{path}: cannot read target list: {exc}") from exc
|
||||
if data.get("schema") != 1 or type(data.get("schema")) is not int or set(data) != {"schema", "targets"}:
|
||||
raise InputError(f"{path}: unsupported target list schema")
|
||||
return validate_targets(data["targets"], str(path))
|
||||
|
||||
|
||||
def validate_targets(targets, where="targets"):
|
||||
if not isinstance(targets, list) or len(targets) < 2:
|
||||
raise InputError(f"{where}: need one reference and at least one candidate")
|
||||
result, names = [], set()
|
||||
for item in targets:
|
||||
if not isinstance(item, dict) or not set(item) <= {"name", "binary", "env"}:
|
||||
raise InputError(f"{where}: invalid target entry {item!r}")
|
||||
name, binary, env = item.get("name"), item.get("binary"), item.get("env", {})
|
||||
if not isinstance(name, str) or not NAME.fullmatch(name) or name in names:
|
||||
raise InputError(f"{where}: invalid or duplicate target name {name!r}")
|
||||
if binary not in BINARIES:
|
||||
raise InputError(f"{where}: target {name} has unknown binary {binary!r}")
|
||||
validate_env(env, f"{where}: target {name}")
|
||||
names.add(name)
|
||||
result.append({"name": name, "binary": binary, "env": dict(env)})
|
||||
if [t["binary"] for t in result].count("reference") != 1:
|
||||
raise InputError(f"{where}: exactly one target must use the reference binary")
|
||||
return result
|
||||
|
||||
|
||||
def validate_env(env, where):
|
||||
if not isinstance(env, dict):
|
||||
raise InputError(f"{where}: env must be a table")
|
||||
for key, value in env.items():
|
||||
if not isinstance(key, str) or not ENV_KEY.fullmatch(key):
|
||||
raise InputError(f"{where}: env key {key!r} must match RUSTFS_[A-Z0-9_]+")
|
||||
if not isinstance(value, str) or not ENV_VALUE.fullmatch(value):
|
||||
raise InputError(f"{where}: env value for {key} has unsupported characters")
|
||||
return env
|
||||
|
||||
|
||||
# --- node ids and reports --------------------------------------------------
|
||||
|
||||
def nodeid_to_case(nodeid):
|
||||
"""Map a pytest node id to the classname::name id that junit_diff uses."""
|
||||
if not NODEID.fullmatch(nodeid):
|
||||
raise InputError(f"invalid pytest node id {nodeid!r}")
|
||||
path, *parts = nodeid.split("::")
|
||||
module = path[: -len(".py")].replace("/", ".")
|
||||
classname = ".".join([module, *parts[:-1]])
|
||||
return junit_diff.case_id(classname, parts[-1])
|
||||
|
||||
|
||||
def read_selected(path):
|
||||
try:
|
||||
lines = Path(path).read_text(encoding="utf-8").splitlines()
|
||||
except FileNotFoundError as exc:
|
||||
raise InputError(f"{path}: selected node id list is missing") from exc
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
raise InputError(f"{path}: cannot read selected node ids: {exc}") from exc
|
||||
nodeids = [line.strip() for line in lines if line.strip()]
|
||||
if not nodeids:
|
||||
raise InputError(f"{path}: selected node id list is empty")
|
||||
if len(set(nodeids)) != len(nodeids):
|
||||
raise InputError(f"{path}: duplicate selected node id")
|
||||
by_case = {}
|
||||
for nodeid in nodeids:
|
||||
case = nodeid_to_case(nodeid)
|
||||
if case in by_case:
|
||||
raise InputError(f"{path}: node ids {by_case[case]} and {nodeid} collide")
|
||||
by_case[case] = nodeid
|
||||
return nodeids, by_case
|
||||
|
||||
|
||||
def target_states(directory):
|
||||
"""Return (nodeids, {nodeid: state}, [flaky nodeids]) for one target run."""
|
||||
directory = Path(directory)
|
||||
if not directory.is_dir():
|
||||
raise InputError(f"{directory}: report directory is missing")
|
||||
nodeids, by_case = read_selected(directory / "selected-nodeids.txt")
|
||||
try:
|
||||
first, _ = junit_diff.parse_report(str(directory / "junit.xml"))
|
||||
except junit_diff.InputError as exc:
|
||||
raise InputError(str(exc)) from exc
|
||||
unexpected = sorted(set(first) - set(by_case))
|
||||
if unexpected:
|
||||
raise InputError(f"{directory}: junit.xml has unselected cases: {', '.join(unexpected[:5])}")
|
||||
states = {by_case[case]: state for case, state in first.items()}
|
||||
flaky = []
|
||||
rerun_path = directory / "rerun-junit.xml"
|
||||
if rerun_path.exists():
|
||||
try:
|
||||
rerun, _ = junit_diff.parse_report(str(rerun_path))
|
||||
except junit_diff.InputError as exc:
|
||||
raise InputError(str(exc)) from exc
|
||||
for case, state in sorted(rerun.items()):
|
||||
nodeid = by_case.get(case)
|
||||
if nodeid is None or states.get(nodeid) != FAIL:
|
||||
raise InputError(f"{rerun_path}: {case} was rerun without failing first")
|
||||
if state == PASS:
|
||||
flaky.append(nodeid)
|
||||
states[nodeid] = PASS if state == PASS else FAIL
|
||||
for nodeid in nodeids:
|
||||
states.setdefault(nodeid, MISSING)
|
||||
return nodeids, states, flaky
|
||||
|
||||
|
||||
def failed_names(directory):
|
||||
_, states, _ = target_states(directory)
|
||||
names = []
|
||||
for nodeid, state in states.items():
|
||||
if state == FAIL:
|
||||
if not nodeid.startswith(TEST_FILE_PREFIX):
|
||||
raise InputError(f"cannot express {nodeid} as an implemented_tests.txt entry")
|
||||
names.append(nodeid[len(TEST_FILE_PREFIX):])
|
||||
return sorted(names)
|
||||
|
||||
|
||||
# --- tolerated differences ---------------------------------------------------
|
||||
|
||||
def read_tolerated(path, candidates):
|
||||
try:
|
||||
lines = Path(path).read_text(encoding="utf-8").splitlines()
|
||||
except FileNotFoundError as exc:
|
||||
raise InputError(f"{path}: tolerated-difference file is missing") from exc
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
raise InputError(f"{path}: cannot read tolerated differences: {exc}") from exc
|
||||
entries = {}
|
||||
for number, line in enumerate(lines, 1):
|
||||
if not line.strip() or line.lstrip().startswith("#"):
|
||||
continue
|
||||
fields = [part.strip() for part in line.split(" | ")]
|
||||
if len(fields) != 4 or not all(fields):
|
||||
raise InputError(f"{path}:{number}: expected '<target> | <node id> | <issue URL> | <reason>'")
|
||||
target, nodeid, issue, _reason = fields
|
||||
if target not in candidates:
|
||||
raise InputError(f"{path}:{number}: {target!r} is not a candidate target")
|
||||
nodeid_to_case(nodeid)
|
||||
if not ISSUE.fullmatch(issue):
|
||||
raise InputError(f"{path}:{number}: tracking issue must be a https://github.com/rustfs/<repo>/issues/<n> URL")
|
||||
if (target, nodeid) in entries:
|
||||
raise InputError(f"{path}:{number}: duplicate entry for {target} {nodeid}")
|
||||
entries[(target, nodeid)] = issue
|
||||
return entries
|
||||
|
||||
|
||||
# --- comparison ----------------------------------------------------------------
|
||||
|
||||
def count(states):
|
||||
result = {PASS: 0, FAIL: 0, SKIP: 0, MISSING: 0}
|
||||
for state in states.values():
|
||||
result[state] += 1
|
||||
return result
|
||||
|
||||
|
||||
def compare(targets, reports, tolerated_path):
|
||||
reference = next(t["name"] for t in targets if t["binary"] == "reference")
|
||||
candidates = [t["name"] for t in targets if t["binary"] == "candidate"]
|
||||
tolerated = read_tolerated(tolerated_path, set(candidates))
|
||||
runs = {t["name"]: target_states(Path(reports) / t["name"]) for t in targets}
|
||||
selected = runs[reference][0]
|
||||
for name, (nodeids, _, _) in runs.items():
|
||||
if nodeids != selected:
|
||||
raise InputError(f"{name}: selected node ids differ from the {reference} run")
|
||||
ref_states = runs[reference][1]
|
||||
differences = []
|
||||
for name in candidates:
|
||||
states = runs[name][1]
|
||||
for nodeid in selected:
|
||||
if ref_states[nodeid] != states[nodeid]:
|
||||
issue = tolerated.get((name, nodeid))
|
||||
differences.append({"target": name, "nodeid": nodeid, "reference": ref_states[nodeid],
|
||||
"candidate": states[nodeid], "tolerated": issue})
|
||||
seen = {(d["target"], d["nodeid"]) for d in differences}
|
||||
stale = [{"target": t, "nodeid": n, "issue": i} for (t, n), i in sorted(tolerated.items()) if (t, n) not in seen]
|
||||
incomplete = {name: [n for n in selected if run[1][n] == MISSING] for name, run in runs.items()}
|
||||
return {
|
||||
"reference": reference,
|
||||
"candidates": candidates,
|
||||
"selected": len(selected),
|
||||
"counts": {name: count(run[1]) for name, run in runs.items()},
|
||||
"flaky": {name: run[2] for name, run in runs.items()},
|
||||
"differences": differences,
|
||||
"stale_tolerated": stale,
|
||||
"incomplete": {name: ids for name, ids in incomplete.items() if ids},
|
||||
}
|
||||
|
||||
|
||||
def failed(summary):
|
||||
return bool(summary["stale_tolerated"] or summary["incomplete"]
|
||||
or any(not d["tolerated"] for d in summary["differences"]))
|
||||
|
||||
|
||||
def render(summary):
|
||||
out = ["## s3-tests reference parity", ""]
|
||||
out.append(f"Reference target `{summary['reference']}`; {summary['selected']} selected node ids per target.")
|
||||
out += ["", "| Target | pass | fail | skip | missing | flaky (passed on rerun) |", "|---|---:|---:|---:|---:|---:|"]
|
||||
for name, c in summary["counts"].items():
|
||||
out.append(f"| `{name}` | {c[PASS]} | {c[FAIL]} | {c[SKIP]} | {c[MISSING]} | {len(summary['flaky'][name])} |")
|
||||
diffs = summary["differences"]
|
||||
out += ["", f"### Differences ({len(diffs)})", ""]
|
||||
if diffs:
|
||||
out += ["| Target | Node id | reference | candidate | Tolerated |", "|---|---|---|---|---|"]
|
||||
for d in diffs:
|
||||
out.append(f"| `{d['target']}` | `{d['nodeid']}` | {d['reference']} | {d['candidate']} | {d['tolerated'] or 'no'} |")
|
||||
else:
|
||||
out.append("No differences.")
|
||||
if summary["stale_tolerated"]:
|
||||
out += ["", "### Stale tolerated entries (no matching difference; remove them)", ""]
|
||||
out += [f"- `{s['target']}` `{s['nodeid']}` ({s['issue']})" for s in summary["stale_tolerated"]]
|
||||
if summary["incomplete"]:
|
||||
out += ["", "### Incomplete runs (selected node ids without a result)", ""]
|
||||
out += [f"- `{name}`: {len(ids)} missing, first `{ids[0]}`" for name, ids in summary["incomplete"].items()]
|
||||
flaky = [(name, ids) for name, ids in summary["flaky"].items() if ids]
|
||||
if flaky:
|
||||
out += ["", "### Flaky (failed, then passed the single rerun; not a difference)", ""]
|
||||
out += [f"- `{name}`: " + ", ".join(f"`{i}`" for i in ids) for name, ids in flaky]
|
||||
out += ["", "Result: **" + ("FAIL" if failed(summary) else "PASS") + "**", ""]
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = parser.add_subparsers(dest="mode", required=True)
|
||||
tgt = sub.add_parser("targets")
|
||||
tgt.add_argument("--config", required=True)
|
||||
env = sub.add_parser("export-env")
|
||||
env.add_argument("--target-json", required=True)
|
||||
fail = sub.add_parser("failed")
|
||||
fail.add_argument("--report", required=True, help="one target's report directory")
|
||||
cmp_ = sub.add_parser("compare")
|
||||
cmp_.add_argument("--targets-json", required=True)
|
||||
cmp_.add_argument("--reports", required=True)
|
||||
cmp_.add_argument("--tolerated", required=True)
|
||||
cmp_.add_argument("--markdown", help="write the summary table here (appended)")
|
||||
cmp_.add_argument("--json", help="write the machine-readable summary here")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
if args.mode == "targets":
|
||||
print(json.dumps(load_targets(args.config), separators=(",", ":")))
|
||||
elif args.mode == "export-env":
|
||||
target = json.loads(args.target_json)
|
||||
if not isinstance(target, dict):
|
||||
raise InputError("target must be a JSON object")
|
||||
for key, value in sorted(validate_env(target.get("env", {}), "target").items()):
|
||||
print(f"{key}={value}")
|
||||
elif args.mode == "failed":
|
||||
for name in failed_names(args.report):
|
||||
print(name)
|
||||
else:
|
||||
targets = validate_targets(json.loads(args.targets_json))
|
||||
summary = compare(targets, args.reports, args.tolerated)
|
||||
text = render(summary)
|
||||
print(text)
|
||||
if args.markdown:
|
||||
with open(args.markdown, "a", encoding="utf-8") as handle:
|
||||
handle.write(text + "\n")
|
||||
if args.json:
|
||||
with open(args.json, "w", encoding="utf-8") as handle:
|
||||
json.dump(summary, handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
return 1 if failed(summary) else 0
|
||||
except (InputError, json.JSONDecodeError) as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Contract tests for how api_coverage.py finds `impl S3 for FS` across the ECFS source set."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import api_coverage # noqa: E402
|
||||
|
||||
IMPL = "impl S3 for FS {\n async fn get_object(&self) {}\n async fn put_object(&self) {}\n}\n"
|
||||
|
||||
|
||||
class ImplementedOperationsTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def write(self, relative: str, text: str) -> None:
|
||||
path = self.root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
def test_reads_the_impl_from_ecfs_when_the_split_directory_is_absent(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n" + IMPL)
|
||||
|
||||
self.assertEqual(api_coverage.implemented_operations(self.root), {"get_object", "put_object"})
|
||||
|
||||
def test_finds_the_impl_after_it_moves_into_the_split_directory(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/bucket_core.rs", "impl FS {\n async fn helper(&self) {}\n}\n")
|
||||
|
||||
self.assertEqual(api_coverage.implemented_operations(self.root), {"get_object", "put_object"})
|
||||
|
||||
def test_missing_ecfs_fails_even_when_the_split_directory_holds_the_impl(self) -> None:
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "ecfs.rs is missing"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_an_impl_in_both_ecfs_and_the_split_directory_fails(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, IMPL)
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "exactly once.*found 2"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_no_impl_anywhere_fails(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/bucket_core.rs", "impl FS {}\n")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "found 0"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_an_impl_in_a_non_rust_file_of_the_split_directory_is_not_source(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/notes.md", IMPL)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "found 0"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_a_split_path_that_is_a_file_fails(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, IMPL)
|
||||
self.write(api_coverage.ECFS_SPLIT_DIR, "stray\n")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "not a directory"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Executable
+332
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contract tests for reference_parity.py: target list, rerun merge, tolerated entries and exit codes."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parent / "reference_parity.py"
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
PREFIX = "s3tests/functional/test_s3.py::"
|
||||
CLASSNAME = "s3tests.functional.test_s3"
|
||||
TARGETS = [{"name": "reference", "binary": "reference", "env": {}},
|
||||
{"name": "candidate-legacy", "binary": "candidate", "env": {}}]
|
||||
ISSUE = "https://github.com/rustfs/rustfs/issues/1"
|
||||
|
||||
|
||||
def junit(states):
|
||||
cases = []
|
||||
for name, state in states.items():
|
||||
body = {"pass": "", "fail": "<failure message='x'/>", "error": "<error message='x'/>",
|
||||
"skip": "<skipped/>"}[state]
|
||||
cases.append(f'<testcase classname="{CLASSNAME}" name="{name}">{body}</testcase>')
|
||||
return f'<testsuites><testsuite name="pytest">{"".join(cases)}</testsuite></testsuites>'
|
||||
|
||||
|
||||
def run(*args):
|
||||
proc = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, check=False)
|
||||
return proc.returncode, proc.stdout, proc.stderr
|
||||
|
||||
|
||||
class Lane:
|
||||
"""A temporary reports directory with one sub-directory per target."""
|
||||
|
||||
def __init__(self, case):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
case.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.reports = self.root / "reports"
|
||||
self.tolerated = self.root / "tolerated.txt"
|
||||
self.tolerated.write_text("# none\n")
|
||||
|
||||
def target(self, name, states, selected=None, rerun=None):
|
||||
directory = self.reports / name
|
||||
directory.mkdir(parents=True)
|
||||
names = selected if selected is not None else list(states)
|
||||
(directory / "selected-nodeids.txt").write_text("".join(PREFIX + n + "\n" for n in names))
|
||||
(directory / "junit.xml").write_text(junit(states))
|
||||
if rerun is not None:
|
||||
(directory / "rerun-junit.xml").write_text(junit(rerun))
|
||||
return directory
|
||||
|
||||
def compare(self, targets=None):
|
||||
summary = self.root / "summary.json"
|
||||
code, out, err = run("compare", "--targets-json", json.dumps(targets or TARGETS),
|
||||
"--reports", str(self.reports), "--tolerated", str(self.tolerated),
|
||||
"--json", str(summary))
|
||||
data = json.loads(summary.read_text()) if summary.exists() else None
|
||||
return code, out, err, data
|
||||
|
||||
|
||||
BASE = {"test_a": "pass", "test_b": "pass", "test_c": "skip"}
|
||||
|
||||
|
||||
class CompareTests(unittest.TestCase):
|
||||
# --- negative cases ---
|
||||
|
||||
def test_pass_versus_fail_is_a_difference(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
|
||||
code, out, _, data = lane.compare()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(data["differences"], [{"target": "candidate-legacy", "nodeid": PREFIX + "test_a",
|
||||
"reference": "pass", "candidate": "fail", "tolerated": None}])
|
||||
self.assertIn("Result: **FAIL**", out)
|
||||
|
||||
def test_error_counts_as_fail(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_b": "error"})
|
||||
code, _, _, data = lane.compare()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(data["differences"][0]["candidate"], "fail")
|
||||
|
||||
def test_reference_fail_candidate_pass_is_also_a_difference(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", {**BASE, "test_a": "fail"})
|
||||
lane.target("candidate-legacy", BASE)
|
||||
code, _, _, data = lane.compare()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(data["differences"][0]["reference"], "fail")
|
||||
|
||||
def test_skip_versus_pass_is_a_difference(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_c": "pass"})
|
||||
code, _, _, data = lane.compare()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(data["differences"][0]["nodeid"], PREFIX + "test_c")
|
||||
|
||||
def test_case_absent_from_candidate_report_is_missing_and_incomplete(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {"test_a": "pass", "test_c": "skip"}, selected=list(BASE))
|
||||
code, out, _, data = lane.compare()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(data["differences"][0]["candidate"], "missing")
|
||||
self.assertEqual(data["incomplete"], {"candidate-legacy": [PREFIX + "test_b"]})
|
||||
self.assertIn("Incomplete runs", out)
|
||||
|
||||
def test_case_missing_from_both_reports_still_fails(self):
|
||||
lane = Lane(self)
|
||||
partial = {"test_a": "pass", "test_c": "skip"}
|
||||
lane.target("reference", partial, selected=list(BASE))
|
||||
lane.target("candidate-legacy", partial, selected=list(BASE))
|
||||
code, _, _, data = lane.compare()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(data["differences"], [])
|
||||
self.assertEqual(set(data["incomplete"]), {"reference", "candidate-legacy"})
|
||||
|
||||
def test_failure_that_also_fails_the_rerun_stays_a_difference(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_a": "fail"})
|
||||
code, _, _, data = lane.compare()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(data["flaky"]["candidate-legacy"], [])
|
||||
|
||||
def test_rerun_of_a_case_that_did_not_fail_is_rejected(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_b": "pass"})
|
||||
code, _, err, _ = lane.compare()
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("rerun without failing first", err)
|
||||
|
||||
def test_missing_target_report_fails_rather_than_skips(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
code, _, err, data = lane.compare()
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIsNone(data)
|
||||
self.assertIn("candidate-legacy: report directory is missing", err)
|
||||
|
||||
def test_missing_junit_fails(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
(lane.target("candidate-legacy", BASE) / "junit.xml").unlink()
|
||||
code, _, err, _ = lane.compare()
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("cannot read", err)
|
||||
|
||||
def test_different_selected_lists_are_rejected(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {"test_a": "pass", "test_b": "pass"})
|
||||
code, _, err, _ = lane.compare()
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("selected node ids differ", err)
|
||||
|
||||
def test_unselected_case_in_junit_is_rejected(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_extra": "pass"}, selected=list(BASE))
|
||||
code, _, err, _ = lane.compare()
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("unselected cases", err)
|
||||
|
||||
def test_stale_tolerated_entry_fails(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", BASE)
|
||||
lane.tolerated.write_text(f"candidate-legacy | {PREFIX}test_a | {ISSUE} | known drift\n")
|
||||
code, out, _, data = lane.compare()
|
||||
self.assertEqual(code, 1)
|
||||
self.assertEqual(data["stale_tolerated"][0]["nodeid"], PREFIX + "test_a")
|
||||
self.assertIn("Stale tolerated entries", out)
|
||||
|
||||
def test_tolerated_entry_for_another_target_does_not_cover_the_difference(self):
|
||||
lane = Lane(self)
|
||||
targets = TARGETS + [{"name": "candidate-gateway", "binary": "candidate", "env": {}}]
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
|
||||
lane.target("candidate-gateway", {**BASE, "test_a": "fail"})
|
||||
lane.tolerated.write_text(f"candidate-gateway | {PREFIX}test_a | {ISSUE} | gateway gap\n")
|
||||
code, _, _, data = lane.compare(targets)
|
||||
self.assertEqual(code, 1)
|
||||
untolerated = [d["target"] for d in data["differences"] if not d["tolerated"]]
|
||||
self.assertEqual(untolerated, ["candidate-legacy"])
|
||||
|
||||
def test_tolerated_entry_without_issue_url_is_rejected(self):
|
||||
for line in (f"candidate-legacy | {PREFIX}test_a | rustfs#1 | reason",
|
||||
f"candidate-legacy | {PREFIX}test_a | {ISSUE} | ",
|
||||
f"candidate-legacy | {PREFIX}test_a | {ISSUE}",
|
||||
f"reference | {PREFIX}test_a | {ISSUE} | reason",
|
||||
f"candidate-legacy | test_a | {ISSUE} | reason"):
|
||||
with self.subTest(line=line):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
|
||||
lane.tolerated.write_text(line + "\n")
|
||||
code, _, err, _ = lane.compare()
|
||||
self.assertEqual(code, 2, err)
|
||||
|
||||
def test_duplicate_tolerated_entry_is_rejected(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
|
||||
entry = f"candidate-legacy | {PREFIX}test_a | {ISSUE} | reason\n"
|
||||
lane.tolerated.write_text(entry * 2)
|
||||
code, _, err, _ = lane.compare()
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("duplicate entry", err)
|
||||
|
||||
def test_missing_tolerated_file_fails(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", BASE)
|
||||
lane.tolerated.unlink()
|
||||
code, _, err, _ = lane.compare()
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("tolerated-difference file is missing", err)
|
||||
|
||||
# --- positive cases ---
|
||||
|
||||
def test_identical_runs_pass(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", BASE)
|
||||
code, out, _, data = lane.compare()
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(data["counts"]["candidate-legacy"], {"pass": 2, "fail": 0, "skip": 1, "missing": 0})
|
||||
self.assertIn("No differences.", out)
|
||||
self.assertIn("Result: **PASS**", out)
|
||||
|
||||
def test_failure_that_passes_the_rerun_is_flaky_not_a_difference(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_a": "fail"}, rerun={"test_a": "pass"})
|
||||
code, out, _, data = lane.compare()
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(data["flaky"]["candidate-legacy"], [PREFIX + "test_a"])
|
||||
self.assertIn("Flaky", out)
|
||||
|
||||
def test_tolerated_difference_passes_and_is_labelled(self):
|
||||
lane = Lane(self)
|
||||
lane.target("reference", BASE)
|
||||
lane.target("candidate-legacy", {**BASE, "test_a": "fail"})
|
||||
lane.tolerated.write_text(f"# comment\ncandidate-legacy | {PREFIX}test_a | {ISSUE} | known drift\n")
|
||||
code, out, _, data = lane.compare()
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(data["differences"][0]["tolerated"], ISSUE)
|
||||
self.assertIn(ISSUE, out)
|
||||
|
||||
|
||||
class TargetTests(unittest.TestCase):
|
||||
def write(self, text):
|
||||
temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(temp.cleanup)
|
||||
path = Path(temp.name) / "targets.toml"
|
||||
path.write_text(text)
|
||||
return path
|
||||
|
||||
def test_invalid_target_lists_are_rejected(self):
|
||||
ref = '[[targets]]\nname = "reference"\nbinary = "reference"\n'
|
||||
cand = '[[targets]]\nname = "candidate-legacy"\nbinary = "candidate"\n'
|
||||
for label, text in {
|
||||
"no schema": ref + cand,
|
||||
"only reference": "schema = 1\n" + ref,
|
||||
"two references": "schema = 1\n" + ref + ref.replace('"reference"\nbinary', '"other"\nbinary'),
|
||||
"no reference": "schema = 1\n" + cand + cand.replace("candidate-legacy", "candidate-two"),
|
||||
"duplicate name": "schema = 1\n" + ref + cand + cand,
|
||||
"unknown binary": "schema = 1\n" + ref + cand.replace('binary = "candidate"', 'binary = "main"'),
|
||||
"bad name": "schema = 1\n" + ref + cand.replace("candidate-legacy", "Candidate Legacy"),
|
||||
"foreign env key": "schema = 1\n" + ref + cand + 'env = { PATH = "/tmp" }\n',
|
||||
"env injection": "schema = 1\n" + ref + cand + 'env = { RUSTFS_X = "a\\nB=c" }\n',
|
||||
"unknown key": "schema = 1\n" + ref + cand + 'stack = "gateway"\n',
|
||||
}.items():
|
||||
with self.subTest(label):
|
||||
code, _, _ = run("targets", "--config", str(self.write(text)))
|
||||
self.assertEqual(code, 2)
|
||||
|
||||
def test_missing_target_file_fails(self):
|
||||
code, _, err = run("targets", "--config", "/nonexistent/targets.toml")
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("target list is missing", err)
|
||||
|
||||
def test_export_env_rejects_foreign_keys(self):
|
||||
code, _, _ = run("export-env", "--target-json", json.dumps({"env": {"LD_PRELOAD": "x"}}))
|
||||
self.assertEqual(code, 2)
|
||||
|
||||
def test_repository_target_list_is_valid(self):
|
||||
code, out, err = run("targets", "--config", str(REPO / ".config/reference-parity-targets.toml"))
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertEqual(json.loads(out), TARGETS)
|
||||
|
||||
def test_export_env_prints_validated_lines(self):
|
||||
code, out, _ = run("export-env", "--target-json", json.dumps({"env": {"RUSTFS_S3_STACK": "gateway"}}))
|
||||
self.assertEqual((code, out), (0, "RUSTFS_S3_STACK=gateway\n"))
|
||||
|
||||
|
||||
class FailedTests(unittest.TestCase):
|
||||
def test_failed_lists_only_failures_in_test_list_form(self):
|
||||
lane = Lane(self)
|
||||
directory = lane.target("reference", {**BASE, "test_a": "fail", "test_d": "error"})
|
||||
code, out, _ = run("failed", "--report", str(directory))
|
||||
self.assertEqual((code, out), (0, "test_a\ntest_d\n"))
|
||||
|
||||
def test_failed_rejects_a_missing_report(self):
|
||||
code, _, err = run("failed", "--report", "/nonexistent/report")
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("report directory is missing", err)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Executable
+237
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env bash
|
||||
# Contract tests for scripts/guard_source_set.sh, the source-set helper that
|
||||
# lets text guards read a file plus the directory a mechanical split moves its
|
||||
# code into (rustfs/backlog#2765).
|
||||
#
|
||||
# Each case builds a throwaway tree; nothing in the real workspace is read.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck source=scripts/guard_source_set.sh
|
||||
source "${GUARD_SOURCE_SET_SCRIPT:-$SCRIPT_DIR/guard_source_set.sh}"
|
||||
TMP_ROOT="$(mktemp -d)"
|
||||
trap 'chmod -R u+rwx "$TMP_ROOT" 2>/dev/null || true; rm -rf "$TMP_ROOT"' EXIT
|
||||
|
||||
passed=0
|
||||
skipped=0
|
||||
case_no=0
|
||||
FX=""
|
||||
STATUS=0
|
||||
ERR=""
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
new_case() {
|
||||
case_no=$((case_no + 1))
|
||||
FX="$TMP_ROOT/case$case_no"
|
||||
mkdir -p "$FX"
|
||||
}
|
||||
|
||||
put() {
|
||||
mkdir -p "$(dirname "$FX/$1")"
|
||||
printf 'fn guarded() {}\n' >"$FX/$1"
|
||||
}
|
||||
|
||||
# run_set <args...>: call guard_source_set on the case tree, capturing status and stderr.
|
||||
run_set() {
|
||||
local err_file="$FX/.stderr"
|
||||
STATUS=0
|
||||
guard_source_set "$FX" "$@" 2>"$err_file" || STATUS=$?
|
||||
ERR="$(cat "$err_file")"
|
||||
}
|
||||
|
||||
sources_are() {
|
||||
local expected actual
|
||||
expected="$(printf '%s\n' "$@")"
|
||||
actual="$(printf '%s\n' "${GUARD_SOURCES[@]}")"
|
||||
[[ "$actual" == "$expected" ]] || fail "case $case_no: sources were [$actual], expected [$expected]"
|
||||
}
|
||||
|
||||
ok() {
|
||||
passed=$((passed + 1))
|
||||
}
|
||||
|
||||
# --- positive cases ---------------------------------------------------------
|
||||
|
||||
# An absent split directory contributes nothing.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 0 ]] || fail "absent split dir: status $STATUS ($ERR)"
|
||||
sources_are src/usecase.rs
|
||||
ok
|
||||
|
||||
# A present split directory adds its .rs files, recursively, in byte order,
|
||||
# after the required file; non-.rs files are not source.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
put src/usecase/zeta.rs
|
||||
put src/usecase/B.rs
|
||||
put src/usecase/a.rs
|
||||
put src/usecase/tests/nested.rs
|
||||
printf 'not rust\n' >"$FX/src/usecase/notes.md"
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 0 ]] || fail "present split dir: status $STATUS ($ERR)"
|
||||
sources_are src/usecase.rs src/usecase/B.rs src/usecase/a.rs src/usecase/tests/nested.rs src/usecase/zeta.rs
|
||||
ok
|
||||
|
||||
# Several split directories are read in argument order; an empty one adds nothing.
|
||||
new_case
|
||||
put src/ecfs.rs
|
||||
put src/legacy_edge/b.rs
|
||||
mkdir -p "$FX/src/empty"
|
||||
put src/other/a.rs
|
||||
run_set src/ecfs.rs src/legacy_edge src/empty src/missing src/other
|
||||
[[ "$STATUS" == 0 ]] || fail "several split dirs: status $STATUS ($ERR)"
|
||||
sources_are src/ecfs.rs src/legacy_edge/b.rs src/other/a.rs
|
||||
ok
|
||||
|
||||
# --- negative cases ---------------------------------------------------------
|
||||
|
||||
# A missing required file fails and names the file.
|
||||
new_case
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "missing required file: status $STATUS"
|
||||
[[ "$ERR" == *"src/usecase.rs"* ]] || fail "missing required file: stderr does not name it: $ERR"
|
||||
sources_are src/usecase.rs
|
||||
ok
|
||||
|
||||
# A missing required file fails even when the split directory holds the code:
|
||||
# the split must keep the file, and a guard must not pass on half its input.
|
||||
new_case
|
||||
put src/usecase/moved.rs
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "missing required file with split dir: status $STATUS"
|
||||
[[ "$ERR" == *"is missing"* ]] || fail "missing required file with split dir: $ERR"
|
||||
ok
|
||||
|
||||
# A required path that is a directory is not a source file.
|
||||
new_case
|
||||
mkdir -p "$FX/src/usecase.rs"
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "required directory: status $STATUS"
|
||||
ok
|
||||
|
||||
# A split path that exists as a regular file fails rather than being skipped.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
printf 'stray\n' >"$FX/src/usecase"
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "split path is a file: status $STATUS"
|
||||
[[ "$ERR" == *"not a directory"* ]] || fail "split path is a file: $ERR"
|
||||
ok
|
||||
|
||||
# A later split path that is a file still fails after an earlier valid one.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
put src/usecase/a.rs
|
||||
printf 'stray\n' >"$FX/src/second"
|
||||
run_set src/usecase.rs src/usecase src/second
|
||||
[[ "$STATUS" == 1 ]] || fail "second split path is a file: status $STATUS"
|
||||
ok
|
||||
|
||||
# An unlistable split directory fails rather than contributing nothing.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
put src/usecase/hidden/a.rs
|
||||
chmod 000 "$FX/src/usecase/hidden"
|
||||
if [[ "$(id -u)" == 0 ]]; then
|
||||
echo "SKIP: unlistable split directory case needs a non-root user (root ignores directory permissions)"
|
||||
skipped=$((skipped + 1))
|
||||
else
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "unlistable split dir: status $STATUS"
|
||||
[[ "$ERR" == *"cannot be listed"* ]] || fail "unlistable split dir: $ERR"
|
||||
ok
|
||||
fi
|
||||
chmod 755 "$FX/src/usecase/hidden"
|
||||
|
||||
# --- module_relative_rg ------------------------------------------------------
|
||||
#
|
||||
# The templates are read from check_architecture_migration_rules.sh itself, so
|
||||
# these cases exercise the rules as shipped, not copies of them.
|
||||
ARCH_RULES="${ARCH_RULES_SCRIPT:-$SCRIPT_DIR/check_architecture_migration_rules.sh}"
|
||||
TEMPLATES=()
|
||||
while IFS= read -r template; do
|
||||
TEMPLATES+=("$template")
|
||||
done < <(awk '
|
||||
pending { sub(/^[[:space:]]*'"'"'/, ""); sub(/'"'"'[[:space:]]*\\$/, ""); print; pending = 0; next }
|
||||
/module_relative_rg rustfs\/src\/(app|storage) \\$/ { pending = 1 }
|
||||
' "$ARCH_RULES")
|
||||
[[ ${#TEMPLATES[@]} == 6 ]] || fail "expected 6 depth-aware rules in $ARCH_RULES, found ${#TEMPLATES[@]}"
|
||||
STORAGE_HELPER_RULE="${TEMPLATES[0]}"
|
||||
STORAGE_OWNER_RULE="${TEMPLATES[1]}"
|
||||
RUNTIME_IMPORT_RULE="${TEMPLATES[2]}"
|
||||
RUNTIME_PATH_RULE="${TEMPLATES[3]}"
|
||||
ECFS_TEST_RULE="${TEMPLATES[4]}"
|
||||
ECFS_TEST_SPLIT_RULE="${TEMPLATES[5]}"
|
||||
|
||||
# relative_hits <module-dir> <template> <relative-file> <line>: rg output for one file.
|
||||
relative_hits() {
|
||||
local module_dir="$1" template="$2" file="$3" line="$4"
|
||||
mkdir -p "$(dirname "$FX/$file")"
|
||||
printf '%s\n' "$line" >"$FX/$file"
|
||||
(cd "$FX" && module_relative_rg "$module_dir" "$template" -n -U --with-filename -- "$file")
|
||||
}
|
||||
|
||||
flagged() {
|
||||
[[ -n "$(relative_hits "$@")" ]] || fail "case $case_no: not flagged: $3: $4"
|
||||
ok
|
||||
}
|
||||
|
||||
not_flagged() {
|
||||
[[ -z "$(relative_hits "$@")" ]] || fail "case $case_no: flagged: $3: $4"
|
||||
ok
|
||||
}
|
||||
|
||||
new_case
|
||||
[[ "$(module_relative_depth rustfs/src/app rustfs/src/app/bucket_usecase.rs)" == 1 ]] || fail "depth of a module file"
|
||||
[[ "$(module_relative_depth rustfs/src/app rustfs/src/app/object/mod.rs)" == 1 ]] || fail "depth of a mod.rs"
|
||||
[[ "$(module_relative_depth rustfs/src/app rustfs/src/app/bucket_usecase/cors.rs)" == 2 ]] || fail "depth of a split file"
|
||||
[[ "$(module_relative_depth rustfs/src/app rustfs/src/app/multipart_usecase/tests/a.rs)" == 3 ]] || fail "depth of a nested split file"
|
||||
ok
|
||||
|
||||
# Forbidden forms: each spelling that names crate::app (or the storage root) is flagged.
|
||||
new_case
|
||||
flagged rustfs/src/app "$STORAGE_HELPER_RULE" rustfs/src/app/bucket_usecase.rs 'use super::StorageError;'
|
||||
flagged rustfs/src/app "$STORAGE_HELPER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::super::StorageError;'
|
||||
flagged rustfs/src/app "$STORAGE_HELPER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use crate::app::StorageError;'
|
||||
flagged rustfs/src/app "$STORAGE_HELPER_RULE" rustfs/src/app/object/mod.rs 'use super::{DiskError, StorageError};'
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/multipart_usecase/complete.rs 'use super::super::{quota, versioning_sys};'
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase.rs 'super::tagging::helper();'
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/tagging.rs 'super::super::tagging::helper();'
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase.rs 'super::super::metadata_sys::get();'
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::super::super::metadata_sys::get();'
|
||||
flagged rustfs/src/app "$RUNTIME_IMPORT_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::super::ECStore;'
|
||||
flagged rustfs/src/app "$RUNTIME_PATH_RULE" rustfs/src/app/lifecycle_transition_api_test/put.rs 'super::super::get_server_info();'
|
||||
flagged rustfs/src/storage "$ECFS_TEST_RULE" rustfs/src/storage/ecfs_test.rs ' use super::super::{BucketMetadata};'
|
||||
flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs 'use super::super::super::{BucketMetadata};'
|
||||
flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs ' use super::super::super::super::{check_preconditions};'
|
||||
|
||||
# Ordinary forms: a split file importing from its own parent module is not flagged.
|
||||
new_case
|
||||
not_flagged rustfs/src/app "$STORAGE_HELPER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::StorageError;'
|
||||
not_flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::{DefaultBucketUsecase, ObjectInfo, ObjectOptions};'
|
||||
not_flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::tagging::helper();'
|
||||
not_flagged rustfs/src/app "$RUNTIME_IMPORT_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::{ECStore, DefaultBucketUsecase};'
|
||||
not_flagged rustfs/src/app "$RUNTIME_PATH_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::get_server_info();'
|
||||
not_flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs ' use super::{BucketMetadata};'
|
||||
|
||||
# A file outside the module directory, or a missing -- separator, is an error, not a silent pass.
|
||||
new_case
|
||||
status=0
|
||||
module_relative_rg rustfs/src/app 'x' -n -- rustfs/src/storage/ecfs.rs 2>/dev/null || status=$?
|
||||
[[ "$status" == 2 ]] || fail "file outside the module dir: status $status"
|
||||
status=0
|
||||
module_relative_rg rustfs/src/app 'x' -n rustfs/src/app/a.rs 2>/dev/null || status=$?
|
||||
[[ "$status" == 2 ]] || fail "missing --: status $status"
|
||||
status=0
|
||||
(cd "$FX" && module_relative_rg rustfs/src/app 'x' -n -- rustfs/src/app/missing.rs) 2>/dev/null || status=$?
|
||||
[[ "$status" == 2 ]] || fail "unreadable file: status $status"
|
||||
ok
|
||||
|
||||
echo "guard_source_set contract tests: $passed passed, $skipped skipped"
|
||||
Executable
+157
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Exercise the reference-binary pin resolver and unpacker without network access."""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / ".github/actions/reference-binary/reference_binary.py"
|
||||
PIN = '''schema = 1
|
||||
repository = "rustfs/rustfs"
|
||||
version = "1.0.5"
|
||||
tag = "1.0.5"
|
||||
|
||||
[platforms.linux-x86_64-gnu]
|
||||
asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
|
||||
sha256 = "{sha}"
|
||||
executable = "rustfs"
|
||||
'''
|
||||
SHA = "a" * 64
|
||||
|
||||
|
||||
def run(*args):
|
||||
proc = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, text=True, check=False)
|
||||
return proc.returncode, proc.stdout, proc.stderr
|
||||
|
||||
|
||||
class Base(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(temp.cleanup)
|
||||
self.root = Path(temp.name)
|
||||
|
||||
def pin(self, text):
|
||||
path = self.root / "pin.toml"
|
||||
path.write_text(text)
|
||||
return path
|
||||
|
||||
def archive(self, members):
|
||||
path = self.root / "rustfs.zip"
|
||||
with zipfile.ZipFile(path, "w") as bundle:
|
||||
for name, data in members.items():
|
||||
bundle.writestr(name, data)
|
||||
return path, hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
class ResolveTests(Base):
|
||||
def test_missing_pin_file_fails(self):
|
||||
code, _, err = run("resolve", "--pin", str(self.root / "absent.toml"), "--platform", "linux-x86_64-gnu")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("reference pin file is missing", err)
|
||||
|
||||
def test_unpinned_platform_fails(self):
|
||||
code, _, err = run("resolve", "--pin", str(self.pin(PIN.format(sha=SHA))), "--platform", "linux-aarch64-gnu")
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("no pin for platform", err)
|
||||
|
||||
def test_malformed_pins_fail(self):
|
||||
good = PIN.format(sha=SHA)
|
||||
for label, text in {
|
||||
"short digest": good.replace(SHA, "a" * 63),
|
||||
"uppercase digest": good.replace(SHA, "A" * 64),
|
||||
"no digest": good.replace(f'sha256 = "{SHA}"\n', ""),
|
||||
"path in asset": good.replace('asset = "rustfs-', 'asset = "../rustfs-'),
|
||||
"non-zip asset": good.replace("v1.0.5.zip", "v1.0.5.deb"),
|
||||
"asset of another version": good.replace("v1.0.5.zip", "v1.0.4.zip"),
|
||||
"path in executable": good.replace('executable = "rustfs"', 'executable = "bin/rustfs"'),
|
||||
"wrong schema": good.replace("schema = 1", "schema = 2"),
|
||||
"boolean schema": good.replace("schema = 1", "schema = true"),
|
||||
"no repository": good.replace('repository = "rustfs/rustfs"\n', ""),
|
||||
"unknown key": good + 'url = "https://example.invalid"\n',
|
||||
"not toml": "schema = \n",
|
||||
}.items():
|
||||
with self.subTest(label):
|
||||
code, _, _ = run("resolve", "--pin", str(self.pin(text)), "--platform", "linux-x86_64-gnu")
|
||||
self.assertEqual(code, 1)
|
||||
|
||||
def test_resolves_identity_as_step_outputs(self):
|
||||
code, out, err = run("resolve", "--pin", str(self.pin(PIN.format(sha=SHA))), "--platform", "linux-x86_64-gnu")
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertEqual(out.splitlines(), ["repository=rustfs/rustfs", "version=1.0.5", "tag=1.0.5",
|
||||
"asset=rustfs-linux-x86_64-gnu-v1.0.5.zip", f"sha256={SHA}",
|
||||
"executable=rustfs"])
|
||||
|
||||
def test_repository_pin_is_valid(self):
|
||||
code, out, err = run("resolve", "--pin", str(ROOT / ".config/reference-binary.toml"),
|
||||
"--platform", "linux-x86_64-gnu")
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertIn("version=1.0.5", out.splitlines())
|
||||
|
||||
|
||||
class UnpackTests(Base):
|
||||
def unpack(self, archive, sha):
|
||||
return run("unpack", "--archive", str(archive), "--sha256", sha, "--executable", "rustfs",
|
||||
"--destination", str(self.root / "out"))
|
||||
|
||||
def test_digest_mismatch_fails_and_extracts_nothing(self):
|
||||
archive, _ = self.archive({"rustfs": b"binary"})
|
||||
code, _, err = self.unpack(archive, SHA)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("does not match the pinned digest", err)
|
||||
self.assertFalse((self.root / "out").exists())
|
||||
|
||||
def test_missing_archive_fails(self):
|
||||
code, _, err = self.unpack(self.root / "absent.zip", SHA)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("missing or unreadable", err)
|
||||
|
||||
def test_archive_without_top_level_executable_fails(self):
|
||||
archive, sha = self.archive({"bin/rustfs": b"binary"})
|
||||
code, _, err = self.unpack(archive, sha)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("exactly one top-level", err)
|
||||
|
||||
def test_empty_executable_fails(self):
|
||||
archive, sha = self.archive({"rustfs": b""})
|
||||
code, _, _ = self.unpack(archive, sha)
|
||||
self.assertEqual(code, 1)
|
||||
|
||||
def test_non_zip_archive_fails(self):
|
||||
archive = self.root / "rustfs.zip"
|
||||
archive.write_bytes(b"not a zip")
|
||||
code, _, err = self.unpack(archive, hashlib.sha256(b"not a zip").hexdigest())
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("not a valid zip", err)
|
||||
|
||||
def test_extracts_only_the_executable(self):
|
||||
archive, sha = self.archive({"rustfs": b"binary", "../escape": b"x", "README": b"r"})
|
||||
code, out, err = self.unpack(archive, sha)
|
||||
self.assertEqual(code, 0, err)
|
||||
target = self.root / "out" / "rustfs"
|
||||
self.assertEqual(out.strip(), f"path={target}")
|
||||
self.assertEqual(target.read_bytes(), b"binary")
|
||||
self.assertTrue(target.stat().st_mode & 0o111)
|
||||
self.assertEqual(sorted(p.name for p in (self.root / "out").iterdir()), ["rustfs"])
|
||||
self.assertFalse((self.root / "escape").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Executable
+223
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end probe for the guards that must survive the mechanical split of
|
||||
# ecfs.rs, bucket_usecase.rs, multipart_usecase.rs and
|
||||
# lifecycle_transition_api_test.rs (rustfs/backlog#2765).
|
||||
#
|
||||
# It copies the working tree, simulates the split (moves real code into
|
||||
# storage/legacy_edge/, app/bucket_usecase/, app/multipart_usecase/ and
|
||||
# app/lifecycle_transition_api_test/), plants one forbidden line per rule in a
|
||||
# split file, and runs the real guard scripts there. Each planted line must be
|
||||
# reported with its split-file path, and nothing in the files that hold only
|
||||
# moved real code or ordinary depth-relative imports may be reported: that is
|
||||
# the "moved and broken -> detected" and "moved -> still passes" proof in one
|
||||
# run per guard. The real workspace is never modified.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${RUSTFS_PYTHON_BIN:-python3}"
|
||||
FX="$(mktemp -d)"
|
||||
trap 'rm -rf "$FX"' EXIT
|
||||
|
||||
failures=0
|
||||
checked=0
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1" >&2
|
||||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
# expect_reported <output-file> <path> <text> <description>
|
||||
expect_reported() {
|
||||
checked=$((checked + 1))
|
||||
if ! grep -F -- "$2" "$1" | grep -qF -- "$3"; then
|
||||
fail "$4: no report naming $2 with: $3"
|
||||
fi
|
||||
}
|
||||
|
||||
# expect_silent <output-file> <path> <description>
|
||||
expect_silent() {
|
||||
checked=$((checked + 1))
|
||||
if grep -qF -- "$2" "$1"; then
|
||||
fail "$3: $2 was reported: $(grep -F -- "$2" "$1" | head -3)"
|
||||
fi
|
||||
}
|
||||
|
||||
# Copy the tracked and untracked-but-not-ignored files of the working tree.
|
||||
(cd "$ROOT_DIR" && git ls-files -z --cached --others --exclude-standard) |
|
||||
"$PYTHON" -c '
|
||||
import os, shutil, sys
|
||||
root, dest = sys.argv[1], sys.argv[2]
|
||||
for rel in sys.stdin.buffer.read().decode().split("\0"):
|
||||
if not rel:
|
||||
continue
|
||||
src = os.path.join(root, rel)
|
||||
if not os.path.lexists(src):
|
||||
continue
|
||||
out = os.path.join(dest, rel)
|
||||
os.makedirs(os.path.dirname(out), exist_ok=True)
|
||||
shutil.copy2(src, out, follow_symlinks=False)
|
||||
' "$ROOT_DIR" "$FX"
|
||||
|
||||
APP="$FX/rustfs/src/app"
|
||||
STORAGE="$FX/rustfs/src/storage"
|
||||
mkdir -p "$APP/bucket_usecase" "$APP/multipart_usecase" "$APP/lifecycle_transition_api_test" \
|
||||
"$STORAGE/legacy_edge/tests"
|
||||
|
||||
# Real code moves: the TRACE hot span leaves bucket_usecase.rs with its
|
||||
# attributes, and the real import blocks of ecfs.rs and the lifecycle
|
||||
# transition tests are copied into split files.
|
||||
"$PYTHON" - "$FX" <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
fx = Path(sys.argv[1])
|
||||
bu = fx / "rustfs/src/app/bucket_usecase.rs"
|
||||
text = bu.read_text()
|
||||
marker = " pub async fn execute_list_objects_v2("
|
||||
assert text.count(marker) == 1, "execute_list_objects_v2 must be defined once"
|
||||
idx = text.index(marker)
|
||||
start = text.rindex("\n", 0, idx) + 1
|
||||
while True:
|
||||
prev = text.rindex("\n", 0, start - 1) + 1
|
||||
if text[prev:start].lstrip().startswith(("#[", "///")):
|
||||
start = prev
|
||||
else:
|
||||
break
|
||||
end = text.index("\n }\n", idx) + len("\n }\n")
|
||||
(fx / "rustfs/src/app/bucket_usecase/listing.rs").write_text(
|
||||
"impl DefaultBucketUsecase {\n" + text[start:end] + "}\n")
|
||||
bu.write_text(text[:start] + text[end:])
|
||||
|
||||
def imports(path):
|
||||
return "".join(l + "\n" for l in path.read_text().splitlines() if l.startswith("use crate::"))
|
||||
|
||||
(fx / "rustfs/src/storage/legacy_edge/edge_imports.rs").write_text(
|
||||
imports(fx / "rustfs/src/storage/ecfs.rs"))
|
||||
(fx / "rustfs/src/app/lifecycle_transition_api_test/imports.rs").write_text(
|
||||
imports(fx / "rustfs/src/app/lifecycle_transition_api_test.rs"))
|
||||
PY
|
||||
|
||||
# Ordinary depth-relative uses of the parent module in split files: none of
|
||||
# them reaches crate::app or the storage root, so no rule may report them.
|
||||
cat >"$APP/bucket_usecase/ordinary.rs" <<'EOF'
|
||||
use super::{DefaultBucketUsecase, ObjectInfo, ObjectOptions};
|
||||
use super::ECStore;
|
||||
use super::StorageError;
|
||||
fn ordinary() {
|
||||
super::tagging::helper();
|
||||
super::get_server_info();
|
||||
}
|
||||
EOF
|
||||
cp "$APP/bucket_usecase/ordinary.rs" "$APP/multipart_usecase/ordinary.rs"
|
||||
cp "$APP/bucket_usecase/ordinary.rs" "$APP/lifecycle_transition_api_test/ordinary.rs"
|
||||
cat >"$STORAGE/legacy_edge/tests/ordinary.rs" <<'EOF'
|
||||
mod tests {
|
||||
use super::{BucketMetadata, check_preconditions};
|
||||
}
|
||||
EOF
|
||||
|
||||
# One forbidden line per rule, each in a split file.
|
||||
cat >"$STORAGE/legacy_edge/bucket_configs.rs" <<'EOF'
|
||||
use crate::app::context::resolve_region;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::storage::core_storage_compat::Probe;
|
||||
use crate::startup_server::LayerProbe;
|
||||
EOF
|
||||
cat >"$APP/bucket_usecase/cors.rs" <<'EOF'
|
||||
use crate::storage::*;
|
||||
use crate::storage::access::ReqInfo;
|
||||
use super::super::StorageError;
|
||||
use super::super::ECStore;
|
||||
fn forbidden() {
|
||||
super::super::get_server_info();
|
||||
}
|
||||
EOF
|
||||
cat >"$APP/multipart_usecase/complete.rs" <<'EOF'
|
||||
use crate::storage::s3_api::multipart;
|
||||
use super::super::{quota, versioning_sys};
|
||||
use crate::app::usecase_storage_compat::Probe;
|
||||
EOF
|
||||
cat >"$APP/lifecycle_transition_api_test/put.rs" <<'EOF'
|
||||
use crate::storage::request_context::RequestContext;
|
||||
fn forbidden() {
|
||||
super::super::tagging::probe();
|
||||
}
|
||||
EOF
|
||||
cat >"$STORAGE/legacy_edge/tests/object_meta.rs" <<'EOF'
|
||||
use super::super::super::{BucketMetadata};
|
||||
mod tests {
|
||||
use super::super::super::super::{check_preconditions};
|
||||
}
|
||||
use crate::storage::core_storage_compat::TestProbe;
|
||||
EOF
|
||||
|
||||
# --- architecture rules ------------------------------------------------------
|
||||
ARCH_OUT="$FX/.arch.out"
|
||||
if (cd "$FX" && ./scripts/check_architecture_migration_rules.sh) >"$ARCH_OUT" 2>&1; then
|
||||
fail "architecture rules passed over planted violations"
|
||||
fi
|
||||
E=rustfs/src/storage/legacy_edge/bucket_configs.rs
|
||||
B=rustfs/src/app/bucket_usecase/cors.rs
|
||||
M=rustfs/src/app/multipart_usecase/complete.rs
|
||||
L=rustfs/src/app/lifecycle_transition_api_test/put.rs
|
||||
T=rustfs/src/storage/legacy_edge/tests/object_meta.rs
|
||||
expect_reported "$ARCH_OUT" "$E:1:" "storage runtime source reads must stay behind" "runtime-source bypass (ECFS set)"
|
||||
expect_reported "$ARCH_OUT" "$E:2:" "ECFS S3 routes must construct app usecases" "ECFS usecase bypass"
|
||||
expect_reported "$ARCH_OUT" "$E:3:" "must use relative core_storage_compat paths" "core_storage_compat (ECFS set)"
|
||||
expect_reported "$ARCH_OUT" "$B:1:" "instead of crate::storage::*" "storage wildcard (bucket split)"
|
||||
expect_reported "$ARCH_OUT" "$B:2:" "consume storage helper APIs through" "storage helper import (bucket split)"
|
||||
expect_reported "$ARCH_OUT" "$B:3:" "consume storage helper APIs through" "depth-aware super::super storage helper"
|
||||
expect_reported "$ARCH_OUT" "$B:4:" "runtime and data-usage facades must stay behind" "depth-aware super::super runtime facade import"
|
||||
expect_reported "$ARCH_OUT" "$B:6:" "runtime and data-usage facades must stay behind" "depth-aware super::super runtime facade path"
|
||||
expect_reported "$ARCH_OUT" "$M:1:" "consume S3 API helpers through" "s3_api bypass (multipart split)"
|
||||
expect_reported "$ARCH_OUT" "$M:2:" "consume storage helper APIs through" "depth-aware super::super owner import"
|
||||
expect_reported "$ARCH_OUT" "$M:3:" "local compatibility consumers must use relative" "usecase_storage_compat (multipart split)"
|
||||
expect_reported "$ARCH_OUT" "$L:1:" "completed storage helper consumers" "test storage helper (lifecycle split)"
|
||||
expect_reported "$ARCH_OUT" "$L:3:" "consume storage helper APIs through" "depth-aware super::super owner path"
|
||||
expect_reported "$ARCH_OUT" "$T:1:" "storage-owner tests must use storage_api test consumer" "ecfs_test import at file depth"
|
||||
expect_reported "$ARCH_OUT" "$T:3:" "storage-owner tests must use storage_api test consumer" "ecfs_test import in an inline tests module"
|
||||
expect_reported "$ARCH_OUT" "$T:5:" "local compatibility consumers must use relative" "ecfs_test core_storage_compat"
|
||||
for silent in rustfs/src/app/bucket_usecase/ordinary.rs rustfs/src/app/multipart_usecase/ordinary.rs \
|
||||
rustfs/src/app/lifecycle_transition_api_test/ordinary.rs rustfs/src/storage/legacy_edge/tests/ordinary.rs \
|
||||
rustfs/src/app/bucket_usecase/listing.rs rustfs/src/storage/legacy_edge/edge_imports.rs \
|
||||
rustfs/src/app/lifecycle_transition_api_test/imports.rs; do
|
||||
expect_silent "$ARCH_OUT" "$silent" "architecture rules over moved or ordinary code"
|
||||
done
|
||||
|
||||
# --- layer dependencies ------------------------------------------------------
|
||||
LAYER_OUT="$FX/.layer.out"
|
||||
if (cd "$FX" && ./scripts/check_layer_dependencies.sh) >"$LAYER_OUT" 2>&1; then
|
||||
fail "layer guard passed over a planted reverse dependency"
|
||||
fi
|
||||
expect_reported "$LAYER_OUT" "dep|$E|interface->composition|" "crate::startup_server::LayerProbe" "legacy_edge keeps the interface layer"
|
||||
expect_silent "$LAYER_OUT" "dep|rustfs/src/storage/legacy_edge/edge_imports.rs|" "moved ecfs imports stay legal"
|
||||
expect_silent "$LAYER_OUT" "dep|rustfs/src/app/lifecycle_transition_api_test/" "moved lifecycle transition tests stay excluded"
|
||||
expect_silent "$LAYER_OUT" "dep|rustfs/src/app/bucket_usecase/listing.rs|" "moved bucket usecase code keeps the app layer"
|
||||
|
||||
# --- logging hot span --------------------------------------------------------
|
||||
LOG_OUT="$FX/.logging.out"
|
||||
checked=$((checked + 1))
|
||||
if ! (cd "$FX" && ./scripts/check_logging_guardrails.sh) >"$LOG_OUT" 2>&1; then
|
||||
fail "logging guardrails failed after a clean move of the TRACE hot span: $(tail -3 "$LOG_OUT")"
|
||||
fi
|
||||
"$PYTHON" - "$APP/bucket_usecase/listing.rs" <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
path = Path(sys.argv[1])
|
||||
text = path.read_text()
|
||||
marker = '#[instrument(level = "trace"'
|
||||
assert text.count(marker) == 1, "the moved hot span must carry one TRACE attribute"
|
||||
path.write_text(text.replace(marker, '#[instrument(level = "debug"'))
|
||||
PY
|
||||
checked=$((checked + 1))
|
||||
if (cd "$FX" && ./scripts/check_logging_guardrails.sh) >"$LOG_OUT" 2>&1; then
|
||||
fail "logging guardrails passed after the moved hot span lost TRACE"
|
||||
elif ! grep -qF "execute_list_objects_v2 must remain TRACE-only" "$LOG_OUT"; then
|
||||
fail "logging guardrails failed for another reason: $(tail -3 "$LOG_OUT")"
|
||||
fi
|
||||
|
||||
if [[ $failures -gt 0 ]]; then
|
||||
echo "split guard rules probe: $failures of $checked checks failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "split guard rules probe: $checked checks passed"
|
||||
Reference in New Issue
Block a user