mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-11 12:43:54 +00:00
Compare commits
103 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| abfe23c02d | |||
| 4d3b8722ab | |||
| 22b0c42a2d | |||
| 9156ba24f4 | |||
| 325a181922 | |||
| 4836f7656a | |||
| ca3128cfd3 | |||
| 8d04f1d2fc | |||
| 977060e457 | |||
| be4ef4bca0 | |||
| 1455f5f4a5 | |||
| 4c7cc8a943 | |||
| 8172df44ec | |||
| 55d931353f | |||
| 7b3701ddc6 | |||
| fc09148f08 | |||
| 2df55c6dea | |||
| c9e1464a2a | |||
| 0e3028d29e | |||
| 365014a385 | |||
| 443e8afee1 | |||
| 782e3e5209 | |||
| cfffb1700e | |||
| cd62f3ba4f | |||
| 7bc419e368 | |||
| 7269243de7 | |||
| c4a7423f99 | |||
| 2db4ed244a | |||
| 93a61df1cf | |||
| feec0e9e4a | |||
| 38baeeb11d | |||
| 1af039c972 | |||
| a2fc7c13b4 | |||
| dbe01e5cb1 | |||
| ffc27fa6db | |||
| 7e8edab0d3 | |||
| 22057ad4c5 | |||
| 279a88ab56 | |||
| b4abe64ada | |||
| add363f084 | |||
| 1f2dbe36e5 | |||
| 67fbafa13e | |||
| 1000860cb8 | |||
| b783f404ad | |||
| 11a991c777 | |||
| 3f2b6f690a | |||
| 88b626e7a0 | |||
| 8cf6187ff4 | |||
| a0a0cee9eb | |||
| 80fff4b5c5 | |||
| d91dfcedd2 | |||
| 79c4fedf8c | |||
| 73c846e09c | |||
| 6938c467d0 | |||
| 5323df7049 | |||
| 55b6fbd260 | |||
| dd677cc121 | |||
| 0c2f327974 | |||
| df42cf1ce8 | |||
| d290d46b34 | |||
| 0d297ed8cd | |||
| b8dffa2f1d | |||
| 62b6f5f3fc | |||
| be0986115c | |||
| 1bd91c8dad | |||
| dcb22bd4d5 | |||
| fa17718444 | |||
| 99abffc57a | |||
| fc061c7930 | |||
| d86bf1360f | |||
| f6097d1df2 | |||
| 7e5d32eecf | |||
| 72da0e21c4 | |||
| 78a0cf321c | |||
| cbe38a120f | |||
| db0828aa1c | |||
| 3ca889b099 | |||
| e6112c526f | |||
| c5967686f1 | |||
| 859de565ad | |||
| 1f7cc593e7 | |||
| 1d5a50f094 | |||
| 53245be9f4 | |||
| c47432e353 | |||
| f622c1bce0 | |||
| a99d8f4f89 | |||
| 4d15b81623 | |||
| 836d30192f | |||
| 860c61ee6d | |||
| f600af4704 | |||
| 9e747e8475 | |||
| 96fc067765 | |||
| e809b4899e | |||
| 584f03f300 | |||
| d52d6fade7 | |||
| 3623d6db2d | |||
| 484f1254ca | |||
| c9c16efbb6 | |||
| f2a59073ca | |||
| c1af94d1d1 | |||
| f84bbfe28f | |||
| 378a2ff7f8 | |||
| b88b53aa6a |
@@ -75,11 +75,6 @@ test-count-ratchet-check: ## Check no workspace crate fell below its test floor
|
||||
@echo "🧪 Checking per-crate test-count ratchet..."
|
||||
./scripts/check_test_count_ratchet.sh
|
||||
|
||||
.PHONY: test-seams-check
|
||||
test-seams-check: ## Check tests outside an operation's owner call its use-case only through rustfs/src/app/test_seams
|
||||
@echo "🧪 Checking cross-owner test seams..."
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
|
||||
@echo "📣 Checking cryptographic capability wording guard..."
|
||||
|
||||
@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check test-seams-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check test-seams-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
|
||||
@@ -62,10 +62,11 @@ script-tests: ## Run shell script tests
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
./scripts/check_s3s_footprint.sh --self-test
|
||||
./scripts/test_check_test_count_ratchet.sh
|
||||
./scripts/test_guard_source_set.sh
|
||||
./scripts/test_split_guard_rules.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_lifecycle_transition_api_test.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_ecfs_legacy_edge.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_reference_binary.py
|
||||
@@ -95,6 +96,7 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_api_coverage.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
|
||||
@@ -25,7 +25,6 @@ rustfs-data-usage 58 0
|
||||
rustfs-ecstore 7858 2
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 361 0
|
||||
rustfs-gateway-adapter 74 0
|
||||
rustfs-heal 1128 3
|
||||
rustfs-heal-contracts 8 0
|
||||
rustfs-iam 415 0
|
||||
|
||||
@@ -101,10 +101,6 @@ runs:
|
||||
shell: bash
|
||||
run: ./scripts/check_test_count_ratchet.sh
|
||||
|
||||
- name: Check cross-owner tests use test seams
|
||||
shell: bash
|
||||
run: python3 ./scripts/check_test_seams.py
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
shell: bash
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
@@ -1021,26 +1021,6 @@
|
||||
"policy_reason": "Preserve operator-disabled state; enabling requires a separate policy decision.",
|
||||
"origin": "baseline"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/manual-main-packages.yml",
|
||||
"name": "Manual main packages",
|
||||
"roles": [
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "event",
|
||||
"timezone": "UTC",
|
||||
"schedule": [],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "none",
|
||||
"result_owner": "manual-main-packages.yml",
|
||||
"disposition": "Packages the origin/main tip as workflow artifacts only. No GitHub Release or download-channel writes.",
|
||||
"monitor_reason": "Operator-requested packaging has no periodic freshness obligation.",
|
||||
"origin": "manual-main-packages"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/minio-interop.yml",
|
||||
"name": "minio-interop",
|
||||
|
||||
@@ -1,415 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Manual packaging of the current origin/main tip. Workflow artifacts only.
|
||||
# Dispatch uses this file from the selected branch. The build fetches
|
||||
# refs/heads/main and checks out that commit, not the dispatch ref or a tag.
|
||||
|
||||
name: Manual main packages
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
architecture:
|
||||
description: "Linux targets: x86 (x86_64 musl+gnu), arm (aarch64 musl+gnu), or all"
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- x86
|
||||
- arm
|
||||
- all
|
||||
default: all
|
||||
version:
|
||||
description: "Binary and package version: MAJOR.MINOR.PATCH, or MAJOR.MINOR.PATCH-(alpha|beta|rc).N. No leading v."
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
CARGO_INCREMENTAL: 0
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: Resolve main tip and version
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
source_sha: ${{ steps.pin.outputs.source_sha }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
toolchain: ${{ steps.inputs.outputs.toolchain }}
|
||||
console: ${{ steps.inputs.outputs.console }}
|
||||
matrix: ${{ steps.matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: main
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Pin the tip of origin/main
|
||||
id: pin
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
|
||||
sha=$(git rev-parse FETCH_HEAD)
|
||||
git checkout --detach "$sha"
|
||||
test "$(git rev-parse HEAD)" = "$sha"
|
||||
printf 'source_sha=%s\n' "$sha" >> "$GITHUB_OUTPUT"
|
||||
printf 'Pinned origin/main %s\n' "$sha"
|
||||
|
||||
- name: Validate package version
|
||||
id: version
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
bash scripts/release/package_manual_artifacts.sh validate \
|
||||
--version "$PACKAGE_VERSION" \
|
||||
--arch x86_64 \
|
||||
--libc gnu | tee "$RUNNER_TEMP/package-version.txt"
|
||||
printf 'version=%s\n' "$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Install stable Rust
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
with:
|
||||
toolchain: stable
|
||||
|
||||
- name: Resolve console assets and toolchain
|
||||
id: inputs
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
toolchain=$(rustc --version | awk '{print $2}')
|
||||
[[ "$toolchain" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
|
||||
gh api repos/rustfs/console/releases/latest > console-release.json
|
||||
TOOLCHAIN="$toolchain" python3 - <<'PY'
|
||||
import json, os, sys
|
||||
sys.path.insert(0, "scripts/release")
|
||||
import candidate
|
||||
console = candidate.console_asset(candidate.read_json("console-release.json"))
|
||||
candidate.emit_output("console", json.dumps(console, separators=(",", ":")))
|
||||
candidate.emit_output("toolchain", os.environ["TOOLCHAIN"])
|
||||
print(f"toolchain={os.environ['TOOLCHAIN']}")
|
||||
print(f"console={console['tag']} {console['name']}")
|
||||
PY
|
||||
|
||||
- name: Select Linux targets
|
||||
id: matrix
|
||||
shell: bash
|
||||
env:
|
||||
ARCHITECTURE: ${{ inputs.architecture }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
matrix=$(bash scripts/release/package_manual_artifacts.sh matrix "$ARCHITECTURE")
|
||||
count=$(python3 -c 'import json,sys; print(len(json.loads(sys.argv[1])["include"]))' "$matrix")
|
||||
[[ "$count" -gt 0 ]]
|
||||
printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT"
|
||||
printf 'Selected %s target(s) for %s\n' "$count" "$ARCHITECTURE"
|
||||
|
||||
package:
|
||||
name: Package RustFS (${{ matrix.target_id }})
|
||||
needs: prepare
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# x86_64 uses sm-standard-4, the official release compile pool: the workspace
|
||||
# release build needs that disk (placement rule: disk) and reads build.yml's
|
||||
# cache without writing it. aarch64 uses ubicloud-standard-16-arm so the
|
||||
# packaged binary and console smoke execute natively (placement rule: hardware).
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 210
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTUP_TOOLCHAIN: ${{ needs.prepare.outputs.toolchain }}
|
||||
RUSTFLAGS: "${{ matrix.rustflags }}"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Checkout the pinned origin/main commit
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ needs.prepare.outputs.source_sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Confirm the checkout is that main commit
|
||||
shell: bash
|
||||
env:
|
||||
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
|
||||
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
|
||||
git merge-base --is-ancestor "$SOURCE_SHA" FETCH_HEAD
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: ${{ needs.prepare.outputs.toolchain }}
|
||||
target: ${{ matrix.target }}
|
||||
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}${{ matrix.os == 'ubicloud-standard-16-arm' && '-ubicloud-arm' || '' }}
|
||||
# build.yml owns this key. This job only restores it.
|
||||
cache-save-if: 'false'
|
||||
install-cross-tools: ${{ matrix.cross }}
|
||||
install-build-packaging-tools: 'true'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Download static console assets
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CONSOLE_IDENTITY: ${{ needs.prepare.outputs.console }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PYCONSOLE'
|
||||
import json, os, pathlib, subprocess, zipfile
|
||||
value = json.loads(os.environ["CONSOLE_IDENTITY"])
|
||||
subprocess.run(["python3", ".github/actions/setup/download_release_asset.py",
|
||||
"--repository", "rustfs/console", "--tag", value["tag"], "--asset", value["name"],
|
||||
"--sha256", value["sha256"], "--output", "console.zip"], check=True)
|
||||
destination = pathlib.Path("rustfs/static")
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile("console.zip") as archive:
|
||||
for member in archive.infolist():
|
||||
path = pathlib.PurePosixPath(member.filename)
|
||||
if path.is_absolute() or ".." in path.parts or "\\" in member.filename:
|
||||
raise SystemExit("unsafe Console archive member")
|
||||
archive.extractall(destination)
|
||||
if not (destination / "index.html").is_file():
|
||||
raise SystemExit("Console archive has no index.html")
|
||||
PYCONSOLE
|
||||
rm -f console.zip
|
||||
|
||||
- name: Build RustFS
|
||||
shell: bash
|
||||
env:
|
||||
RUSTFS_BUILD_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_OS: ${{ matrix.build_os }}
|
||||
BUILD_FEATURES: ${{ matrix.features }}
|
||||
BUILD_GLIBC: ${{ matrix.glibc }}
|
||||
BUILD_CROSS: ${{ matrix.cross }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
touch rustfs/build.rs
|
||||
|
||||
feature_args=()
|
||||
if [[ -n "$BUILD_FEATURES" ]]; then
|
||||
feature_args=(--features "$BUILD_FEATURES")
|
||||
fi
|
||||
|
||||
# gnu targets pin glibc 2.31 through cargo-zigbuild (<target>.<glibc>).
|
||||
target_arg="$BUILD_TARGET"
|
||||
if [[ -n "$BUILD_GLIBC" ]]; then
|
||||
target_arg="${target_arg}.${BUILD_GLIBC}"
|
||||
fi
|
||||
|
||||
profile_args=()
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
|
||||
profile_args=(--config 'profile.release.strip="none"')
|
||||
fi
|
||||
|
||||
case "$BUILD_CROSS" in
|
||||
true)
|
||||
cargo zigbuild --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
|
||||
;;
|
||||
false)
|
||||
cargo build --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
|
||||
;;
|
||||
*)
|
||||
printf 'unexpected cross flag: %q\n' "$BUILD_CROSS" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
bin="target/${BUILD_TARGET}/release/rustfs"
|
||||
version_output=$("$bin" --version)
|
||||
first=${version_output%%$'\n'*}
|
||||
if [[ "$first" != "rustfs ${RUSTFS_BUILD_VERSION}" ]]; then
|
||||
printf 'binary version line is %q, expected %q\n' "$first" "rustfs ${RUSTFS_BUILD_VERSION}" >&2
|
||||
printf '%s\n' "$version_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
|
||||
! grep -Fxq "build os : ${BUILD_OS}" <<< "$version_output" || \
|
||||
! grep -Fxq "rust channel : stable-${BUILD_TARGET}" <<< "$version_output"; then
|
||||
printf 'release binary reports an unexpected compile target\n' >&2
|
||||
printf '%s\n' "$version_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$BUILD_GLIBC" ]]; then
|
||||
python3 scripts/release/check_glibc.py "$BUILD_GLIBC" "$bin"
|
||||
fi
|
||||
|
||||
- name: Generate the CPU symbol catalog
|
||||
if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
|
||||
shell: bash
|
||||
env:
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo metadata --locked --format-version 1 --filter-platform "$BUILD_TARGET" \
|
||||
--features rustfs/pyroscope > "$RUNNER_TEMP/cpu-catalog-metadata.json"
|
||||
features=$(jq -ce '
|
||||
(.packages[] | select(.name == "rustfs") | .id) as $id |
|
||||
.resolve.nodes[] | select(.id == $id) | .features | map(select(. != "default")) | sort
|
||||
' "$RUNNER_TEMP/cpu-catalog-metadata.json")
|
||||
release_dir="target/${BUILD_TARGET}/release"
|
||||
rm -f "$release_dir/rustfs.cpu-symbol-catalog.json" "$release_dir/rustfs.cpu-symbol-catalog.sha256"
|
||||
cargo run --locked -p rustfs-connect-symbol-catalog -- \
|
||||
"$release_dir/rustfs" "$(git rev-parse HEAD)" "$BUILD_TARGET" "$features" \
|
||||
"$release_dir/rustfs.cpu-symbol-catalog.json" > "$release_dir/rustfs.cpu-symbol-catalog.sha256"
|
||||
|
||||
- name: Install DEB and RPM tools
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ruby ruby-dev build-essential rpm fakeroot cpio
|
||||
sudo gem install fpm --version 1.17.0 --no-document
|
||||
|
||||
- name: Package zip, deb, and rpm
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_FEATURES: ${{ matrix.features }}
|
||||
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
binary="target/${BUILD_TARGET}/release/rustfs"
|
||||
extra=()
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
|
||||
extra=(
|
||||
--zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.json"
|
||||
--zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.sha256"
|
||||
)
|
||||
fi
|
||||
bash scripts/release/package_manual_artifacts.sh package \
|
||||
--version "$PACKAGE_VERSION" \
|
||||
--arch "$ARCH" \
|
||||
--libc "$LIBC" \
|
||||
--binary "$binary" \
|
||||
--output dist \
|
||||
"${extra[@]}"
|
||||
printf 'Packaged %s from origin/main %s\n' "$PACKAGE_VERSION" "$SOURCE_SHA"
|
||||
|
||||
- name: Verify the packaged binary
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_OS: ${{ matrix.build_os }}
|
||||
BUILD_FEATURES: ${{ matrix.features }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
stem="rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}"
|
||||
package_file="dist/${stem}.zip"
|
||||
[[ -f "$package_file" && -f "dist/${stem}.deb" && -f "dist/${stem}.rpm" ]]
|
||||
[[ "$(find dist -maxdepth 1 -type f | wc -l)" -eq 3 ]]
|
||||
archive_members=$(unzip -Z1 "$package_file" | sort)
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
|
||||
expected_members=$(printf '%s\n' rustfs rustfs.cpu-symbol-catalog.json rustfs.cpu-symbol-catalog.sha256 | sort)
|
||||
else
|
||||
expected_members=$(printf '%s\n' rustfs)
|
||||
fi
|
||||
if [[ "$archive_members" != "$expected_members" ]]; then
|
||||
printf 'Linux package does not contain the expected files\n' >&2
|
||||
printf '%s\n' "$archive_members" >&2
|
||||
exit 1
|
||||
fi
|
||||
binary_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$binary_dir"' EXIT
|
||||
unzip -q "$package_file" rustfs -d "$binary_dir"
|
||||
version_output=$("$binary_dir/rustfs" --version)
|
||||
first=${version_output%%$'\n'*}
|
||||
if [[ "$first" != "rustfs ${PACKAGE_VERSION}" ]]; then
|
||||
printf 'packaged binary version line is %q, expected %q\n' "$first" "rustfs ${PACKAGE_VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
|
||||
! grep -Fxq "build os : ${BUILD_OS}" <<< "$version_output"; then
|
||||
printf 'packaged binary reports an unexpected compile target\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Verify packaged console
|
||||
if: matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'aarch64-unknown-linux-gnu'
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
package_file="dist/rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}.zip"
|
||||
|
||||
find_free_port() {
|
||||
python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
PY
|
||||
}
|
||||
|
||||
package_dir=$(mktemp -d)
|
||||
data_dir=$(mktemp -d)
|
||||
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
|
||||
server_pid=""
|
||||
trap '
|
||||
if [[ -n "${server_pid:-}" ]]; then
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$package_dir" "$data_dir"
|
||||
' EXIT
|
||||
|
||||
unzip -q "$package_file" -d "$package_dir"
|
||||
api_port=$(find_free_port)
|
||||
console_port=$(find_free_port)
|
||||
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
|
||||
|
||||
"$package_dir/rustfs" server \
|
||||
--address "127.0.0.1:${api_port}" \
|
||||
--console-enable \
|
||||
--console-address "127.0.0.1:${console_port}" \
|
||||
--access-key console-smoke \
|
||||
--secret-key console-smoke-secret \
|
||||
"$data_dir" >"$log_file" 2>&1 &
|
||||
server_pid=$!
|
||||
|
||||
for _ in {1..40}; do
|
||||
response=$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)
|
||||
if [[ "$response" == 200\ text/html* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
|
||||
printf 'Console endpoint did not return 200 text/html: %s\n' "$response" >&2
|
||||
cat "$log_file"
|
||||
exit 1
|
||||
|
||||
- name: Upload packages
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: packages-${{ matrix.target_id }}-v${{ needs.prepare.outputs.version }}
|
||||
path: dist/*
|
||||
if-no-files-found: error
|
||||
compression-level: 0
|
||||
retention-days: 30
|
||||
@@ -112,11 +112,9 @@ jobs:
|
||||
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
|
||||
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
|
||||
# label was abandoned (no docker.sock on the pod its only run landed on).
|
||||
# sm-standard-2 has no docker.sock either (weekly runs failed at "Enable
|
||||
# buildx" from 2026-09-27 to 2026-11-08), so keep this job on ubuntu-latest.
|
||||
# TODO(ci-8): scheduled-failure alerting (auto-open issue on a red weekly
|
||||
# run) is added by the ci-8 composite action; do not implement it here.
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
|
||||
Generated
+2
-140
@@ -4698,12 +4698,6 @@ version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||
|
||||
[[package]]
|
||||
name = "futures-timer"
|
||||
version = "3.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.34"
|
||||
@@ -7920,34 +7914,10 @@ version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.14.0",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"phf_shared 0.14.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.12.1"
|
||||
@@ -10029,114 +9999,6 @@ dependencies = [
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway"
|
||||
version = "0.57.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"futures-timer",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"httparse",
|
||||
"hyper",
|
||||
"pin-project-lite",
|
||||
"rustfs-gateway-core",
|
||||
"rustfs-gateway-http",
|
||||
"rustfs-gateway-macros",
|
||||
"rustfs-gateway-sig",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-gateway-xml",
|
||||
"sha2 0.11.1",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-adapter"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"http 1.5.0",
|
||||
"rustfs-credentials",
|
||||
"rustfs-gateway",
|
||||
"rustfs-gateway-http",
|
||||
"rustfs-iam",
|
||||
"rustfs-policy",
|
||||
"serde_json",
|
||||
"time",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-core"
|
||||
version = "0.54.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http 1.5.0",
|
||||
"md-5 0.11.0",
|
||||
"percent-encoding",
|
||||
"rustfs-gateway-http",
|
||||
"rustfs-gateway-sig",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-gateway-xml",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-http"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"httparse",
|
||||
"memchr",
|
||||
"phf 0.14.0",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"sha2 0.11.1",
|
||||
"smallvec",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-macros"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-sig"
|
||||
version = "0.13.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"rustfs-gateway-http",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.1",
|
||||
"smallvec",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-stream"
|
||||
version = "0.2.0"
|
||||
@@ -12741,10 +12603,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -26,7 +26,6 @@ members = [
|
||||
"crates/ecstore", # Erasure coding storage implementation
|
||||
"crates/e2e_test", # End-to-end test suite
|
||||
"crates/filemeta", # File metadata management
|
||||
"crates/gateway-adapter", # RustFS implementations of the gateway extension traits
|
||||
"crates/heal", # Erasure set and object healing
|
||||
"crates/heal-contracts", # Heal request/response channel contracts
|
||||
"crates/iam", # Identity and Access Management
|
||||
@@ -94,7 +93,6 @@ redundant_clone = "warn"
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.5" }
|
||||
rustfs-gateway-adapter = { path = "crates/gateway-adapter", version = "1.0.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.5" }
|
||||
@@ -151,12 +149,6 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.57.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
|
||||
+4
-22
@@ -24,10 +24,9 @@
|
||||
#
|
||||
# Supports cross-compilation for amd64 and arm64 via TARGETPLATFORM.
|
||||
|
||||
# Do not declare TARGETPLATFORM, BUILDPLATFORM or TARGETARCH here: a global
|
||||
# declaration without a value shadows BuildKit's automatic values, so the
|
||||
# stage-level re-declarations below see empty strings. They are only
|
||||
# declared inside the stage that uses them.
|
||||
ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES=""
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
@@ -170,24 +169,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
# TARGETARCH is only provided by BuildKit. Fall back to TARGETPLATFORM (for
|
||||
# example set by docker-compose) and then to the build host, like the cargo
|
||||
# step above, so every path that builds this file passes a real architecture.
|
||||
RUN set -eu; \
|
||||
rc_arch="${TARGETARCH:-}"; \
|
||||
if [ -z "${rc_arch}" ]; then \
|
||||
case "${TARGETPLATFORM:-}" in \
|
||||
linux/amd64) rc_arch=amd64 ;; \
|
||||
linux/arm64) rc_arch=arm64 ;; \
|
||||
*) \
|
||||
case "$(uname -m)" in \
|
||||
x86_64) rc_arch=amd64 ;; \
|
||||
aarch64|arm64) rc_arch=arm64 ;; \
|
||||
*) echo "TARGETARCH is empty and TARGETPLATFORM='${TARGETPLATFORM:-}' is not linux/amd64 or linux/arm64; cannot choose an rc architecture on $(uname -m)" >&2; exit 1 ;; \
|
||||
esac ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
sh /usr/src/rustfs/scripts/download_rc.sh "${rc_arch}" musl "${RC_RELEASE}" /usr/local/bin/rc
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
# -----------------------------
|
||||
# Development stage (keeps toolchain)
|
||||
|
||||
@@ -1285,9 +1285,7 @@ async fn list_parts_disk_loss_after_metadata_read(
|
||||
let error = result.expect_err("one missing directory and three unavailable participants cannot prove an empty upload");
|
||||
let response = error.raw_response().expect("ListParts must return an S3 service response");
|
||||
assert_eq!(response.status().as_u16(), 503);
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("ServiceUnavailable"));
|
||||
assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("SlowDownRead"));
|
||||
assert!(response.headers().get("x-rustfs-error-cause").is_none());
|
||||
assert!(response.headers().get("retry-after").is_none());
|
||||
assert!(
|
||||
|
||||
@@ -493,12 +493,7 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
|
||||
assert!(legacy.parts().is_empty());
|
||||
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
|
||||
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(
|
||||
current.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable")
|
||||
);
|
||||
assert_eq!(current.as_service_error().and_then(|error| error.meta().code()), Some("SlowDownRead"));
|
||||
dist.cluster.start_node_from_binary(3, previous).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
|
||||
for (path, before) in original_files {
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-gateway-adapter"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "RustFS implementations of the rustfs-gateway extension traits (credentials, policy facts, virtual hosts, bucket owner and request id)."
|
||||
keywords = ["rustfs", "gateway", "s3", "iam"]
|
||||
categories = ["web-programming", "authentication"]
|
||||
authors.workspace = true
|
||||
publish = false
|
||||
|
||||
[package.metadata.gateway]
|
||||
# Documentary only: this crate is a ring-2 consumer of the gateway (it depends on
|
||||
# RustFS business crates). The RustFS repository does not run the gateway's ring guard.
|
||||
ring = 2
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
http = { workspace = true }
|
||||
rustfs-gateway = { workspace = true }
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-iam = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
rustfs-gateway-http = { workspace = true }
|
||||
time = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
@@ -1,215 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`BucketOwnerSource`]: the one owner RustFS reports, for a bucket that exists.
|
||||
//!
|
||||
//! Responsible for: confirming through a [`BucketLookup`] that a bucket exists, answering the
|
||||
//! single owner id the deployment reports ([`RustfsBucketOwners::new`]) for it, and turning every
|
||||
//! outcome that is not a usable owner into the gateway's one refusal, [`BucketOwnerError`].
|
||||
//! NOT responsible for: choosing the owner id (the binary passes the constant it already reports
|
||||
//! as the owner in listings), comparing it with `x-amz-expected-bucket-owner` (the gateway does,
|
||||
//! byte for byte), or authorizing anything.
|
||||
//! Upstream: the gateway's expected-owner check, which consults this source only when a request
|
||||
//! carries the header, after authorization allowed it. Downstream: the lookup the binary supplies
|
||||
//! over the ecstore bucket metadata system.
|
||||
//!
|
||||
//! # What an owner is in RustFS
|
||||
//!
|
||||
//! RustFS stores no per-bucket owner: bucket ACL writes are no-ops and nothing writes the ACL
|
||||
//! document, so the only owner RustFS reports anywhere (ListBuckets, ACL and multipart listings)
|
||||
//! is one constant. The source answers exactly that constant. A caller therefore passes the
|
||||
//! gateway's check only by presenting the owner id RustFS lists, which separates no tenants; it is
|
||||
//! a compatibility answer, and the constant's own documentation says it must not decide
|
||||
//! authorization.
|
||||
//!
|
||||
//! # Difference from legacy (ledger entry R-071)
|
||||
//!
|
||||
//! Legacy RustFS ignores `x-amz-expected-bucket-owner` entirely. On the gateway stack a request
|
||||
//! carrying it is served only when the value equals the reported owner id; otherwise it is `403
|
||||
//! AccessDenied`. A missing bucket and a backend failure with the header present are also `403`
|
||||
//! at the pinned gateway, where legacy answers `404` and `5xx`: [`BucketOwnerError`] has one value,
|
||||
//! so it cannot express either. [`RustfsBucketOwners::resolve`] keeps the two apart at this
|
||||
//! adapter's boundary so the gateway can surface them once it grows the classes.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{BoxFuture, BucketName, BucketOwnerError, BucketOwnerSource};
|
||||
|
||||
/// Why a bucket yielded no owner.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum OwnerLookupError {
|
||||
/// The bucket does not exist.
|
||||
NoSuchBucket,
|
||||
/// The metadata backend failed, could not be reached, or the owner is not usable (an empty
|
||||
/// owner id is a misconfiguration), so the bucket's state is unknown.
|
||||
Backend,
|
||||
}
|
||||
|
||||
/// The narrow slice of RustFS bucket metadata an owner answer needs: whether a bucket exists.
|
||||
///
|
||||
/// Exists so the source can be tested against a fake. The binary implements it over the ecstore
|
||||
/// bucket metadata system, mapping `BucketNotFound` to [`OwnerLookupError::NoSuchBucket`] and every
|
||||
/// other error to [`OwnerLookupError::Backend`].
|
||||
pub trait BucketLookup: Send + Sync + 'static {
|
||||
/// `Ok(())` when `bucket` exists.
|
||||
fn check<'a>(&'a self, bucket: &'a str) -> BoxFuture<'a, Result<(), OwnerLookupError>>;
|
||||
}
|
||||
|
||||
/// A [`BucketOwnerSource`] answering one owner id for every bucket that exists.
|
||||
pub struct RustfsBucketOwners<L> {
|
||||
lookup: L,
|
||||
owner: Arc<str>,
|
||||
}
|
||||
|
||||
impl<L: BucketLookup> RustfsBucketOwners<L> {
|
||||
/// A source answering `owner` for every bucket `lookup` finds.
|
||||
///
|
||||
/// An empty `owner` is no owner: every answer is then [`OwnerLookupError::Backend`].
|
||||
#[must_use]
|
||||
pub fn new(lookup: L, owner: impl Into<Arc<str>>) -> Self {
|
||||
Self {
|
||||
lookup,
|
||||
owner: owner.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The owner of `bucket`, or why there is none. Missing and failing stay distinct here.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`OwnerLookupError::NoSuchBucket`] when the bucket does not exist;
|
||||
/// [`OwnerLookupError::Backend`] on any lookup failure or when the configured owner is empty.
|
||||
pub async fn resolve(&self, bucket: &str) -> Result<Arc<str>, OwnerLookupError> {
|
||||
self.lookup.check(bucket).await?;
|
||||
if self.owner.is_empty() {
|
||||
return Err(OwnerLookupError::Backend);
|
||||
}
|
||||
Ok(Arc::clone(&self.owner))
|
||||
}
|
||||
}
|
||||
|
||||
impl<L: BucketLookup> BucketOwnerSource for RustfsBucketOwners<L> {
|
||||
fn owner<'a>(&'a self, bucket: &'a BucketName) -> BoxFuture<'a, Result<Arc<str>, BucketOwnerError>> {
|
||||
Box::pin(async move {
|
||||
// The pinned gateway collapses both classes (R-071), so both are the one refusal.
|
||||
self.resolve(bucket.as_str())
|
||||
.await
|
||||
.map_err(|_: OwnerLookupError| BucketOwnerError::unavailable())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// Stands for the owner id RustFS reports; the binary passes its own constant.
|
||||
const OWNER: &str = "reported-owner-id";
|
||||
|
||||
struct Fake {
|
||||
answer: Result<(), OwnerLookupError>,
|
||||
asked: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl Fake {
|
||||
fn answering(answer: Result<(), OwnerLookupError>) -> Self {
|
||||
Self {
|
||||
answer,
|
||||
asked: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketLookup for Fake {
|
||||
fn check<'a>(&'a self, bucket: &'a str) -> BoxFuture<'a, Result<(), OwnerLookupError>> {
|
||||
self.asked.lock().unwrap().push(bucket.to_owned());
|
||||
let answer = self.answer;
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
fn source(answer: Result<(), OwnerLookupError>) -> RustfsBucketOwners<Fake> {
|
||||
RustfsBucketOwners::new(Fake::answering(answer), OWNER)
|
||||
}
|
||||
|
||||
async fn gateway_answer(answer: Result<(), OwnerLookupError>) -> Result<Arc<str>, BucketOwnerError> {
|
||||
source(answer).owner(&BucketName::new("photos").expect("a bucket name")).await
|
||||
}
|
||||
|
||||
/// Positive: an existing bucket's owner is the configured id, byte for byte.
|
||||
#[tokio::test]
|
||||
async fn an_existing_bucket_has_the_configured_owner() {
|
||||
assert_eq!(gateway_answer(Ok(())).await.as_deref(), Ok(OWNER));
|
||||
}
|
||||
|
||||
/// Positive: every bucket gets the same owner, since RustFS stores none per bucket.
|
||||
#[tokio::test]
|
||||
async fn every_bucket_has_the_same_owner() {
|
||||
let source = source(Ok(()));
|
||||
assert_eq!(source.resolve("photos").await.as_deref(), Ok(OWNER));
|
||||
assert_eq!(source.resolve("backups").await.as_deref(), Ok(OWNER));
|
||||
}
|
||||
|
||||
/// Positive: the lookup is asked about exactly the bucket the gateway named.
|
||||
#[tokio::test]
|
||||
async fn the_lookup_is_asked_about_the_named_bucket() {
|
||||
let source = source(Ok(()));
|
||||
source
|
||||
.owner(&BucketName::new("photos").expect("a bucket name"))
|
||||
.await
|
||||
.expect("an owner");
|
||||
assert_eq!(*source.lookup.asked.lock().unwrap(), vec!["photos".to_owned()]);
|
||||
}
|
||||
|
||||
/// Negative: a missing bucket is reported as missing at the adapter boundary.
|
||||
#[tokio::test]
|
||||
async fn a_missing_bucket_is_reported_as_missing() {
|
||||
assert_eq!(
|
||||
source(Err(OwnerLookupError::NoSuchBucket)).resolve("photos").await,
|
||||
Err(OwnerLookupError::NoSuchBucket)
|
||||
);
|
||||
}
|
||||
|
||||
/// Negative: a failing backend is reported as a failure, never as an owner and never as missing.
|
||||
#[tokio::test]
|
||||
async fn a_backend_failure_is_reported_as_a_failure() {
|
||||
assert_eq!(
|
||||
source(Err(OwnerLookupError::Backend)).resolve("photos").await,
|
||||
Err(OwnerLookupError::Backend)
|
||||
);
|
||||
}
|
||||
|
||||
/// Negative: the pinned gateway sees one refusal for both, and never an owner.
|
||||
#[tokio::test]
|
||||
async fn the_gateway_is_refused_in_both_cases() {
|
||||
assert_eq!(
|
||||
gateway_answer(Err(OwnerLookupError::NoSuchBucket)).await,
|
||||
Err(BucketOwnerError::unavailable())
|
||||
);
|
||||
assert_eq!(gateway_answer(Err(OwnerLookupError::Backend)).await, Err(BucketOwnerError::unavailable()));
|
||||
}
|
||||
|
||||
/// Negative: an empty configured owner is no owner; it must not equal an empty assertion.
|
||||
#[tokio::test]
|
||||
async fn an_empty_owner_is_refused() {
|
||||
let source = RustfsBucketOwners::new(Fake::answering(Ok(())), "");
|
||||
assert_eq!(source.resolve("photos").await, Err(OwnerLookupError::Backend));
|
||||
assert_eq!(
|
||||
source.owner(&BucketName::new("photos").expect("a bucket name")).await,
|
||||
Err(BucketOwnerError::unavailable())
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,446 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`CredentialProvider`], answered by RustFS IAM.
|
||||
//!
|
||||
//! Responsible for: resolving one access key id to the secret the gateway verifies a signature
|
||||
//! with, and classifying every other outcome as either a business result
|
||||
//! ([`CredentialLookup::NotFound`], eligible for the gateway's negative cache) or an operational
|
||||
//! fault ([`ProviderError`], counted and never cached).
|
||||
//! NOT responsible for: authorization of any kind, session-token claim validation, or deciding
|
||||
//! what the client is answered with. The gateway answers every unusable credential identically.
|
||||
//!
|
||||
//! # Secret handling
|
||||
//!
|
||||
//! Secrets travel only inside the gateway's [`Credentials`], whose `Debug` redacts them. No type
|
||||
//! in this module derives or implements `Debug`, and no error produced here carries text: the
|
||||
//! IAM error is classified and dropped. A secret that fails the crate's effective-length rule
|
||||
//! (`crate::MIN_SECRET_KEY_LEN` bytes once trailing NULs are removed) is never handed to the
|
||||
//! gateway, whatever the lookup says about its validity.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{BoxFuture, CredentialLookup, CredentialProvider, Credentials, ProviderError, SessionBinding};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::store::Store;
|
||||
use rustfs_iam::sys::IamSys;
|
||||
use rustfs_policy::auth::{ACCOUNT_OFF, UserIdentity};
|
||||
|
||||
use crate::root::{RootSource, is_usable_secret};
|
||||
|
||||
/// The opaque issuer recorded in the [`rustfs_gateway::SessionBinding`] of every RustFS STS
|
||||
/// credential.
|
||||
pub const STS_SESSION_ISSUER: &str = "rustfs-sts";
|
||||
|
||||
/// Access keys with this prefix belong to Keystone token authentication, which never verifies an
|
||||
/// AWS signature.
|
||||
const KEYSTONE_ACCESS_KEY_PREFIX: &str = "keystone:";
|
||||
|
||||
/// The narrow slice of IAM that a credential lookup needs.
|
||||
///
|
||||
/// Exists so the adapter can be tested against a fake: IAM exports no in-memory backend.
|
||||
/// Implemented for [`IamSys`], whose `check_key` loads a user from storage on a cache miss.
|
||||
pub trait IamKeyLookup: Send + Sync + 'static {
|
||||
/// Whether the IAM store has finished loading.
|
||||
fn is_ready(&self) -> bool;
|
||||
|
||||
/// Resolves an access key, with [`IamSys::check_key`]'s contract: `Ok((None, _))` for an
|
||||
/// unknown key, `Ok((Some(identity), valid))` otherwise, where `valid` is false for a
|
||||
/// disabled, expired, or malformed credential.
|
||||
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>>;
|
||||
}
|
||||
|
||||
impl<T: Store> IamKeyLookup for IamSys<T> {
|
||||
fn is_ready(&self) -> bool {
|
||||
IamSys::is_ready(self)
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
Box::pin(IamSys::check_key(self, access_key))
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`CredentialProvider`] over RustFS IAM and the live root credentials.
|
||||
///
|
||||
/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
|
||||
/// policy source it was assembled with.
|
||||
pub struct IamCredentials {
|
||||
iam: Arc<dyn IamKeyLookup>,
|
||||
root: Arc<dyn RootSource>,
|
||||
}
|
||||
|
||||
impl IamCredentials {
|
||||
pub(crate) fn new(iam: Arc<dyn IamKeyLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
Self { iam, root }
|
||||
}
|
||||
|
||||
async fn resolve(&self, access_key: &str) -> Result<CredentialLookup, ProviderError> {
|
||||
// Deliberate change from the legacy s3s path (`rustfs/src/auth.rs` `get_secret_key`),
|
||||
// which answered a `keystone:` key with an empty secret. Keystone authenticates by token,
|
||||
// so there is no AWS secret to verify a signature against: the key is unknown here.
|
||||
if access_key.starts_with(KEYSTONE_ACCESS_KEY_PREFIX) {
|
||||
return Ok(CredentialLookup::NotFound);
|
||||
}
|
||||
|
||||
// Legacy answers every key with an internal error while the root credentials are not
|
||||
// initialized; without them no key can be classified as root or not.
|
||||
let Some(root) = self.root.current() else {
|
||||
return Err(ProviderError::Unavailable);
|
||||
};
|
||||
// Byte-exact, as the trait requires: the access key id is signed material.
|
||||
if root.access_key() == access_key {
|
||||
return Ok(CredentialLookup::Found(root.credentials().clone_credentials()));
|
||||
}
|
||||
|
||||
if !self.iam.is_ready() {
|
||||
return Err(ProviderError::Unavailable);
|
||||
}
|
||||
|
||||
match self.iam.check_key(access_key).await {
|
||||
// IAM refuses such a secret through `valid`, but this provider does not rely on every
|
||||
// lookup doing so: a short secret is never usable, so it is not a credential at all.
|
||||
Ok((Some(identity), _)) if !is_usable_secret(identity.credentials.secret_key.as_bytes()) => {
|
||||
Ok(CredentialLookup::NotFound)
|
||||
}
|
||||
Ok((Some(identity), valid)) => to_gateway_credentials(access_key, &identity, valid).map(CredentialLookup::Found),
|
||||
Ok((None, _)) => Ok(CredentialLookup::NotFound),
|
||||
Err(error) => classify_iam_error(&error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies an IAM failure without keeping any of its text.
|
||||
fn classify_iam_error(error: &IamError) -> Result<CredentialLookup, ProviderError> {
|
||||
match error {
|
||||
IamError::NoSuchUser(_)
|
||||
| IamError::NoSuchServiceAccount(_)
|
||||
| IamError::NoSuchTempAccount(_)
|
||||
| IamError::NoSuchAccount(_) => Ok(CredentialLookup::NotFound),
|
||||
IamError::IamSysNotInitialized => Err(ProviderError::Unavailable),
|
||||
_ => Err(ProviderError::Backend),
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the gateway credential for one IAM identity. No authorization happens here: a
|
||||
/// disabled or invalid principal still resolves, marked disabled, and the gateway refuses it
|
||||
/// exactly as it refuses an unknown key.
|
||||
fn to_gateway_credentials(access_key: &str, identity: &UserIdentity, valid: bool) -> Result<Credentials, ProviderError> {
|
||||
let cred = &identity.credentials;
|
||||
let mut credentials = Credentials::new(access_key, cred.secret_key.as_bytes()).map_err(|_| ProviderError::Backend)?;
|
||||
|
||||
// A service account carries a claims token internally, but its clients never present it, so
|
||||
// it authenticates as a long-term credential (as `check_session_token` treats it in legacy).
|
||||
if !cred.session_token.is_empty() && !cred.is_service_account() {
|
||||
// A temporary credential without an expiry cannot become a gateway session, which always
|
||||
// ends. Refusing it as a fault is the fail-closed answer; a never-ending session is not.
|
||||
let expiration = cred.expiration.ok_or(ProviderError::Backend)?;
|
||||
let binding = SessionBinding::new(STS_SESSION_ISSUER, expiration.unix_timestamp()).map_err(|_| ProviderError::Backend)?;
|
||||
credentials = credentials
|
||||
.with_session(&cred.session_token, binding)
|
||||
.map_err(|_| ProviderError::Backend)?;
|
||||
}
|
||||
|
||||
if !valid || cred.status == ACCOUNT_OFF {
|
||||
credentials = credentials.disable();
|
||||
}
|
||||
Ok(credentials)
|
||||
}
|
||||
|
||||
impl CredentialProvider for IamCredentials {
|
||||
fn lookup<'a>(&'a self, access_key_id: &'a str) -> BoxFuture<'a, Result<CredentialLookup, ProviderError>> {
|
||||
Box::pin(self.resolve(access_key_id))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::root::{RootCell, RootCredentials};
|
||||
use rustfs_credentials::Credentials as IamCredential;
|
||||
use rustfs_gateway::RequestNow;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const ACCESS_KEY: &str = "AKIDUSER0001";
|
||||
const SECRET: &str = "user-secret-key-0001";
|
||||
|
||||
type Answer = Box<dyn Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync>;
|
||||
|
||||
struct FakeIam {
|
||||
ready: bool,
|
||||
answer: Answer,
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl IamKeyLookup for FakeIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let answer = (self.answer)();
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
fn fake(
|
||||
ready: bool,
|
||||
answer: impl Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync + 'static,
|
||||
) -> Arc<FakeIam> {
|
||||
Arc::new(FakeIam {
|
||||
ready,
|
||||
answer: Box::new(answer),
|
||||
calls: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
fn user(status: &str) -> IamCredential {
|
||||
IamCredential {
|
||||
access_key: ACCESS_KEY.to_owned(),
|
||||
secret_key: SECRET.to_owned(),
|
||||
status: status.to_owned(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn found(cred: IamCredential, valid: bool) -> Result<(Option<UserIdentity>, bool), IamError> {
|
||||
Ok((Some(UserIdentity::new(cred)), valid))
|
||||
}
|
||||
|
||||
fn root_cell() -> Arc<RootCell> {
|
||||
let root = RootCredentials::new("rootadmin", "root-secret-key", "sts-signing-key").expect("valid root credentials");
|
||||
Arc::new(RootCell::new(Some(root)))
|
||||
}
|
||||
|
||||
async fn lookup(iam: Arc<FakeIam>, access_key: &str) -> Result<CredentialLookup, ProviderError> {
|
||||
IamCredentials::new(iam, root_cell()).lookup(access_key).await
|
||||
}
|
||||
|
||||
fn expect_found(result: Result<CredentialLookup, ProviderError>) -> Credentials {
|
||||
match result {
|
||||
Ok(CredentialLookup::Found(credentials)) => credentials,
|
||||
Ok(CredentialLookup::NotFound) => panic!("expected Found, got NotFound"),
|
||||
Err(error) => panic!("expected Found, got {error:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_not_found(result: &Result<CredentialLookup, ProviderError>) -> bool {
|
||||
matches!(result, Ok(CredentialLookup::NotFound))
|
||||
}
|
||||
|
||||
/// Positive — an enabled user resolves to exactly its secret, as a long-term credential.
|
||||
#[tokio::test]
|
||||
async fn credentials_found_user_resolves_to_its_secret() {
|
||||
let iam = fake(true, || found(user("on"), true));
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
assert_eq!(credentials.secret().expose(), SECRET.as_bytes());
|
||||
assert_eq!(credentials.identity().access_key_id(), ACCESS_KEY);
|
||||
assert!(!credentials.is_disabled());
|
||||
assert!(credentials.session_binding().is_none());
|
||||
}
|
||||
|
||||
/// Negative — a key IAM does not know is a business result, not a fault.
|
||||
#[tokio::test]
|
||||
async fn credentials_unknown_key_is_not_found() {
|
||||
let iam = fake(true, || Ok((None, false)));
|
||||
assert!(is_not_found(&lookup(iam, "AKIDUNKNOWN").await));
|
||||
}
|
||||
|
||||
/// Negative — every "no such principal" error IAM can surface is `NotFound`. Answering these
|
||||
/// as `Backend` would keep them out of the negative cache and count every guessed key as a
|
||||
/// backend fault.
|
||||
#[tokio::test]
|
||||
async fn credentials_no_such_principal_errors_are_not_found() {
|
||||
let errors: [fn() -> IamError; 4] = [
|
||||
|| IamError::NoSuchUser(ACCESS_KEY.to_owned()),
|
||||
|| IamError::NoSuchServiceAccount(ACCESS_KEY.to_owned()),
|
||||
|| IamError::NoSuchTempAccount(ACCESS_KEY.to_owned()),
|
||||
|| IamError::NoSuchAccount(ACCESS_KEY.to_owned()),
|
||||
];
|
||||
for error in errors {
|
||||
let iam = fake(true, move || Err(error()));
|
||||
let result = lookup(iam, ACCESS_KEY).await;
|
||||
assert!(is_not_found(&result), "{:?} must be NotFound", error());
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — any other IAM failure is a backend fault, never `NotFound` and never `Found`.
|
||||
#[tokio::test]
|
||||
async fn credentials_backend_error_is_backend() {
|
||||
let iam = fake(true, || Err(IamError::StringError("storage read failed".to_owned())));
|
||||
assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
|
||||
}
|
||||
|
||||
/// Negative — a `keystone:` key never reaches IAM and never resolves to a secret.
|
||||
///
|
||||
/// Deliberate change from the legacy s3s path, which answered such a key with an empty
|
||||
/// secret: Keystone authenticates by token, so there is no AWS secret to verify against.
|
||||
#[tokio::test]
|
||||
async fn credentials_keystone_prefix_is_not_found_without_iam() {
|
||||
let iam = fake(true, || found(user("on"), true));
|
||||
let result = lookup(iam.clone(), "keystone:project-user").await;
|
||||
assert!(is_not_found(&result));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Negative — a switched-off or otherwise invalid user resolves, but disabled, and the same
|
||||
/// user enabled does not.
|
||||
#[tokio::test]
|
||||
async fn credentials_disabled_user_is_disabled() {
|
||||
let off = expect_found(lookup(fake(true, || found(user("off"), false)), ACCESS_KEY).await);
|
||||
assert!(off.is_disabled());
|
||||
assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
|
||||
|
||||
let invalid = expect_found(lookup(fake(true, || found(user("on"), false)), ACCESS_KEY).await);
|
||||
assert!(invalid.is_disabled());
|
||||
|
||||
let on = expect_found(lookup(fake(true, || found(user("on"), true)), ACCESS_KEY).await);
|
||||
assert!(!on.is_disabled());
|
||||
}
|
||||
|
||||
/// Positive — an STS credential carries its token and its lifetime as one session binding.
|
||||
#[tokio::test]
|
||||
async fn credentials_sts_carries_session_binding() {
|
||||
let expires_at = 4_000_000_000_i64;
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "sts-session-token".to_owned();
|
||||
cred.parent_user = "parent".to_owned();
|
||||
cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
let binding = credentials.session_binding().expect("an STS credential has a session");
|
||||
assert_eq!(binding.expires_at_unix_seconds(), expires_at);
|
||||
assert_eq!(binding.issuer(), STS_SESSION_ISSUER);
|
||||
let now = RequestNow::from_unix_seconds(expires_at - 1);
|
||||
assert!(credentials.admit(Some(b"sts-session-token"), now).is_ok());
|
||||
assert!(credentials.admit(None, now).is_err());
|
||||
}
|
||||
|
||||
/// Negative — an STS credential with no expiry cannot be expressed as a session the gateway
|
||||
/// would ever expire, so it is a backend fault rather than a session that never ends.
|
||||
#[tokio::test]
|
||||
async fn credentials_sts_without_expiry_is_backend() {
|
||||
let iam = fake(true, || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "sts-session-token".to_owned();
|
||||
found(cred, true)
|
||||
});
|
||||
assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
|
||||
}
|
||||
|
||||
/// Positive — a service account carries a token internally but authenticates as a long-term
|
||||
/// credential: clients never present that token.
|
||||
#[tokio::test]
|
||||
async fn credentials_service_account_is_long_term() {
|
||||
let iam = fake(true, || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "service-account-claims-token".to_owned();
|
||||
cred.parent_user = "parent".to_owned();
|
||||
cred.claims = Some([(rustfs_credentials::IAM_POLICY_CLAIM_NAME_SA.to_owned(), "inherited-policy".into())].into());
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
assert!(credentials.session_binding().is_none());
|
||||
assert!(credentials.admit(None, RequestNow::from_unix_seconds(0)).is_ok());
|
||||
}
|
||||
|
||||
/// Negative — before IAM is ready a lookup is `Unavailable`, not `NotFound`: a key must not
|
||||
/// be negatively cached because the store had not loaded yet.
|
||||
#[tokio::test]
|
||||
async fn credentials_iam_not_ready_is_unavailable() {
|
||||
let iam = fake(false, || found(user("on"), true));
|
||||
assert!(matches!(lookup(iam.clone(), ACCESS_KEY).await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let uninitialized = fake(true, || Err(IamError::IamSysNotInitialized));
|
||||
assert!(matches!(lookup(uninitialized, ACCESS_KEY).await, Err(ProviderError::Unavailable)));
|
||||
}
|
||||
|
||||
/// Positive — the injected root credentials resolve without IAM, even before it is ready, and
|
||||
/// nothing else is answered with the root secret.
|
||||
#[tokio::test]
|
||||
async fn credentials_root_resolves_without_iam() {
|
||||
let iam = fake(false, || found(user("on"), true));
|
||||
let provider = IamCredentials::new(iam.clone(), root_cell());
|
||||
let root = expect_found(provider.lookup("rootadmin").await);
|
||||
assert_eq!(root.secret().expose(), b"root-secret-key");
|
||||
assert!(matches!(provider.lookup("ROOTADMIN").await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Negative — a lookup that calls an identity valid but holds an empty or short secret is not
|
||||
/// trusted: such a secret is not a credential, so the key is `NotFound`, never `Found`.
|
||||
#[tokio::test]
|
||||
async fn credentials_short_secret_is_never_found() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.secret_key = weak.to_owned();
|
||||
found(cred, true)
|
||||
});
|
||||
let result = lookup(iam, ACCESS_KEY).await;
|
||||
assert!(is_not_found(&result), "{weak:?} must be NotFound");
|
||||
}
|
||||
for usable in ["12345678", "abc\0defg"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.secret_key = usable.to_owned();
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
assert_eq!(credentials.secret().expose(), usable.as_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a switched-off account is disabled even when the lookup calls it valid.
|
||||
#[tokio::test]
|
||||
async fn credentials_off_status_alone_disables() {
|
||||
let off = expect_found(lookup(fake(true, || found(user("off"), true)), ACCESS_KEY).await);
|
||||
assert!(off.is_disabled());
|
||||
assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
|
||||
}
|
||||
|
||||
/// Negative — an STS credential admits only its own session's token: the token of another
|
||||
/// session, or none, is refused.
|
||||
#[tokio::test]
|
||||
async fn credentials_sts_refuses_another_sessions_token() {
|
||||
let expires_at = 4_000_000_000_i64;
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "session-token-a".to_owned();
|
||||
cred.parent_user = "parent".to_owned();
|
||||
cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
let now = RequestNow::from_unix_seconds(expires_at - 1);
|
||||
assert!(credentials.admit(Some(b"session-token-b"), now).is_err());
|
||||
assert!(credentials.admit(Some(b"session-token-a"), now).is_ok());
|
||||
}
|
||||
|
||||
/// Negative — without root credentials no key can be classified, so every lookup is
|
||||
/// `Unavailable` and IAM is not asked.
|
||||
#[tokio::test]
|
||||
async fn credentials_without_root_are_unavailable() {
|
||||
let iam = fake(true, || found(user("on"), true));
|
||||
let provider = IamCredentials::new(iam.clone(), Arc::new(RootCell::new(None)));
|
||||
assert!(matches!(provider.lookup(ACCESS_KEY).await, Err(ProviderError::Unavailable)));
|
||||
assert!(matches!(provider.lookup("rootadmin").await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
@@ -1,306 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`HostResolver`], built from RustFS's configured domain list.
|
||||
//!
|
||||
//! Responsible for: turning the `RUSTFS_SERVER_DOMAINS` list into the gateway's legacy RustFS
|
||||
//! virtual-host resolver.
|
||||
//! NOT responsible for: parsing a `Host` header, matching a domain, validating bucket names (all
|
||||
//! done by [`LegacyRustfsVirtualHosts`]), reading the environment, or deciding that a console
|
||||
//! deployment serves no domains (the caller passes an empty list then, as `s3_host_domains` in
|
||||
//! `rustfs/src/server/http.rs` returns `None`). An empty list needs no branch here:
|
||||
//! [`LegacyRustfsVirtualHosts`] built from no domain reads every host path-style, exactly as the
|
||||
//! gateway's `PathStyleOnly` does.
|
||||
//! Upstream: the RustFS configuration (`Config::server_domains`). Downstream: the gateway's
|
||||
//! `ServiceBuilder`, which holds the resolver as `Arc<dyn HostResolver>`.
|
||||
//!
|
||||
//! # What "the same as `MultiDomain`" means
|
||||
//!
|
||||
//! RustFS today hands the same list to the s3s framework's multi-domain host, which ignores ports
|
||||
//! on both sides when matching a domain, takes the whole prefix in front of a matched domain as
|
||||
//! the bucket, compares a domain exactly (so a host is matched on the case it arrives in), and,
|
||||
//! for a host no domain matches, falls back to CNAME-style: a host that is itself a bucket name
|
||||
//! (lower-cased) is that bucket. [`LegacyRustfsVirtualHosts`] reproduces exactly that.
|
||||
//!
|
||||
//! What reads **path-style**: an empty domain list; a configured domain itself, with or without a
|
||||
//! port; an IP address, or a socket address (`10.0.0.1`, `[::1]:9000`); a host that is a valid
|
||||
//! domain but, lower-cased, is no bucket name (it carries a port, is over 63 bytes, or starts
|
||||
//! with `xn--`). What reads as a **bucket**: anything under a configured domain, and any other
|
||||
//! bucket-shaped host, which is a bucket of its own (`localhost`, `badexample.com`, `EXAMPLE.COM`).
|
||||
//! What is **refused** before routing: a host that is no domain (`under_score.org`, a bare
|
||||
//! `[::1]`, a trailing dot), and a bucket under a domain that the bucket rules refuse
|
||||
//! (`PHOTOS.example.com`, `ab.example.com`). The tests below pin each of those readings.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{HostResolver, LegacyDomainError, LegacyRustfsVirtualHosts};
|
||||
|
||||
/// Builds the host resolver for RustFS's configured domain list.
|
||||
///
|
||||
/// An empty list reads every host path-style. Otherwise the list is read as legacy RustFS reads
|
||||
/// it: a later entry whose name without its port repeats an earlier one is dropped, and a list
|
||||
/// that legacy RustFS refuses at start-up is refused here.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`LegacyDomainError`] for an entry that is not a domain, or for two entries where one equals or
|
||||
/// is a subdomain of the other.
|
||||
pub fn rustfs_host_resolver<I, S>(domains: I) -> Result<Arc<dyn HostResolver>, LegacyDomainError>
|
||||
where
|
||||
I: IntoIterator<Item = S>,
|
||||
S: AsRef<str>,
|
||||
{
|
||||
Ok(Arc::new(LegacyRustfsVirtualHosts::new(domains)?))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::Method;
|
||||
use rustfs_gateway::{Addressing, HostQuery, HostRefusal, TargetKind};
|
||||
|
||||
/// What the resolver reads `GET path` with `Host: host` as: the host-named bucket (`None` for
|
||||
/// path style), the routed target, and the refusal, if any.
|
||||
fn read(resolver: &dyn HostResolver, path: &str, host: &str) -> (Option<String>, TargetKind, Option<HostRefusal>) {
|
||||
let method = Method::GET;
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(path)
|
||||
.header("host", host)
|
||||
.body(())
|
||||
.expect("a valid request");
|
||||
let effective = rustfs_gateway_http::effective_host(&request).expect("an acceptable host");
|
||||
let query = HostQuery {
|
||||
host: &effective,
|
||||
path,
|
||||
method: &method,
|
||||
};
|
||||
let resolved = resolver.resolve(&query);
|
||||
let bucket = match &resolved.addressing {
|
||||
Addressing::VirtualHosted { bucket, .. } => Some(bucket.as_str().to_owned()),
|
||||
Addressing::Path => None,
|
||||
};
|
||||
(bucket, resolved.target, resolver.refusal(&query))
|
||||
}
|
||||
|
||||
fn with_domains(domains: &[&str]) -> Arc<dyn HostResolver> {
|
||||
rustfs_host_resolver(domains).expect("a list RustFS accepts")
|
||||
}
|
||||
|
||||
fn bucket(domains: &[&str], host: &str) -> Option<String> {
|
||||
read(&*with_domains(domains), "/key", host).0
|
||||
}
|
||||
|
||||
/// Positive: a host under a configured domain names the bucket in front of it.
|
||||
#[test]
|
||||
fn host_under_domain_names_the_bucket() {
|
||||
assert_eq!(
|
||||
read(&*with_domains(&["example.com"]), "/key", "photos.example.com"),
|
||||
(Some("photos".to_owned()), TargetKind::Object, None)
|
||||
);
|
||||
assert_eq!(read(&*with_domains(&["example.com"]), "/", "photos.example.com").1, TargetKind::Bucket);
|
||||
}
|
||||
|
||||
/// Positive: the port on the host, or on the configured domain, does not take part in matching.
|
||||
#[test]
|
||||
fn ports_are_ignored_on_both_sides() {
|
||||
assert_eq!(bucket(&["example.com"], "photos.example.com:9000").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&["example.com:9000"], "photos.example.com").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&["example.com:9000"], "photos.example.com:1234").as_deref(), Some("photos"));
|
||||
}
|
||||
|
||||
/// Positive: the whole prefix in front of the domain is the bucket, dots kept.
|
||||
#[test]
|
||||
fn whole_prefix_is_the_bucket() {
|
||||
assert_eq!(bucket(&["example.com"], "my.dotted.bkt.example.com").as_deref(), Some("my.dotted.bkt"));
|
||||
}
|
||||
|
||||
/// Positive: with several domains, each one resolves its own subdomains.
|
||||
#[test]
|
||||
fn every_configured_domain_resolves() {
|
||||
let domains = ["example.com", "s3.local"];
|
||||
assert_eq!(bucket(&domains, "abc.example.com").as_deref(), Some("abc"));
|
||||
assert_eq!(bucket(&domains, "xyz.s3.local").as_deref(), Some("xyz"));
|
||||
}
|
||||
|
||||
/// Positive: the domain itself is the endpoint, so the bucket comes from the path.
|
||||
#[test]
|
||||
fn the_domain_itself_is_path_style() {
|
||||
assert_eq!(bucket(&["example.com"], "example.com"), None);
|
||||
assert_eq!(bucket(&["example.com"], "example.com:9000"), None);
|
||||
}
|
||||
|
||||
/// Negative: no configured domain means path style only, for every host shape.
|
||||
#[test]
|
||||
fn no_domain_is_path_style_only() {
|
||||
for host in ["photos.example.com", "localhost", "localhost:9000", "10.0.0.1"] {
|
||||
assert_eq!(bucket(&[], host), None, "{host}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: an address, and a host carrying a port that no domain matches, are path style.
|
||||
#[test]
|
||||
fn addresses_and_unmatched_ports_are_path_style() {
|
||||
for host in ["10.0.0.1", "10.0.0.1:9000", "localhost:9000", "other.org:9000"] {
|
||||
assert_eq!(bucket(&["example.com"], host), None, "{host}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: a host that only ends in the same letters as a domain is not under it. It is not
|
||||
/// stripped to `bad`; the only reading it gets is the CNAME-style one, the whole host.
|
||||
#[test]
|
||||
fn suffix_lookalike_is_not_under_the_domain() {
|
||||
assert_eq!(bucket(&["example.com"], "badexample.com").as_deref(), Some("badexample.com"));
|
||||
assert_eq!(bucket(&["example.com"], "example.com.evil.org").as_deref(), Some("example.com.evil.org"));
|
||||
}
|
||||
|
||||
/// Negative: matching a configured domain is exact on case, as the legacy multi-domain host does it, so a
|
||||
/// mixed-case host is not stripped to its prefix; it is the lower-cased whole host.
|
||||
#[test]
|
||||
fn mixed_case_host_is_not_stripped_to_a_prefix() {
|
||||
assert_eq!(bucket(&["example.com"], "Photos.Example.COM").as_deref(), Some("photos.example.com"));
|
||||
}
|
||||
|
||||
/// Negative: a host that is no domain at all is refused, not routed.
|
||||
#[test]
|
||||
fn a_host_that_is_no_domain_is_refused() {
|
||||
let resolver = with_domains(&["example.com"]);
|
||||
assert_eq!(read(&*resolver, "/k", "under_score.org").2, Some(HostRefusal::UnusableHost));
|
||||
}
|
||||
|
||||
/// Negative: a bucket the host names that bucket rules refuse is refused.
|
||||
#[test]
|
||||
fn a_host_named_bucket_the_rules_refuse_is_refused() {
|
||||
let resolver = with_domains(&["example.com"]);
|
||||
assert_eq!(read(&*resolver, "/k", "ab.example.com").2, Some(HostRefusal::RefusedBucket));
|
||||
assert_eq!(read(&*resolver, "/k", "bad_host.example.com").2, Some(HostRefusal::RefusedBucket));
|
||||
}
|
||||
|
||||
/// Negative: a list legacy RustFS refuses at start-up is refused.
|
||||
#[test]
|
||||
fn domain_lists_legacy_rustfs_refuses_are_refused() {
|
||||
for list in [
|
||||
vec!["example.com", "a.example.com"],
|
||||
vec!["example.com", "com"],
|
||||
vec!["bad_domain"],
|
||||
vec![""],
|
||||
vec!["example.com:notaport"],
|
||||
] {
|
||||
assert!(rustfs_host_resolver(&list).is_err(), "{list:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive: a later repeat of a name, whatever its port, is dropped rather than refused, as
|
||||
/// `s3_host_domains` drops it before the legacy multi-domain host sees the list.
|
||||
#[test]
|
||||
fn a_repeat_of_a_domain_is_dropped() {
|
||||
assert_eq!(
|
||||
bucket(&["example.com:9411", "example.com:9000"], "xyz.example.com").as_deref(),
|
||||
Some("xyz")
|
||||
);
|
||||
}
|
||||
|
||||
/// The refusal alone, for hosts whose bucket reading is not the point.
|
||||
fn refusal(domains: &[&str], host: &str) -> Option<HostRefusal> {
|
||||
read(&*with_domains(domains), "/k", host).2
|
||||
}
|
||||
|
||||
/// Positive: a bucket-shaped host no domain matches is a bucket of its own, `localhost` and
|
||||
/// an upper-case spelling of the domain among them.
|
||||
#[test]
|
||||
fn bucket_shaped_hosts_outside_every_domain_are_their_own_bucket() {
|
||||
assert_eq!(bucket(&["example.com"], "localhost").as_deref(), Some("localhost"));
|
||||
assert_eq!(bucket(&["example.com"], "EXAMPLE.COM").as_deref(), Some("example.com"));
|
||||
assert_eq!(bucket(&["example.com"], "cdn.other.org").as_deref(), Some("cdn.other.org"));
|
||||
}
|
||||
|
||||
/// Negative: a bucket the rules refuse under a domain is refused, and reads no bucket.
|
||||
#[test]
|
||||
fn an_upper_case_bucket_under_the_domain_is_refused() {
|
||||
assert_eq!(bucket(&["example.com"], "PHOTOS.example.com"), None);
|
||||
assert_eq!(refusal(&["example.com"], "PHOTOS.example.com"), Some(HostRefusal::RefusedBucket));
|
||||
}
|
||||
|
||||
/// Negative: an IPv6 literal. The bare form is no domain and is refused; with a port it is a
|
||||
/// socket address and reads path-style.
|
||||
#[test]
|
||||
fn ipv6_literals_follow_the_address_rules() {
|
||||
assert_eq!(refusal(&["example.com"], "[::1]"), Some(HostRefusal::UnusableHost));
|
||||
assert_eq!(bucket(&["example.com"], "[::1]:9000"), None);
|
||||
assert_eq!(refusal(&["example.com"], "[::1]:9000"), None);
|
||||
}
|
||||
|
||||
/// Negative: a trailing root dot makes the host no domain, so it is refused, not stripped.
|
||||
#[test]
|
||||
fn a_trailing_dot_is_refused() {
|
||||
assert_eq!(refusal(&["example.com"], "photos.example.com."), Some(HostRefusal::UnusableHost));
|
||||
assert_eq!(bucket(&["example.com"], "photos.example.com."), None);
|
||||
}
|
||||
|
||||
/// Negative: an empty label never reaches the resolver; the gateway's wire layer refuses the
|
||||
/// host first (legacy answers 400 as well).
|
||||
#[test]
|
||||
fn an_empty_label_is_refused_before_resolution() {
|
||||
for host in ["a..b.example.com", ".example.com"] {
|
||||
let request = http::Request::builder()
|
||||
.uri("/k")
|
||||
.header("host", host)
|
||||
.body(())
|
||||
.expect("a valid request");
|
||||
assert!(rustfs_gateway_http::effective_host(&request).is_err(), "{host}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: a punycode label is no bucket under a domain; outside every domain the host is
|
||||
/// a valid domain that is no bucket name, so it reads path-style.
|
||||
#[test]
|
||||
fn a_punycode_label_is_never_a_bucket() {
|
||||
assert_eq!(refusal(&["example.com"], "xn--bcher-kva.example.com"), Some(HostRefusal::RefusedBucket));
|
||||
assert_eq!(bucket(&["example.com"], "xn--bcher-kva.de"), None);
|
||||
assert_eq!(refusal(&["example.com"], "xn--bcher-kva.de"), None);
|
||||
}
|
||||
|
||||
/// Negative: a label over 63 bytes is no bucket, under a domain or outside one.
|
||||
#[test]
|
||||
fn a_label_over_63_bytes_is_never_a_bucket() {
|
||||
let long = "a".repeat(64);
|
||||
let under = format!("{long}.example.com");
|
||||
assert_eq!(refusal(&["example.com"], &under), Some(HostRefusal::RefusedBucket));
|
||||
let outside = format!("{long}.org");
|
||||
assert_eq!(bucket(&["example.com"], &outside), None);
|
||||
assert_eq!(refusal(&["example.com"], &outside), None);
|
||||
let at_limit = "a".repeat(63);
|
||||
assert_eq!(bucket(&["example.com"], &format!("{at_limit}.example.com")), Some(at_limit));
|
||||
}
|
||||
|
||||
/// Positive and negative: two configured domains where one merely ends in the other's letters
|
||||
/// both work, and each domain itself is path-style.
|
||||
#[test]
|
||||
fn a_domain_and_its_suffix_lookalike_are_both_served() {
|
||||
let domains = ["example.com", "badexample.com"];
|
||||
assert_eq!(bucket(&domains, "photos.example.com").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&domains, "photos.badexample.com").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&domains, "badexample.com"), None);
|
||||
assert_eq!(bucket(&domains, "bad.example.com").as_deref(), Some("bad"));
|
||||
}
|
||||
|
||||
/// Negative: a configured domain is matched exactly, case included.
|
||||
#[test]
|
||||
fn a_configured_domain_is_matched_on_its_case() {
|
||||
assert_eq!(bucket(&["Example.COM"], "photos.Example.COM").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&["Example.COM"], "photos.example.com").as_deref(), Some("photos.example.com"));
|
||||
assert_eq!(bucket(&["Example.COM"], "Example.COM"), None);
|
||||
}
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! RustFS implementations of the `rustfs-gateway` extension traits.
|
||||
//!
|
||||
//! This is the gateway's ring 2: it may depend on RustFS business crates, and nothing in the
|
||||
//! gateway depends on it. Each module adapts one extension point to the RustFS subsystem that
|
||||
//! owns the answer.
|
||||
|
||||
pub mod bucket_owner;
|
||||
pub mod credentials;
|
||||
pub mod host;
|
||||
pub mod policy;
|
||||
pub mod root;
|
||||
pub mod trace;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{Authenticator, CredentialProvider, ServiceBuilder};
|
||||
|
||||
pub use bucket_owner::{BucketLookup, OwnerLookupError, RustfsBucketOwners};
|
||||
pub use credentials::{IamCredentials, IamKeyLookup, STS_SESSION_ISSUER};
|
||||
pub use host::rustfs_host_resolver;
|
||||
pub use policy::{IamPolicySource, IamPrincipalLookup, PrincipalKind, RustfsPrincipal};
|
||||
pub use root::{MIN_SECRET_KEY_LEN, RootCredentials, RootCredentialsError, RootSource};
|
||||
pub use trace::{AttachOutcome, RequestIdCarrier, attach_host_request_id, host_request_id};
|
||||
|
||||
/// The IAM-backed credential provider and policy source, assembled together.
|
||||
///
|
||||
/// Both halves read the same IAM handle and the same live [`RootSource`], so a key cannot be root
|
||||
/// to one and an ordinary user to the other. Neither half can be taken out on its own: the fields
|
||||
/// are private, and [`IamAdapters::install`] consumes the pair and hands both to one builder.
|
||||
///
|
||||
/// Mixing the halves of two assemblies does not compile:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn mix(a: rustfs_gateway_adapter::IamAdapters, b: rustfs_gateway_adapter::IamAdapters) {
|
||||
/// let _mixed = (a.credentials, b.policy);
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Nor does building a half outside the pair:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use std::sync::Arc;
|
||||
/// use rustfs_gateway_adapter::{IamPolicySource, IamPrincipalLookup, RootSource};
|
||||
/// fn build(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) {
|
||||
/// let _alone = IamPolicySource::new(iam, root);
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Installing the pair is the one supported path:
|
||||
///
|
||||
/// ```no_run
|
||||
/// use std::sync::Arc;
|
||||
/// use rustfs_gateway::{RegionSet, ServiceBuilder, SigV4Authenticator};
|
||||
/// use rustfs_gateway_adapter::{IamAdapters, IamPrincipalLookup, RootSource};
|
||||
/// fn assemble(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>, regions: RegionSet) -> ServiceBuilder {
|
||||
/// IamAdapters::new(iam, root).install(ServiceBuilder::new(), |provider| SigV4Authenticator::new(provider, regions))
|
||||
/// }
|
||||
/// ```
|
||||
pub struct IamAdapters {
|
||||
credentials: IamCredentials,
|
||||
policy: IamPolicySource,
|
||||
}
|
||||
|
||||
impl IamAdapters {
|
||||
/// Assembles both adapters over one IAM handle and one root source. The binary passes a
|
||||
/// source that reads its live root credentials on every call (see [`RootSource`]).
|
||||
#[must_use]
|
||||
pub fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
Self {
|
||||
credentials: IamCredentials::new(iam.clone(), root.clone()),
|
||||
policy: IamPolicySource::new(iam, root),
|
||||
}
|
||||
}
|
||||
|
||||
/// Installs both halves into `builder`: the credential provider through the authenticator
|
||||
/// `authenticator` builds over it (for example `SigV4Authenticator::new(provider, regions)`
|
||||
/// with the RustFS profile applied), and the policy source.
|
||||
///
|
||||
/// Consumes the pair and gives both halves to this one builder. The `authenticator` closure
|
||||
/// receives the provider handle and may keep a clone of it; nothing here prevents that, so
|
||||
/// the closure must not hand it to a second builder.
|
||||
#[must_use]
|
||||
pub fn install<A: Authenticator>(
|
||||
self,
|
||||
builder: ServiceBuilder,
|
||||
authenticator: impl FnOnce(Arc<dyn CredentialProvider>) -> A,
|
||||
) -> ServiceBuilder {
|
||||
let authenticator = authenticator(Arc::new(self.credentials));
|
||||
builder.authenticator(authenticator).policy_source(self.policy)
|
||||
}
|
||||
}
|
||||
@@ -1,779 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`PolicySource`], answered by RustFS IAM.
|
||||
//!
|
||||
//! Responsible for: the one reading of the caller's IAM facts a request is judged against — which
|
||||
//! principal kind it is, the user it acts for, its groups, its token claims, and whether it owns
|
||||
//! the deployment — packed as a [`RustfsPrincipal`] into the gateway's [`PolicySnapshot`].
|
||||
//! NOT responsible for: evaluating any policy, choosing which policy documents apply (that stays
|
||||
//! in `IamSys::is_allowed`, which owns the STS, OIDC, session-policy and service-account rules), or
|
||||
//! verifying the signature or the session token (the gateway did both before calling this).
|
||||
//! Upstream: the gateway's verified [`Identity`]. Downstream: the RustFS authorizer, which reads
|
||||
//! the payload back with `PolicySnapshot::get::<RustfsPrincipal>()`.
|
||||
//!
|
||||
//! # Failure classes
|
||||
//!
|
||||
//! Every refusal is a [`PolicyError`] with a constant label and nothing from IAM or the request:
|
||||
//! IAM not ready, not initialized, or failing, or root credentials not initialized, is
|
||||
//! [`PolicyError::unavailable`]; a principal IAM no longer knows is [`UNKNOWN_PRINCIPAL`]; a
|
||||
//! disabled one, or one whose secret fails the effective-length rule, is
|
||||
//! [`DISABLED_PRINCIPAL`]; an identity
|
||||
//! whose credential kind disagrees with IAM's record is [`SESSION_MISMATCH`]; a token whose claims
|
||||
//! cannot be read is [`PolicyError::malformed`]. None of them is ever an empty snapshot: an empty
|
||||
//! snapshot is what an anonymous caller gets, and answering a fault with it would judge an
|
||||
//! authenticated caller as anonymous.
|
||||
//!
|
||||
//! # Secret handling
|
||||
//!
|
||||
//! [`RustfsPrincipal`] holds no secret key and no session token, and implements no `Debug`, so a
|
||||
//! snapshot formatted into a log cannot print its claims either. The root credentials and the STS
|
||||
//! token-signing key are read per request from the shared [`RootSource`] and never stored here.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_credentials::Credentials as IamCredential;
|
||||
use rustfs_gateway::{BoxFuture, Identity, PolicyError, PolicySnapshot, PolicySource};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::store::Store;
|
||||
use rustfs_iam::sys::{
|
||||
IamSys, SESSION_POLICY_NAME, get_claims_from_token_with_secret, get_claims_from_token_with_secret_allow_missing_exp,
|
||||
is_rustfs_oidc_claims,
|
||||
};
|
||||
use rustfs_policy::auth::ACCOUNT_OFF;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::credentials::IamKeyLookup;
|
||||
use crate::root::{RootCredentials, RootSource, is_usable_secret};
|
||||
|
||||
/// [`PolicyError`] label: IAM does not know the authenticated principal (any longer).
|
||||
pub const UNKNOWN_PRINCIPAL: &str = "unknown-principal";
|
||||
/// [`PolicyError`] label: IAM holds the principal disabled, expired, or otherwise invalid.
|
||||
pub const DISABLED_PRINCIPAL: &str = "disabled-principal";
|
||||
/// [`PolicyError`] label: the identity's credential kind (session or long-term) disagrees with
|
||||
/// what IAM holds for its access key.
|
||||
pub const SESSION_MISMATCH: &str = "session-mismatch";
|
||||
|
||||
/// The narrow slice of IAM a policy reading needs, on top of [`IamKeyLookup`].
|
||||
///
|
||||
/// Implemented for [`IamSys`]; exists so the source can be tested against a fake.
|
||||
pub trait IamPrincipalLookup: IamKeyLookup {
|
||||
/// The built-in IAM groups `user` belongs to now, as `IamSys::user_group_memberships`
|
||||
/// answers. These may include groups granted after a credential was issued.
|
||||
fn group_memberships(&self, user: &str) -> Vec<String>;
|
||||
}
|
||||
|
||||
impl<T: Store> IamPrincipalLookup for IamSys<T> {
|
||||
fn group_memberships(&self, user: &str) -> Vec<String> {
|
||||
IamSys::user_group_memberships(self, user)
|
||||
}
|
||||
}
|
||||
|
||||
/// Which kind of credential the principal authenticated with.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PrincipalKind {
|
||||
/// The deployment's root credentials.
|
||||
Root,
|
||||
/// A long-term IAM user.
|
||||
User,
|
||||
/// A service account, acting for its parent user.
|
||||
ServiceAccount,
|
||||
/// An STS temporary credential, acting for its parent user.
|
||||
Temporary,
|
||||
}
|
||||
|
||||
/// The caller's IAM facts: the payload of every [`PolicySnapshot`] [`IamPolicySource`] produces.
|
||||
///
|
||||
/// Carries no secret key and no session token, and deliberately implements no `Debug`.
|
||||
pub struct RustfsPrincipal {
|
||||
kind: PrincipalKind,
|
||||
account: String,
|
||||
username: String,
|
||||
parent_user: String,
|
||||
credential_groups: Option<Vec<String>>,
|
||||
group_memberships: Vec<String>,
|
||||
claims: Option<HashMap<String, Value>>,
|
||||
is_owner: bool,
|
||||
}
|
||||
|
||||
impl RustfsPrincipal {
|
||||
/// The kind of credential the caller authenticated with.
|
||||
#[must_use]
|
||||
pub fn kind(&self) -> PrincipalKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
/// The access key the caller signed with (IAM `Args::account`).
|
||||
#[must_use]
|
||||
pub fn account(&self) -> &str {
|
||||
&self.account
|
||||
}
|
||||
|
||||
/// The user the caller acts as: the parent user for an STS credential or a service account,
|
||||
/// the access key otherwise.
|
||||
#[must_use]
|
||||
pub fn username(&self) -> &str {
|
||||
&self.username
|
||||
}
|
||||
|
||||
/// The parent user IAM records for the credential; empty for root and long-term users.
|
||||
#[must_use]
|
||||
pub fn parent_user(&self) -> &str {
|
||||
&self.parent_user
|
||||
}
|
||||
|
||||
/// The groups recorded on the credential itself (IAM `Args::groups`).
|
||||
#[must_use]
|
||||
pub fn credential_groups(&self) -> &Option<Vec<String>> {
|
||||
&self.credential_groups
|
||||
}
|
||||
|
||||
/// The groups IAM currently lists for [`Self::username`], read once with the snapshot.
|
||||
#[must_use]
|
||||
pub fn group_memberships(&self) -> &[String] {
|
||||
&self.group_memberships
|
||||
}
|
||||
|
||||
/// The claims the credential's token carries; `None` when it has none.
|
||||
#[must_use]
|
||||
pub fn claims(&self) -> Option<&HashMap<String, Value>> {
|
||||
self.claims.as_ref()
|
||||
}
|
||||
|
||||
/// Whether the caller owns the deployment (IAM `Args::is_owner`): root, or a credential of
|
||||
/// root that no session policy or external OIDC identity narrows.
|
||||
#[must_use]
|
||||
pub fn is_owner(&self) -> bool {
|
||||
self.is_owner
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`PolicySource`] over RustFS IAM and the live root credentials.
|
||||
///
|
||||
/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
|
||||
/// credential provider it was assembled with.
|
||||
pub struct IamPolicySource {
|
||||
iam: Arc<dyn IamPrincipalLookup>,
|
||||
root: Arc<dyn RootSource>,
|
||||
}
|
||||
|
||||
impl IamPolicySource {
|
||||
pub(crate) fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
Self { iam, root }
|
||||
}
|
||||
|
||||
async fn read(&self, identity: Option<&Identity>) -> Result<PolicySnapshot, PolicyError> {
|
||||
let Some(identity) = identity else {
|
||||
return Ok(PolicySnapshot::empty());
|
||||
};
|
||||
let principal = self.principal(identity).await?;
|
||||
Ok(PolicySnapshot::of(Arc::new(principal)))
|
||||
}
|
||||
|
||||
async fn principal(&self, identity: &Identity) -> Result<RustfsPrincipal, PolicyError> {
|
||||
let access_key = identity.access_key_id();
|
||||
let has_session = identity.session().is_some();
|
||||
// Without root credentials neither ownership nor STS claims can be decided.
|
||||
let Some(root) = self.root.current() else {
|
||||
return Err(PolicyError::unavailable());
|
||||
};
|
||||
let root_access_key = root.access_key();
|
||||
|
||||
if access_key == root_access_key {
|
||||
// Root has no session token, so a session identity for its key is not one IAM issued.
|
||||
if has_session {
|
||||
return Err(PolicyError::because(SESSION_MISMATCH));
|
||||
}
|
||||
return Ok(RustfsPrincipal {
|
||||
kind: PrincipalKind::Root,
|
||||
account: access_key.to_owned(),
|
||||
username: access_key.to_owned(),
|
||||
parent_user: String::new(),
|
||||
credential_groups: None,
|
||||
group_memberships: Vec::new(),
|
||||
claims: None,
|
||||
is_owner: true,
|
||||
});
|
||||
}
|
||||
|
||||
if !self.iam.is_ready() {
|
||||
return Err(PolicyError::unavailable());
|
||||
}
|
||||
let cred = match self.iam.check_key(access_key).await {
|
||||
Ok((Some(identity), true)) => identity.credentials,
|
||||
Ok((Some(_), false)) => return Err(PolicyError::because(DISABLED_PRINCIPAL)),
|
||||
Ok((None, _)) => return Err(PolicyError::because(UNKNOWN_PRINCIPAL)),
|
||||
Err(error) => return Err(classify_iam_error(&error)),
|
||||
};
|
||||
if cred.status == ACCOUNT_OFF || !is_usable_secret(cred.secret_key.as_bytes()) {
|
||||
return Err(PolicyError::because(DISABLED_PRINCIPAL));
|
||||
}
|
||||
|
||||
let kind = if cred.is_service_account() {
|
||||
PrincipalKind::ServiceAccount
|
||||
} else if !cred.session_token.is_empty() {
|
||||
PrincipalKind::Temporary
|
||||
} else {
|
||||
PrincipalKind::User
|
||||
};
|
||||
// The gateway admitted this identity against the credential IAM returned, so the kinds
|
||||
// agree unless IAM's record changed in between; refuse rather than judge a stale reading.
|
||||
if has_session != (kind == PrincipalKind::Temporary) {
|
||||
return Err(PolicyError::because(SESSION_MISMATCH));
|
||||
}
|
||||
|
||||
let claims = read_claims(kind, &cred, &root)?;
|
||||
let username = match kind {
|
||||
PrincipalKind::Temporary | PrincipalKind::ServiceAccount => cred.parent_user.clone(),
|
||||
PrincipalKind::Root | PrincipalKind::User => cred.access_key.clone(),
|
||||
};
|
||||
let membership_subject = if username.is_empty() { access_key } else { username.as_str() };
|
||||
let group_memberships = self.iam.group_memberships(membership_subject);
|
||||
let is_owner = acts_for_root(&cred, claims.as_ref(), root_access_key);
|
||||
|
||||
Ok(RustfsPrincipal {
|
||||
kind,
|
||||
account: access_key.to_owned(),
|
||||
username,
|
||||
parent_user: cred.parent_user,
|
||||
credential_groups: cred.groups,
|
||||
group_memberships,
|
||||
claims,
|
||||
is_owner,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The claims of the credential's own token. The gateway admitted the presented token only when it
|
||||
/// equals the stored one, so reading the stored token reads the presented one.
|
||||
fn read_claims(
|
||||
kind: PrincipalKind,
|
||||
cred: &IamCredential,
|
||||
root: &RootCredentials,
|
||||
) -> Result<Option<HashMap<String, Value>>, PolicyError> {
|
||||
// As in legacy `check_claims_from_token_with_context`: no token, no claims.
|
||||
if cred.session_token.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let claims = match kind {
|
||||
PrincipalKind::ServiceAccount => {
|
||||
get_claims_from_token_with_secret_allow_missing_exp(&cred.session_token, &cred.secret_key)
|
||||
.map_err(|_| PolicyError::malformed())?
|
||||
}
|
||||
PrincipalKind::Temporary => get_claims_from_token_with_secret(&cred.session_token, root.sts_signing_key())
|
||||
.map_err(|_| PolicyError::malformed())?,
|
||||
PrincipalKind::Root | PrincipalKind::User => return Ok(None),
|
||||
};
|
||||
Ok(if claims.is_empty() { None } else { Some(claims) })
|
||||
}
|
||||
|
||||
/// Classifies an IAM failure without keeping any of its text.
|
||||
fn classify_iam_error(error: &IamError) -> PolicyError {
|
||||
match error {
|
||||
IamError::NoSuchUser(_)
|
||||
| IamError::NoSuchServiceAccount(_)
|
||||
| IamError::NoSuchTempAccount(_)
|
||||
| IamError::NoSuchAccount(_) => PolicyError::because(UNKNOWN_PRINCIPAL),
|
||||
_ => PolicyError::unavailable(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Root ownership as the legacy edge decides it (`rustfs/src/auth.rs` `has_root_access`): the
|
||||
/// credential is root's or acts for root, and neither a session policy nor an external OIDC
|
||||
/// identity narrows it.
|
||||
fn acts_for_root(cred: &IamCredential, claims: Option<&HashMap<String, Value>>, root_access_key: &str) -> bool {
|
||||
(cred.access_key == root_access_key || cred.parent_user == root_access_key)
|
||||
&& !claims.is_some_and(|claims| claims.contains_key(SESSION_POLICY_NAME) || is_rustfs_oidc_claims(claims))
|
||||
}
|
||||
|
||||
impl PolicySource for IamPolicySource {
|
||||
fn snapshot<'a>(&'a self, identity: Option<&'a Identity>) -> BoxFuture<'a, Result<PolicySnapshot, PolicyError>> {
|
||||
Box::pin(self.read(identity))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::root::RootCell;
|
||||
use rustfs_credentials::{Credentials as IamCredential, IAM_POLICY_CLAIM_NAME_SA};
|
||||
use rustfs_gateway::SessionBinding;
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::sys::SESSION_POLICY_NAME;
|
||||
use rustfs_iam::utils::generate_jwt;
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const ROOT_ACCESS_KEY: &str = "rootadmin";
|
||||
const ROOT_SECRET: &str = "root-secret-key-0001";
|
||||
/// Deliberately not the root secret, so every STS test proves which key claims are read with.
|
||||
const SIGNING_KEY: &str = "sts-signing-key-0001";
|
||||
const ACCESS_KEY: &str = "AKIDUSER0001";
|
||||
const SECRET: &str = "user-secret-key-0001";
|
||||
const PARENT: &str = "alice";
|
||||
const EXPIRES_AT: i64 = 4_000_000_000;
|
||||
|
||||
type KeyAnswer = Result<(Option<UserIdentity>, bool), IamError>;
|
||||
type Answer = Box<dyn Fn() -> KeyAnswer + Send + Sync>;
|
||||
|
||||
struct FakeIam {
|
||||
ready: bool,
|
||||
answer: Answer,
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl IamKeyLookup for FakeIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, KeyAnswer> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let answer = (self.answer)();
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
impl IamPrincipalLookup for FakeIam {
|
||||
/// Distinct groups per name, so a test can tell which name the source asked about.
|
||||
fn group_memberships(&self, user: &str) -> Vec<String> {
|
||||
match user {
|
||||
PARENT => vec!["parent-group".to_owned()],
|
||||
ACCESS_KEY => vec!["key-group".to_owned()],
|
||||
ROOT_ACCESS_KEY => vec!["root-group".to_owned()],
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fake(ready: bool, answer: impl Fn() -> KeyAnswer + Send + Sync + 'static) -> Arc<FakeIam> {
|
||||
Arc::new(FakeIam {
|
||||
ready,
|
||||
answer: Box::new(answer),
|
||||
calls: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
fn root() -> RootCredentials {
|
||||
RootCredentials::new(ROOT_ACCESS_KEY, ROOT_SECRET, SIGNING_KEY).expect("valid root credentials")
|
||||
}
|
||||
|
||||
fn source(iam: Arc<FakeIam>) -> IamPolicySource {
|
||||
IamPolicySource::new(iam, Arc::new(RootCell::new(Some(root()))))
|
||||
}
|
||||
|
||||
fn user() -> IamCredential {
|
||||
IamCredential {
|
||||
access_key: ACCESS_KEY.to_owned(),
|
||||
secret_key: SECRET.to_owned(),
|
||||
status: "on".to_owned(),
|
||||
groups: Some(vec!["cred-group".to_owned()]),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn sts_token(claims: &[(&str, Value)], secret: &str) -> String {
|
||||
let mut map: HashMap<String, Value> = claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
|
||||
map.insert("exp".to_owned(), Value::from(EXPIRES_AT));
|
||||
generate_jwt(&map, secret).expect("a token")
|
||||
}
|
||||
|
||||
fn sts(parent: &str, token: String) -> IamCredential {
|
||||
IamCredential {
|
||||
session_token: token,
|
||||
parent_user: parent.to_owned(),
|
||||
expiration: Some(OffsetDateTime::from_unix_timestamp(EXPIRES_AT).expect("valid timestamp")),
|
||||
..user()
|
||||
}
|
||||
}
|
||||
|
||||
fn found(cred: IamCredential) -> KeyAnswer {
|
||||
Ok((Some(UserIdentity::new(cred)), true))
|
||||
}
|
||||
|
||||
fn long_term(access_key: &str) -> Identity {
|
||||
Identity::new(access_key).expect("valid access key")
|
||||
}
|
||||
|
||||
fn session(access_key: &str) -> Identity {
|
||||
let binding = SessionBinding::new(crate::STS_SESSION_ISSUER, EXPIRES_AT).expect("valid binding");
|
||||
long_term(access_key).with_session_binding(binding)
|
||||
}
|
||||
|
||||
async fn principal_of(source: &IamPolicySource, identity: &Identity) -> PolicySnapshot {
|
||||
source.snapshot(Some(identity)).await.expect("a snapshot")
|
||||
}
|
||||
|
||||
fn payload(snapshot: &PolicySnapshot) -> &RustfsPrincipal {
|
||||
snapshot.get::<RustfsPrincipal>().expect("the payload is a RustfsPrincipal")
|
||||
}
|
||||
|
||||
fn refusal(result: Result<PolicySnapshot, PolicyError>) -> PolicyError {
|
||||
match result {
|
||||
Ok(snapshot) => panic!("expected a refusal, got a snapshot (empty: {})", snapshot.is_empty()),
|
||||
Err(error) => error,
|
||||
}
|
||||
}
|
||||
|
||||
/// Compile-time: `RustfsPrincipal` has no `Debug`, so no formatting of a snapshot payload can
|
||||
/// reach its claims. The two blanket impls make the path ambiguous, and so a compile error, as
|
||||
/// soon as `RustfsPrincipal: Debug` holds.
|
||||
#[test]
|
||||
fn policy_principal_has_no_debug() {
|
||||
trait AmbiguousIfDebug<A> {
|
||||
fn some_item() {}
|
||||
}
|
||||
impl<T: ?Sized> AmbiguousIfDebug<()> for T {}
|
||||
impl<T: ?Sized + core::fmt::Debug> AmbiguousIfDebug<u8> for T {}
|
||||
let _ = <RustfsPrincipal as AmbiguousIfDebug<_>>::some_item;
|
||||
}
|
||||
|
||||
/// Positive — a long-term user's facts: itself as username, both group sources, no claims,
|
||||
/// not the owner.
|
||||
#[tokio::test]
|
||||
async fn policy_user_snapshot_carries_iam_facts() {
|
||||
let iam = fake(true, || found(user()));
|
||||
let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
|
||||
let principal = payload(&snapshot);
|
||||
assert!(principal.kind() == PrincipalKind::User);
|
||||
assert_eq!(principal.account(), ACCESS_KEY);
|
||||
assert_eq!(principal.username(), ACCESS_KEY);
|
||||
assert_eq!(principal.parent_user(), "");
|
||||
assert_eq!(principal.credential_groups(), &Some(vec!["cred-group".to_owned()]));
|
||||
assert_eq!(principal.group_memberships(), ["key-group".to_owned()]);
|
||||
assert!(principal.claims().is_none());
|
||||
assert!(!principal.is_owner());
|
||||
}
|
||||
|
||||
/// Positive — an STS credential acts for its parent user, and its claims are the ones its
|
||||
/// token carries, read with the root secret that signed it.
|
||||
#[tokio::test]
|
||||
async fn policy_sts_snapshot_carries_parent_user_and_claims() {
|
||||
let token = sts_token(&[("sub", Value::from("alice-session"))], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(PARENT, token.clone())));
|
||||
let snapshot = principal_of(&source(iam), &session(ACCESS_KEY)).await;
|
||||
let principal = payload(&snapshot);
|
||||
assert!(principal.kind() == PrincipalKind::Temporary);
|
||||
assert_eq!(principal.account(), ACCESS_KEY);
|
||||
assert_eq!(principal.parent_user(), PARENT);
|
||||
assert_eq!(principal.username(), PARENT);
|
||||
let claims = principal.claims().expect("an STS principal has claims");
|
||||
assert_eq!(claims.get("sub"), Some(&Value::from("alice-session")));
|
||||
assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
|
||||
assert!(!principal.is_owner());
|
||||
}
|
||||
|
||||
/// Positive — a service account's claims are read with its own secret, and it acts for its
|
||||
/// parent.
|
||||
#[tokio::test]
|
||||
async fn policy_service_account_snapshot_reads_its_own_token() {
|
||||
let mut token_claims: HashMap<String, Value> = HashMap::new();
|
||||
token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
|
||||
token_claims.insert("parent".to_owned(), Value::from(PARENT));
|
||||
let token = generate_jwt(&token_claims, SECRET).expect("a token");
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user();
|
||||
cred.session_token = token.clone();
|
||||
cred.parent_user = PARENT.to_owned();
|
||||
cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
|
||||
found(cred)
|
||||
});
|
||||
let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
|
||||
let principal = payload(&snapshot);
|
||||
assert!(principal.kind() == PrincipalKind::ServiceAccount);
|
||||
assert_eq!(principal.username(), PARENT);
|
||||
assert_eq!(principal.claims().and_then(|c| c.get("parent")), Some(&Value::from(PARENT)));
|
||||
assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
|
||||
assert!(!principal.is_owner());
|
||||
}
|
||||
|
||||
/// Positive and negative — root is the owner without asking IAM, even before IAM is ready;
|
||||
/// an STS credential of root is the owner too, but not once a session policy narrows it.
|
||||
#[tokio::test]
|
||||
async fn policy_owner_is_root_or_unrestricted_root_session() {
|
||||
let not_ready = fake(false, || found(user()));
|
||||
let snapshot = principal_of(&source(not_ready.clone()), &long_term(ROOT_ACCESS_KEY)).await;
|
||||
let root = payload(&snapshot);
|
||||
assert!(root.kind() == PrincipalKind::Root);
|
||||
assert!(root.is_owner());
|
||||
assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let plain = sts_token(&[], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, plain.clone())));
|
||||
assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
|
||||
let narrowed = sts_token(&[(SESSION_POLICY_NAME, Value::from("e30"))], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, narrowed.clone())));
|
||||
assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
}
|
||||
|
||||
/// Negative — an anonymous caller is an empty snapshot, and IAM is not asked.
|
||||
#[tokio::test]
|
||||
async fn policy_anonymous_is_empty_snapshot() {
|
||||
let iam = fake(true, || found(user()));
|
||||
let snapshot = source(iam.clone()).snapshot(None).await.expect("an anonymous snapshot");
|
||||
assert!(snapshot.is_empty());
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Negative — IAM that is not ready, not initialized, or failing is `unavailable`: never an
|
||||
/// empty snapshot, which an authorizer would judge as an anonymous caller.
|
||||
#[tokio::test]
|
||||
async fn policy_iam_unavailable_is_unavailable() {
|
||||
let not_ready = fake(false, || found(user()));
|
||||
let error = refusal(source(not_ready.clone()).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::unavailable());
|
||||
assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let failures: [fn() -> IamError; 2] = [
|
||||
|| IamError::IamSysNotInitialized,
|
||||
|| IamError::StringError("storage read failed".to_owned()),
|
||||
];
|
||||
for failure in failures {
|
||||
let iam = fake(true, move || Err(failure()));
|
||||
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::unavailable(), "{:?}", failure());
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a principal IAM no longer knows is refused as unknown, not as unavailable and
|
||||
/// not as an empty snapshot.
|
||||
#[tokio::test]
|
||||
async fn policy_unknown_principal_is_refused() {
|
||||
let answers: [fn() -> KeyAnswer; 2] = [|| Ok((None, false)), || Err(IamError::NoSuchUser(ACCESS_KEY.to_owned()))];
|
||||
for answer in answers {
|
||||
let error = refusal(source(fake(true, answer)).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(UNKNOWN_PRINCIPAL));
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a disabled or invalid principal is refused rather than given a snapshot.
|
||||
#[tokio::test]
|
||||
async fn policy_disabled_principal_is_refused() {
|
||||
let invalid = fake(true, || Ok((Some(UserIdentity::new(user())), false)));
|
||||
let error = refusal(source(invalid).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
|
||||
|
||||
let off = fake(true, || {
|
||||
let mut cred = user();
|
||||
cred.status = "off".to_owned();
|
||||
found(cred)
|
||||
});
|
||||
let error = refusal(source(off).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
|
||||
}
|
||||
|
||||
/// Negative — an STS token not signed by root yields no claims at all: it is `malformed`.
|
||||
#[tokio::test]
|
||||
async fn policy_sts_token_with_foreign_signature_is_malformed() {
|
||||
let forged = sts_token(&[("sub", Value::from("alice-session"))], "not-the-root-secret");
|
||||
let iam = fake(true, move || found(sts(PARENT, forged.clone())));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::malformed());
|
||||
}
|
||||
|
||||
/// Negative — the identity's credential kind must match what IAM holds: a session identity for
|
||||
/// a long-term key, or a long-term identity for an STS key, is refused.
|
||||
#[tokio::test]
|
||||
async fn policy_session_kind_mismatch_is_refused() {
|
||||
let token = sts_token(&[], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(PARENT, token.clone())));
|
||||
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
|
||||
|
||||
let iam = fake(true, || found(user()));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
|
||||
|
||||
let iam = fake(false, || found(user()));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ROOT_ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
|
||||
}
|
||||
|
||||
fn service_account(parent: &str, extra_claims: &[(&str, Value)]) -> KeyAnswer {
|
||||
let mut token_claims: HashMap<String, Value> = extra_claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
|
||||
token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
|
||||
let mut cred = user();
|
||||
cred.session_token = generate_jwt(&token_claims, SECRET).expect("a token");
|
||||
cred.parent_user = parent.to_owned();
|
||||
cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
|
||||
found(cred)
|
||||
}
|
||||
|
||||
/// Negative — a valid-flagged identity whose secret is empty or short gets no snapshot.
|
||||
#[tokio::test]
|
||||
async fn policy_short_secret_principal_is_refused() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user();
|
||||
cred.secret_key = weak.to_owned();
|
||||
found(cred)
|
||||
});
|
||||
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL), "{weak:?}");
|
||||
}
|
||||
for usable in ["12345678", "abc\0defg"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user();
|
||||
cred.secret_key = usable.to_owned();
|
||||
found(cred)
|
||||
});
|
||||
assert!(payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).kind() == PrincipalKind::User);
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a root session issued through external OIDC is not the owner, though its parent
|
||||
/// is root; the same session without the OIDC issuer is.
|
||||
#[tokio::test]
|
||||
async fn policy_oidc_root_session_is_not_owner() {
|
||||
let oidc = sts_token(
|
||||
&[
|
||||
("iss", Value::from("rustfs-oidc")),
|
||||
("oidc_provider", Value::from("corp-idp")),
|
||||
("sub", Value::from("alice@corp")),
|
||||
],
|
||||
SIGNING_KEY,
|
||||
);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, oidc.clone())));
|
||||
assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
|
||||
let local = sts_token(
|
||||
&[
|
||||
("iss", Value::from("rustfs-sts")),
|
||||
("oidc_provider", Value::from("corp-idp")),
|
||||
("sub", Value::from("alice@corp")),
|
||||
],
|
||||
SIGNING_KEY,
|
||||
);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, local.clone())));
|
||||
assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
}
|
||||
|
||||
/// Positive and negative — a service account of root is the owner; one narrowed by a session
|
||||
/// policy is not, and neither is a service account of an ordinary user.
|
||||
#[tokio::test]
|
||||
async fn policy_service_account_of_root_is_owner() {
|
||||
let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[]));
|
||||
let principal = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
|
||||
assert!(payload(&principal).kind() == PrincipalKind::ServiceAccount);
|
||||
assert!(payload(&principal).is_owner());
|
||||
|
||||
let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[(SESSION_POLICY_NAME, Value::from("e30"))]));
|
||||
assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
|
||||
|
||||
let iam = fake(true, || service_account(PARENT, &[]));
|
||||
assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
|
||||
}
|
||||
|
||||
/// Negative — STS claims are read with the STS signing key, not with the root secret: a token
|
||||
/// signed with the root secret while the signing key differs is `malformed`.
|
||||
#[tokio::test]
|
||||
async fn policy_sts_claims_use_the_signing_key() {
|
||||
let root_signed = sts_token(&[("sub", Value::from("alice-session"))], ROOT_SECRET);
|
||||
let iam = fake(true, move || found(sts(PARENT, root_signed.clone())));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::malformed());
|
||||
}
|
||||
|
||||
/// Negative — without root credentials no principal can be judged: `unavailable`, never an
|
||||
/// empty snapshot, and IAM is not asked.
|
||||
#[tokio::test]
|
||||
async fn policy_without_root_is_unavailable() {
|
||||
let iam = fake(true, || found(user()));
|
||||
let source = IamPolicySource::new(iam.clone(), Arc::new(RootCell::new(None)));
|
||||
for identity in [long_term(ACCESS_KEY), long_term(ROOT_ACCESS_KEY)] {
|
||||
assert_eq!(refusal(source.snapshot(Some(&identity)).await), PolicyError::unavailable());
|
||||
}
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
type LiveRoot = Arc<std::sync::Mutex<Option<(String, String)>>>;
|
||||
|
||||
/// The primary path: a source that reads a live accessor on every call, as the binary's does,
|
||||
/// with nothing cached in between.
|
||||
fn live_root_source(live: &LiveRoot) -> Arc<dyn RootSource> {
|
||||
let accessor = live.clone();
|
||||
Arc::new(move || {
|
||||
let guard = accessor.lock().ok()?;
|
||||
let (access_key, secret_key) = guard.as_ref()?;
|
||||
RootCredentials::new(access_key, secret_key, SIGNING_KEY).ok().map(Arc::new)
|
||||
})
|
||||
}
|
||||
|
||||
fn set_live(live: &LiveRoot, value: Option<(&str, &str)>) {
|
||||
*live.lock().expect("live root lock") = value.map(|(ak, sk)| (ak.to_owned(), sk.to_owned()));
|
||||
}
|
||||
|
||||
/// Positive and negative — the two adapters read one live root accessor: a root change is seen
|
||||
/// by both on the next request with no second step, the old root key stops being root to
|
||||
/// both, and an uninitialized or unusable root makes both unavailable.
|
||||
#[tokio::test]
|
||||
async fn adapters_share_one_live_root() {
|
||||
use rustfs_gateway::{CredentialLookup, CredentialProvider, ProviderError};
|
||||
|
||||
let live: LiveRoot = Arc::default();
|
||||
set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
|
||||
let iam = fake(true, || Ok((None, false)));
|
||||
let adapters = crate::IamAdapters::new(iam, live_root_source(&live));
|
||||
|
||||
let found = adapters.credentials.lookup(ROOT_ACCESS_KEY).await;
|
||||
assert!(matches!(found, Ok(CredentialLookup::Found(_))));
|
||||
let snapshot = principal_of(&adapters.policy, &long_term(ROOT_ACCESS_KEY)).await;
|
||||
assert!(payload(&snapshot).is_owner());
|
||||
|
||||
set_live(&live, Some(("newroot", "new-root-secret-01")));
|
||||
match adapters.credentials.lookup("newroot").await {
|
||||
Ok(CredentialLookup::Found(credentials)) => assert_eq!(credentials.secret().expose(), b"new-root-secret-01"),
|
||||
_ => panic!("the new root key must resolve to the new secret"),
|
||||
}
|
||||
assert!(matches!(
|
||||
adapters.credentials.lookup(ROOT_ACCESS_KEY).await,
|
||||
Ok(CredentialLookup::NotFound)
|
||||
));
|
||||
assert!(payload(&principal_of(&adapters.policy, &long_term("newroot")).await).is_owner());
|
||||
let old = refusal(adapters.policy.snapshot(Some(&long_term(ROOT_ACCESS_KEY))).await);
|
||||
assert_eq!(old, PolicyError::because(UNKNOWN_PRINCIPAL));
|
||||
|
||||
for unusable in [None, Some(("newroot", "\0\0\0\0\0\0\0\0"))] {
|
||||
set_live(&live, unusable);
|
||||
assert!(matches!(adapters.credentials.lookup("newroot").await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(
|
||||
refusal(adapters.policy.snapshot(Some(&long_term("newroot"))).await),
|
||||
PolicyError::unavailable()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive — `install` builds the authenticator exactly once, over a provider that reads the
|
||||
/// pair's root source.
|
||||
#[tokio::test]
|
||||
async fn adapters_install_hands_over_the_paired_provider_once() {
|
||||
use rustfs_gateway::{CredentialLookup, CredentialProvider, RegionSet, ServiceBuilder, SigV4Authenticator};
|
||||
|
||||
let live: LiveRoot = Arc::default();
|
||||
set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
|
||||
let adapters = crate::IamAdapters::new(fake(true, || Ok((None, false))), live_root_source(&live));
|
||||
let mut handed = Vec::new();
|
||||
let _builder = adapters.install(ServiceBuilder::new(), |provider| {
|
||||
handed.push(provider.clone());
|
||||
SigV4Authenticator::new(provider, RegionSet::new(["us-east-1"]).expect("a region"))
|
||||
});
|
||||
assert_eq!(handed.len(), 1);
|
||||
let found = handed[0].lookup(ROOT_ACCESS_KEY).await;
|
||||
assert!(matches!(found, Ok(CredentialLookup::Found(_))));
|
||||
}
|
||||
}
|
||||
@@ -1,216 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The deployment's root credentials, as every adapter in this crate reads them.
|
||||
//!
|
||||
//! Responsible for: [`RootCredentials`], the validated root key pair plus the key STS session
|
||||
//! tokens are signed with; [`RootSource`], the one live handle both the credential provider and
|
||||
//! the policy source read on every request; and the effective-secret-length rule every secret in
|
||||
//! this crate is held to.
|
||||
//! NOT responsible for: where the root keys come from or when they change. The binary implements
|
||||
//! [`RootSource`] directly over its live root accessor — a closure is enough — so there is no
|
||||
//! second copy of root to keep in step.
|
||||
//! Upstream: the binary's runtime credentials. Downstream: `crate::credentials`, `crate::policy`.
|
||||
//!
|
||||
//! # Why one source
|
||||
//!
|
||||
//! Legacy reads the current root credentials on every request. Two copies taken at assembly could
|
||||
//! disagree, or go stale after a root change, and a key one adapter still treats as root would be
|
||||
//! answered with the root secret or judged as the owner. Both adapters therefore hold the same
|
||||
//! `Arc<dyn RootSource>`, handed to them together by [`crate::IamAdapters::new`].
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::Credentials;
|
||||
|
||||
/// The shortest effective secret key, in bytes, that any credential in this crate may verify a
|
||||
/// signature with. The same bound IAM applies to its own credentials; a signature over a shorter
|
||||
/// (or empty) key is one anybody can compute.
|
||||
pub const MIN_SECRET_KEY_LEN: usize = 8;
|
||||
|
||||
/// The number of bytes of `secret` a signature actually depends on: its length after trailing NUL
|
||||
/// bytes are removed.
|
||||
///
|
||||
/// HMAC pads a key shorter than its block with zero bytes, and SigV4's first key is `"AWS4"`
|
||||
/// followed by the secret, so trailing NULs add nothing: eight NUL bytes sign exactly like the
|
||||
/// empty secret, and `"abc"` followed by five NULs signs like `"abc"`. A secret of NULs only has
|
||||
/// an effective length of zero. An interior NUL is followed by real key bytes, which change the
|
||||
/// padded key, so it counts like any other byte.
|
||||
pub(crate) fn effective_secret_len(secret: &[u8]) -> usize {
|
||||
secret.iter().rposition(|byte| *byte != 0).map_or(0, |last| last + 1)
|
||||
}
|
||||
|
||||
/// Whether `secret` may verify a signature: its [`effective_secret_len`] is at least
|
||||
/// [`MIN_SECRET_KEY_LEN`]. The one rule for root, IAM credentials and the STS signing key.
|
||||
pub(crate) fn is_usable_secret(secret: &[u8]) -> bool {
|
||||
effective_secret_len(secret) >= MIN_SECRET_KEY_LEN
|
||||
}
|
||||
|
||||
/// Why [`RootCredentials::new`] refused its input. Carries no input text.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RootCredentialsError {
|
||||
/// The access key id is not one the gateway can hold.
|
||||
InvalidAccessKey,
|
||||
/// The root secret key fails the effective-length rule.
|
||||
WeakSecretKey,
|
||||
/// The STS token-signing key fails the effective-length rule.
|
||||
WeakSigningKey,
|
||||
}
|
||||
|
||||
impl core::fmt::Display for RootCredentialsError {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::InvalidAccessKey => "root access key is not usable",
|
||||
Self::WeakSecretKey => "root secret key is shorter than the minimum length",
|
||||
Self::WeakSigningKey => "STS token-signing key is shorter than the minimum length",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for RootCredentialsError {}
|
||||
|
||||
/// The root key pair and the STS token-signing key, validated together.
|
||||
///
|
||||
/// Implements no `Debug`; the secrets are readable only inside this crate.
|
||||
pub struct RootCredentials {
|
||||
credentials: Credentials,
|
||||
sts_signing_key: Box<str>,
|
||||
}
|
||||
|
||||
impl RootCredentials {
|
||||
/// Validates the root credentials and the key IAM signs STS session tokens with (today the
|
||||
/// root secret, `rustfs_iam::manager::get_token_signing_key`; pass it separately so a future
|
||||
/// key separation changes no adapter).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`RootCredentialsError`] when either secret fails the effective-length rule (fewer than
|
||||
/// [`MIN_SECRET_KEY_LEN`] bytes once trailing NUL bytes are removed) or the access key id is not
|
||||
/// one the gateway can hold.
|
||||
pub fn new(access_key: &str, secret_key: &str, sts_signing_key: &str) -> Result<Self, RootCredentialsError> {
|
||||
if !is_usable_secret(secret_key.as_bytes()) {
|
||||
return Err(RootCredentialsError::WeakSecretKey);
|
||||
}
|
||||
if !is_usable_secret(sts_signing_key.as_bytes()) {
|
||||
return Err(RootCredentialsError::WeakSigningKey);
|
||||
}
|
||||
let credentials =
|
||||
Credentials::new(access_key, secret_key.as_bytes()).map_err(|_| RootCredentialsError::InvalidAccessKey)?;
|
||||
Ok(Self {
|
||||
credentials,
|
||||
sts_signing_key: sts_signing_key.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The root access key id, a public identifier.
|
||||
#[must_use]
|
||||
pub fn access_key(&self) -> &str {
|
||||
self.credentials.identity().access_key_id()
|
||||
}
|
||||
|
||||
pub(crate) fn credentials(&self) -> &Credentials {
|
||||
&self.credentials
|
||||
}
|
||||
|
||||
pub(crate) fn sts_signing_key(&self) -> &str {
|
||||
&self.sts_signing_key
|
||||
}
|
||||
}
|
||||
|
||||
/// The live root credentials, read on every request by both adapters.
|
||||
///
|
||||
/// The binary implements it directly over RustFS's live root accessor, so a root change needs no
|
||||
/// second step. Any `Fn() -> Option<Arc<RootCredentials>>` is a source; for example, over the
|
||||
/// server context:
|
||||
///
|
||||
/// ```ignore
|
||||
/// let root = move || {
|
||||
/// let cred = ctx.action_credentials().get()?;
|
||||
/// let signing_key = ctx.token_signing_key()?;
|
||||
/// RootCredentials::new(&cred.access_key, &cred.secret_key, &signing_key).ok().map(Arc::new)
|
||||
/// };
|
||||
/// let adapters = IamAdapters::new(iam, Arc::new(root));
|
||||
/// ```
|
||||
///
|
||||
/// Root credentials that fail validation read as `None`, which both adapters answer as
|
||||
/// unavailable.
|
||||
pub trait RootSource: Send + Sync + 'static {
|
||||
/// The current root credentials; `None` while they are not initialized, which both adapters
|
||||
/// answer as unavailable rather than guessing.
|
||||
fn current(&self) -> Option<Arc<RootCredentials>>;
|
||||
}
|
||||
|
||||
impl<F> RootSource for F
|
||||
where
|
||||
F: Fn() -> Option<Arc<RootCredentials>> + Send + Sync + 'static,
|
||||
{
|
||||
fn current(&self) -> Option<Arc<RootCredentials>> {
|
||||
self()
|
||||
}
|
||||
}
|
||||
|
||||
/// Test helper: a [`RootSource`] holding fixed root credentials. Production code reads the live
|
||||
/// accessor instead, so that there is no copy of root to keep in step.
|
||||
#[cfg(test)]
|
||||
pub(crate) struct RootCell(Option<Arc<RootCredentials>>);
|
||||
|
||||
#[cfg(test)]
|
||||
impl RootCell {
|
||||
pub(crate) fn new(root: Option<RootCredentials>) -> Self {
|
||||
Self(root.map(Arc::new))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl RootSource for RootCell {
|
||||
fn current(&self) -> Option<Arc<RootCredentials>> {
|
||||
self.0.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Negative — an empty or short root secret is refused at construction, so no source can ever
|
||||
/// hand out a root key anybody could sign with; eight bytes is accepted.
|
||||
#[test]
|
||||
fn root_short_secret_is_refused() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0", "abcdefg\0"] {
|
||||
let refused = RootCredentials::new("rootadmin", weak, "signing-key-0001");
|
||||
assert!(matches!(refused, Err(RootCredentialsError::WeakSecretKey)), "{} bytes", weak.len());
|
||||
}
|
||||
assert!(RootCredentials::new("rootadmin", "12345678", "signing-key-0001").is_ok());
|
||||
}
|
||||
|
||||
/// Positive — an interior NUL is followed by real key bytes and counts: `"abc\0defg"` has an
|
||||
/// effective length of eight and is accepted, as is the same secret with trailing NULs.
|
||||
#[test]
|
||||
fn root_interior_nul_counts_toward_length() {
|
||||
for secret in ["abc\0defg", "abc\0defg\0\0"] {
|
||||
assert!(RootCredentials::new("rootadmin", secret, "signing-key-0001").is_ok(), "{secret:?}");
|
||||
assert!(RootCredentials::new("rootadmin", "root-secret-key-0001", secret).is_ok(), "{secret:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — an empty or short STS signing key is refused at construction.
|
||||
#[test]
|
||||
fn root_short_signing_key_is_refused() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
|
||||
let refused = RootCredentials::new("rootadmin", "root-secret-key-0001", weak);
|
||||
assert!(matches!(refused, Err(RootCredentialsError::WeakSigningKey)), "{} bytes", weak.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The request id RustFS's outer layer minted, handed to the gateway as its own when that is safe.
|
||||
//!
|
||||
//! Responsible for: reading the id from the context the outer layer put in the request's
|
||||
//! extensions ([`RequestIdCarrier`]) and, when it is inside the gateway's identifier alphabet and
|
||||
//! length, inserting it as the gateway's [`HostRequestId`].
|
||||
//! NOT responsible for: minting any id, refusing any request (nothing here can fail a request),
|
||||
//! choosing the response headers (the gateway writes `x-amz-request-id` from the id it was
|
||||
//! handed, and RustFS's outer layer overwrites that header on S3 paths), or the layer that mints
|
||||
//! (`rustfs/src/server/layer.rs`). It never reads a header itself.
|
||||
//! Upstream: RustFS's `RequestContext`, which the binary makes a [`RequestIdCarrier`]. Downstream:
|
||||
//! the gateway service, which reads the [`HostRequestId`] extension once, at the top.
|
||||
//!
|
||||
//! # Whose id it is
|
||||
//!
|
||||
//! On S3 paths the outer layer mints a server-owned UUID. On admin, STS and console paths the
|
||||
//! context copies the caller's `x-request-id` (or `x-amz-request-id`), as legacy does, so there
|
||||
//! the id can be caller-chosen. Either way the id is attached only when it is inert, that is 1 to
|
||||
//! 64 bytes of ASCII letters, digits, `-`, `.`, `_` and `~`, which is safe in a header, an XML
|
||||
//! document and a log line. Anything else is not attached and the request is **forwarded
|
||||
//! unchanged**: the gateway then mints its own id. No request is ever refused because of its id.
|
||||
//!
|
||||
//! # Known difference when nothing is attached
|
||||
//!
|
||||
//! The gateway answers with the id it minted (error body `<RequestId>`, its events and audit
|
||||
//! records) while, on S3 paths, the outer layer overwrites the response header with RustFS's id.
|
||||
//! So for such a request the header and the error body name different ids. That needs a caller-
|
||||
//! chosen id outside the alphabet, or a request that never passed the outer layer.
|
||||
//!
|
||||
//! # Why this is not a `TraceSource`
|
||||
//!
|
||||
//! The gateway's `TraceSource::mint` takes no request, so a source cannot read the id RustFS
|
||||
//! already minted; the gateway's documented path for a host that identifies its own requests is
|
||||
//! the [`HostRequestId`] extension, and this module fills it.
|
||||
//!
|
||||
//! # Why a carrier trait
|
||||
//!
|
||||
//! `RequestContext` lives in the `rustfs` binary crate, which a library crate cannot depend on.
|
||||
//! The binary implements [`RequestIdCarrier`] for it, and this module looks it up in the
|
||||
//! extensions by that type.
|
||||
|
||||
use rustfs_gateway::HostRequestId;
|
||||
|
||||
/// A type in the request extensions that holds the id the outer layer minted or propagated.
|
||||
pub trait RequestIdCarrier: Send + Sync + 'static {
|
||||
/// The canonical request id: the value RustFS logs and the response should carry.
|
||||
fn request_id(&self) -> &str;
|
||||
}
|
||||
|
||||
/// What [`attach_host_request_id`] did. Ignoring it is harmless: every outcome forwards the request.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum AttachOutcome {
|
||||
/// The carrier's id was inserted as the request's [`HostRequestId`].
|
||||
Attached,
|
||||
/// The request carries no context from the outer layer; nothing was inserted.
|
||||
NoContext,
|
||||
/// The carrier's id is empty, too long, or outside the gateway's alphabet; nothing was
|
||||
/// inserted and the id was not repaired.
|
||||
UnusableId,
|
||||
}
|
||||
|
||||
/// The gateway's [`HostRequestId`] for the id `carrier` holds, or `None` when it is not inert.
|
||||
#[must_use]
|
||||
pub fn host_request_id<C: RequestIdCarrier + ?Sized>(carrier: &C) -> Option<HostRequestId> {
|
||||
HostRequestId::new(carrier.request_id()).ok()
|
||||
}
|
||||
|
||||
/// Reads the `C` the outer layer put in `request`'s extensions and, when its id is inert, inserts
|
||||
/// it as the request's [`HostRequestId`].
|
||||
///
|
||||
/// Never refuses the request: the caller forwards it whatever the outcome.
|
||||
pub fn attach_host_request_id<C: RequestIdCarrier, B>(request: &mut http::Request<B>) -> AttachOutcome {
|
||||
let Some(carrier) = request.extensions().get::<C>() else {
|
||||
return AttachOutcome::NoContext;
|
||||
};
|
||||
let Some(id) = host_request_id(carrier) else {
|
||||
return AttachOutcome::UnusableId;
|
||||
};
|
||||
request.extensions_mut().insert(id);
|
||||
AttachOutcome::Attached
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Ctx(String);
|
||||
|
||||
impl RequestIdCarrier for Ctx {
|
||||
fn request_id(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
const UUID: &str = "7c9e6679-7425-40de-944b-e07fc1f90ae7";
|
||||
|
||||
fn request_with(id: &str) -> http::Request<()> {
|
||||
let mut request = http::Request::new(());
|
||||
request.extensions_mut().insert(Ctx(id.to_owned()));
|
||||
request
|
||||
}
|
||||
|
||||
fn handed_over(request: &http::Request<()>) -> Option<String> {
|
||||
request
|
||||
.extensions()
|
||||
.get::<HostRequestId>()
|
||||
.map(|id| id.request_id().as_str().to_owned())
|
||||
}
|
||||
|
||||
fn attach(request: &mut http::Request<()>) -> AttachOutcome {
|
||||
attach_host_request_id::<Ctx, _>(request)
|
||||
}
|
||||
|
||||
/// Positive: the id RustFS minted is the id the gateway is handed, byte for byte.
|
||||
#[test]
|
||||
fn the_minted_id_is_handed_over_unchanged() {
|
||||
let mut request = request_with(UUID);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached);
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(UUID));
|
||||
}
|
||||
|
||||
/// Positive: every shape RustFS mints is accepted, including its two fallbacks.
|
||||
#[test]
|
||||
fn every_shape_rustfs_mints_is_accepted() {
|
||||
for id in [UUID, "trace-0123456789abcdef", "req-0123456789abcdef"] {
|
||||
let mut request = request_with(id);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached, "{id}");
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(id));
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive: a caller-supplied id that is inert (admin and STS paths copy it) is attached, and
|
||||
/// the longest allowed one is accepted.
|
||||
#[test]
|
||||
fn an_inert_caller_supplied_id_is_attached() {
|
||||
let longest = "a".repeat(64);
|
||||
for id in ["abc", "my.trace_id~1-2", longest.as_str()] {
|
||||
let mut request = request_with(id);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached, "{id}");
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(id));
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive: two requests keep their own ids; nothing is shared or minted between them.
|
||||
#[test]
|
||||
fn each_request_keeps_its_own_id() {
|
||||
let mut first = request_with("first-id");
|
||||
let mut second = request_with("second-id");
|
||||
attach(&mut first);
|
||||
attach(&mut second);
|
||||
assert_eq!(handed_over(&first).as_deref(), Some("first-id"));
|
||||
assert_eq!(handed_over(&second).as_deref(), Some("second-id"));
|
||||
}
|
||||
|
||||
/// Positive: with both a carrier and an `x-amz-request-id` header, the carrier's id wins.
|
||||
#[test]
|
||||
fn the_carrier_wins_over_a_request_id_header() {
|
||||
let mut request = http::Request::builder()
|
||||
.header("x-amz-request-id", "caller-chosen")
|
||||
.header("x-request-id", "caller-chosen")
|
||||
.body(())
|
||||
.expect("a request");
|
||||
request.extensions_mut().insert(Ctx(UUID.to_owned()));
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached);
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(UUID));
|
||||
}
|
||||
|
||||
/// Negative: a request the outer layer never saw gets no id, and the request is not refused.
|
||||
#[test]
|
||||
fn a_missing_context_hands_over_nothing() {
|
||||
let mut request = http::Request::new(());
|
||||
assert_eq!(attach(&mut request), AttachOutcome::NoContext);
|
||||
assert_eq!(handed_over(&request), None);
|
||||
}
|
||||
|
||||
/// Negative: a header named like the id is not the id; only the context is read.
|
||||
#[test]
|
||||
fn a_request_id_header_alone_is_not_read() {
|
||||
let mut request = http::Request::builder()
|
||||
.header("x-amz-request-id", "caller-chosen")
|
||||
.header("x-request-id", "caller-chosen")
|
||||
.body(())
|
||||
.expect("a request");
|
||||
assert_eq!(attach(&mut request), AttachOutcome::NoContext);
|
||||
assert_eq!(handed_over(&request), None);
|
||||
}
|
||||
|
||||
/// Negative: an id outside the alphabet or the length is never attached or repaired, and the
|
||||
/// outcome says so rather than failing the request.
|
||||
#[test]
|
||||
fn an_id_the_gateway_cannot_carry_is_not_attached() {
|
||||
let too_long = "a".repeat(65);
|
||||
for id in [
|
||||
"",
|
||||
"has space",
|
||||
"line\r\nbreak",
|
||||
"<RequestId>",
|
||||
"quo\"te",
|
||||
"Root=1-5759e988",
|
||||
too_long.as_str(),
|
||||
] {
|
||||
let mut request = request_with(id);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::UnusableId, "{id:?}");
|
||||
assert_eq!(handed_over(&request), None, "{id:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: the unusable id is not repaired into a shorter or cleaned-up one.
|
||||
#[test]
|
||||
fn host_request_id_does_not_repair() {
|
||||
assert!(host_request_id(&Ctx("has space".to_owned())).is_none());
|
||||
assert!(host_request_id(&Ctx(format!("{UUID} "))).is_none());
|
||||
assert!(host_request_id(&Ctx(UUID.to_owned())).is_some());
|
||||
}
|
||||
}
|
||||
@@ -843,7 +843,7 @@ mod tests {
|
||||
};
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
crate::app::test_seams::list_objects_v2(&DefaultBucketUsecase::from_global(), request),
|
||||
DefaultBucketUsecase::from_global().execute_list_objects_v2(request),
|
||||
)
|
||||
.await
|
||||
.expect("listing must complete within its bounded source budget")
|
||||
@@ -1241,23 +1241,19 @@ mod tests {
|
||||
let mut list = get.clone().map_input(|_| input.clone());
|
||||
list.uri = http::Uri::from_static("/?list-type=2");
|
||||
let usecase = crate::app::object::DefaultObjectUsecase::from_global();
|
||||
let get_error =
|
||||
tokio::time::timeout(Duration::from_secs(10), crate::app::test_seams::get_object(&usecase, get))
|
||||
.await
|
||||
.expect("GET stays local")
|
||||
.expect_err("local object is absent");
|
||||
let get_error = tokio::time::timeout(Duration::from_secs(10), usecase.execute_get_object(get))
|
||||
.await
|
||||
.expect("GET stays local")
|
||||
.expect_err("local object is absent");
|
||||
assert_eq!(*get_error.code(), S3ErrorCode::NoSuchKey);
|
||||
let head_error = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
crate::app::test_seams::head_object(&usecase, head),
|
||||
)
|
||||
.await
|
||||
.expect("HEAD stays local")
|
||||
.expect_err("local object is absent");
|
||||
let head_error = tokio::time::timeout(Duration::from_secs(10), usecase.execute_head_object(head))
|
||||
.await
|
||||
.expect("HEAD stays local")
|
||||
.expect_err("local object is absent");
|
||||
assert_eq!(*head_error.code(), S3ErrorCode::NoSuchKey);
|
||||
let listing = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
crate::app::test_seams::list_objects_v2(&DefaultBucketUsecase::from_global(), list),
|
||||
DefaultBucketUsecase::from_global().execute_list_objects_v2(list),
|
||||
)
|
||||
.await
|
||||
.expect("LIST stays local")
|
||||
|
||||
@@ -3912,27 +3912,21 @@ mod tests {
|
||||
req
|
||||
}
|
||||
|
||||
fn usecase_method_source<'a>(source: &'a str, method: &str) -> &'a str {
|
||||
let public_start_marker = format!("pub async fn {method}");
|
||||
let private_start_marker = format!("async fn {method}");
|
||||
let (start, marker_len) = if let Some(start) = source.find(&public_start_marker) {
|
||||
(start, public_start_marker.len())
|
||||
} else {
|
||||
let start = source.find(&private_start_marker).expect("method should exist");
|
||||
(start, private_start_marker.len())
|
||||
};
|
||||
let rest = &source[start + marker_len..];
|
||||
let end = [rest.find("\n pub async fn "), rest.find("\n async fn ")]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.min()
|
||||
.unwrap_or(rest.len());
|
||||
&rest[..end]
|
||||
/// Production text of this file plus `bucket_usecase/`, where a split moves its methods.
|
||||
fn bucket_usecase_sources() -> crate::guard_sources::SourceSet {
|
||||
crate::guard_sources::SourceSet::production("src/app/bucket_usecase.rs", "src/app/bucket_usecase")
|
||||
}
|
||||
|
||||
/// The one `async fn method` of the set, through its own closing line, so a neighbour of any
|
||||
/// visibility or a test module can never satisfy a check about it.
|
||||
fn usecase_method_source<'a>(source: &'a crate::guard_sources::SourceSet, method: &str) -> &'a str {
|
||||
source.method_body(method)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_create_and_delete_use_admitted_fresh_tasks() {
|
||||
let source = include_str!("bucket_usecase.rs");
|
||||
let sources = bucket_usecase_sources();
|
||||
let source = &sources;
|
||||
for (method, operation, inner_call) in [
|
||||
(
|
||||
"execute_create_bucket",
|
||||
@@ -3957,7 +3951,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn bucket_metadata_config_changes_notify_peer_metadata_reload() {
|
||||
let source = include_str!("bucket_usecase.rs");
|
||||
let sources = bucket_usecase_sources();
|
||||
let source = &sources;
|
||||
for (method, operation, scanner_maintenance_change) in [
|
||||
("execute_delete_bucket_policy", "delete bucket policy", false),
|
||||
("execute_put_bucket_policy", "put bucket policy", false),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,384 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared fixtures for the lifecycle integration tests; owned by no operation batch.
|
||||
|
||||
use super::*;
|
||||
|
||||
static GLOBAL_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>)> = OnceLock::new();
|
||||
|
||||
pub(super) const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
pub(super) const RUSTFS_EXPECTED_CURRENT_VERSION_ID: &str = "x-rustfs-expected-current-version-id";
|
||||
|
||||
fn init_tracing() {
|
||||
rustfs_test_utils::init_tracing();
|
||||
}
|
||||
|
||||
pub(super) async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
init_tracing();
|
||||
|
||||
if let Some((paths, ecstore)) = GLOBAL_ENV.get() {
|
||||
return (paths.clone(), ecstore.clone());
|
||||
}
|
||||
|
||||
// Production startup installs this proof through the notification-system
|
||||
// fleet probe. This isolated single-node harness constructs ECStore
|
||||
// directly, so model that completed all-v6 probe before any journal permit
|
||||
// or background expiry worker can exist.
|
||||
install_all_v6_fleet_capability_proof();
|
||||
|
||||
let test_base_dir = format!("/tmp/rustfs_app_lifecycle_test_{}", Uuid::new_v4());
|
||||
let temp_dir = PathBuf::from(&test_base_dir);
|
||||
if temp_dir.exists() {
|
||||
fs::remove_dir_all(&temp_dir).await.ok();
|
||||
}
|
||||
fs::create_dir_all(&temp_dir).await.unwrap();
|
||||
|
||||
let disk_paths = vec![
|
||||
temp_dir.join("disk1"),
|
||||
temp_dir.join("disk2"),
|
||||
temp_dir.join("disk3"),
|
||||
temp_dir.join("disk4"),
|
||||
];
|
||||
|
||||
for disk_path in &disk_paths {
|
||||
fs::create_dir_all(disk_path).await.unwrap();
|
||||
}
|
||||
|
||||
let mut endpoints = Vec::new();
|
||||
for (i, disk_path) in disk_paths.iter().enumerate() {
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().unwrap()).unwrap();
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(i);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
|
||||
let pool_endpoints = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "test".to_string(),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
};
|
||||
|
||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||
|
||||
super::super::storage_api::test::runtime::init_local_disks(endpoint_pools.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let server_addr: std::net::SocketAddr = "127.0.0.1:9003".parse().unwrap();
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let buckets_list = ecstore
|
||||
.list_bucket(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
|
||||
// The usecases resolve the object store through the ambient AppContext
|
||||
// since the backlog#1052 InstanceContext migration; without one installed
|
||||
// every store-touching usecase call fails with InternalError "Not init".
|
||||
// Publish a default-interface context wrapping this test store.
|
||||
crate::app::runtime_sources::install_test_app_context(ecstore.clone()).await;
|
||||
|
||||
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
|
||||
|
||||
(disk_paths, ecstore)
|
||||
}
|
||||
|
||||
pub(super) async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
|
||||
(**ecstore)
|
||||
.make_bucket(
|
||||
bucket_name,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("Failed to create test bucket");
|
||||
}
|
||||
|
||||
pub(super) async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) -> ObjectInfo {
|
||||
let mut reader = PutObjReader::from_vec(data.to_vec());
|
||||
(**ecstore)
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to upload test object")
|
||||
}
|
||||
|
||||
pub(super) async fn set_bucket_lifecycle_transition_with_tier(
|
||||
bucket_name: &str,
|
||||
storage_class: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let lifecycle_xml = format!(
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>test-rule</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter>
|
||||
<Prefix>test/</Prefix>
|
||||
</Filter>
|
||||
<Transition>
|
||||
<Days>0</Days>
|
||||
<StorageClass>{storage_class}</StorageClass>
|
||||
</Transition>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
);
|
||||
|
||||
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn set_bucket_object_lock_enabled(bucket_name: &str) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let object_lock_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ObjectLockConfiguration>
|
||||
<ObjectLockEnabled>Enabled</ObjectLockEnabled>
|
||||
</ObjectLockConfiguration>"#;
|
||||
metadata_sys::update(bucket_name, OBJECT_LOCK_CONFIG, object_lock_xml.as_bytes().to_vec()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Register the shared [`MockWarmBackend`] into this instance's tier config
|
||||
/// manager. Thin wrapper over the shared `register_mock_tier` helper
|
||||
/// (rustfs/backlog#1148 ilm-6).
|
||||
pub(super) async fn register_mock_tier(tier_name: &str) -> MockWarmBackend {
|
||||
register_mock_tier_util(¤t_tier_config_handle(), tier_name).await
|
||||
}
|
||||
|
||||
pub(super) async fn wait_for_transition(
|
||||
ecstore: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: Duration,
|
||||
) -> Result<ObjectInfo, String> {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
|
||||
loop {
|
||||
let observed = (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await;
|
||||
let observed = match observed {
|
||||
Ok(info) if info.transitioned_object.status == "complete" => return Ok(info),
|
||||
other => other,
|
||||
};
|
||||
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
let last_observation = match observed {
|
||||
Ok(info) => format!("transitioned_status={:?}", info.transitioned_object.status),
|
||||
Err(error) => format!("metadata_error={:?}", error.code()),
|
||||
};
|
||||
let state = get_global_transition_state();
|
||||
return Err(format!(
|
||||
"transition did not complete within {timeout:?}; {last_observation}; pending={}, active={}, compensation_pending={}, compensation_running={}",
|
||||
state.pending_tasks(),
|
||||
state.active_tasks(),
|
||||
state.compensation_pending_tasks(),
|
||||
state.compensation_running_tasks(),
|
||||
));
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY: this helper is used only by `#[serial]` tests and runs under the single-threaded Tokio
|
||||
// runtime (`worker_threads = 1`), so no concurrent test can mutate process environment during the
|
||||
// `env::set_var` / `env::remove_var` window.
|
||||
#[allow(unsafe_code)]
|
||||
pub(super) async fn with_forced_immediate_enqueue_timeout<F, Fut>(backend: &MockWarmBackend, test_fn: F)
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = ()>,
|
||||
{
|
||||
let transition_state = get_global_transition_state();
|
||||
let missed_before = transition_state.missed_immediate_tasks();
|
||||
let compensation_before = transition_state.compensation_scheduled_tasks();
|
||||
let put_barrier = backend.arm_put_barrier().await;
|
||||
let original = env::var_os(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT);
|
||||
unsafe {
|
||||
env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, "1");
|
||||
}
|
||||
let result = std::panic::AssertUnwindSafe(test_fn()).catch_unwind().await;
|
||||
match original {
|
||||
Some(value) => unsafe {
|
||||
env::set_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT, value);
|
||||
},
|
||||
None => unsafe {
|
||||
env::remove_var(ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT);
|
||||
},
|
||||
}
|
||||
if let Err(err) = result {
|
||||
std::panic::resume_unwind(err);
|
||||
}
|
||||
|
||||
assert!(
|
||||
transition_state.missed_immediate_tasks() > missed_before,
|
||||
"the operation should exercise the forced immediate enqueue failure"
|
||||
);
|
||||
assert!(
|
||||
transition_state.compensation_scheduled_tasks() > compensation_before,
|
||||
"the operation should schedule automatic compensation backfill"
|
||||
);
|
||||
|
||||
// Separate automatic discovery and queue latency from the metadata commit deadline.
|
||||
put_barrier.wait_until_paused().await;
|
||||
assert_eq!(backend.object_count().await, 1, "compensation should store the body before committing");
|
||||
put_barrier.release();
|
||||
}
|
||||
|
||||
pub(super) async fn wait_for_delete_marker(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
|
||||
loop {
|
||||
if let Ok(info) = ecstore.get_object_info(bucket, object, &ObjectOptions::default()).await
|
||||
&& info.delete_marker
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return false;
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
input,
|
||||
method,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn streaming_blob_from_bytes(data: &[u8]) -> StreamingBlob {
|
||||
let body = Bytes::copy_from_slice(data);
|
||||
StreamingBlob::wrap::<_, Infallible>(stream::once(async move { Ok(body) }))
|
||||
}
|
||||
|
||||
pub(super) async fn read_object_bytes(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> Vec<u8> {
|
||||
let mut reader = (**ecstore)
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to read object");
|
||||
let mut buf = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut buf)
|
||||
.await
|
||||
.expect("Failed to drain object reader");
|
||||
buf
|
||||
}
|
||||
|
||||
pub(super) async fn live_object_version_count(ecstore: &Arc<ECStore>, bucket: &str, object: &str) -> usize {
|
||||
let versions = ecstore
|
||||
.clone()
|
||||
.list_object_versions(bucket, object, None, None, None, 1000)
|
||||
.await
|
||||
.expect("Failed to list object versions");
|
||||
|
||||
versions
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|info| info.name == object && !info.delete_marker)
|
||||
.count()
|
||||
}
|
||||
|
||||
pub(super) async fn install_delete_replication_config(
|
||||
bucket: &str,
|
||||
delete_enabled: bool,
|
||||
replica_modifications: bool,
|
||||
required_tag: Option<(&str, &str)>,
|
||||
) {
|
||||
use super::super::storage_api::test::{bucket::utils::serialize, get_global_bucket_metadata_sys, set_bucket_metadata};
|
||||
|
||||
let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized");
|
||||
let metadata = {
|
||||
let sys = sys.read().await;
|
||||
sys.get(bucket)
|
||||
.await
|
||||
.expect("bucket metadata should be cached before replication config injection")
|
||||
};
|
||||
let mut metadata = (*metadata).clone();
|
||||
let target = "arn:aws:s3:::target-bucket";
|
||||
let delete_status = if delete_enabled {
|
||||
Status::ENABLED.to_string()
|
||||
} else {
|
||||
Status::DISABLED.to_string()
|
||||
};
|
||||
|
||||
metadata.versioning_config_xml = b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec();
|
||||
metadata.versioning_config = Some(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
});
|
||||
let filter = required_tag.map(|(key, value)| PersistedReplicationFilter {
|
||||
tag: Some(PersistedReplicationTag {
|
||||
key: Some(key.to_string()),
|
||||
value: Some(value.to_string()),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
let source_selection_criteria = replica_modifications.then(|| PersistedSourceSelectionCriteria {
|
||||
replica_modifications: Some(PersistedReplicationStatus {
|
||||
status: Status::ENABLED.to_string(),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let replication_config = PersistedReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![PersistedReplicationRule {
|
||||
delete_marker_replication: Some(PersistedOptionalReplicationStatus {
|
||||
status: Some(Status::DISABLED.to_string()),
|
||||
}),
|
||||
delete_replication: Some(PersistedReplicationStatus { status: delete_status }),
|
||||
destination: PersistedReplicationDestination {
|
||||
bucket: target.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter,
|
||||
id: Some("delete".to_string()),
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria,
|
||||
status: Status::ENABLED.to_string(),
|
||||
}],
|
||||
};
|
||||
metadata.replication_config_xml = serialize(&replication_config).expect("replication test config should serialize");
|
||||
metadata.replication_config = Some(replication_config);
|
||||
set_bucket_metadata(bucket.to_string(), metadata)
|
||||
.await
|
||||
.expect("delete replication metadata should be installed for the request test");
|
||||
}
|
||||
@@ -1,654 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! CopyObject tests, including tests that also PUT (batch E owns CopyObject).
|
||||
|
||||
use super::common::*;
|
||||
use super::*;
|
||||
use crate::app::test_seams;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn copy_object_if_none_match_existing_destination_returns_precondition_failed() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let src_bucket = format!("test-copy-if-none-match-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let dst_bucket = format!("test-copy-if-none-match-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let src_object = "test/source.txt";
|
||||
let dst_object = "test/destination.txt";
|
||||
let src_payload = b"conditional copy source payload";
|
||||
let dst_payload = b"conditional copy destination payload";
|
||||
|
||||
create_test_bucket(&ecstore, src_bucket.as_str()).await;
|
||||
create_test_bucket(&ecstore, dst_bucket.as_str()).await;
|
||||
let _ = upload_test_object(&ecstore, src_bucket.as_str(), src_object, src_payload).await;
|
||||
let _ = upload_test_object(&ecstore, dst_bucket.as_str(), dst_object, dst_payload).await;
|
||||
|
||||
let dst_info = ecstore
|
||||
.get_object_info(dst_bucket.as_str(), dst_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to fetch destination object info");
|
||||
let dst_etag = dst_info.etag.expect("destination object should have an ETag");
|
||||
let initial_versions = live_object_version_count(&ecstore, dst_bucket.as_str(), dst_object).await;
|
||||
|
||||
for (condition, expected_code) in [
|
||||
(ETagCondition::Any, S3ErrorCode::PreconditionFailed),
|
||||
(ETagCondition::ETag(ETag::Strong(dst_etag)), S3ErrorCode::NotImplemented),
|
||||
(
|
||||
ETagCondition::ETag(ETag::Strong("different-etag".to_string())),
|
||||
S3ErrorCode::NotImplemented,
|
||||
),
|
||||
] {
|
||||
let header = condition.to_http_header().expect("conditional COPY header should encode");
|
||||
// Cover header-only callers and the DTO-plus-header shape of parsed HTTP requests.
|
||||
for if_none_match in [None, Some(condition)] {
|
||||
let has_dto = if_none_match.is_some();
|
||||
let copy_input = CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: src_bucket.clone().into(),
|
||||
key: src_object.to_string().into(),
|
||||
version_id: None,
|
||||
})
|
||||
.bucket(dst_bucket.clone())
|
||||
.key(dst_object.to_string())
|
||||
.if_none_match(if_none_match)
|
||||
.build()
|
||||
.expect("conditional COPY input should build");
|
||||
let mut req = build_request(copy_input, Method::PUT);
|
||||
req.headers.insert(IF_NONE_MATCH, header.clone());
|
||||
|
||||
let err = Box::pin(usecase.execute_copy_object(req))
|
||||
.await
|
||||
.expect_err("conditional COPY must reject the replacement");
|
||||
assert_eq!(err.code(), &expected_code, "header={header:?}, has_dto={has_dto}");
|
||||
assert_eq!(
|
||||
read_object_bytes(&ecstore, dst_bucket.as_str(), dst_object).await,
|
||||
dst_payload,
|
||||
"failed conditional CopyObject must not overwrite the current destination object"
|
||||
);
|
||||
assert_eq!(
|
||||
live_object_version_count(&ecstore, dst_bucket.as_str(), dst_object).await,
|
||||
initial_versions,
|
||||
"failed conditional CopyObject must not create a version"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state object-version integration test: runs serialized in the CI ILM Integration (serial) lane"]
|
||||
async fn undo_copy_requires_the_observed_current_version() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let fs = FS::new();
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-undo-copy-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
let put = |payload: &'static [u8]| {
|
||||
PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.body(Some(streaming_blob_from_bytes(payload)))
|
||||
.content_length(Some(payload.len() as i64))
|
||||
.build()
|
||||
.unwrap()
|
||||
};
|
||||
let original = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"original"), Method::PUT)))
|
||||
.await
|
||||
.expect("initial version should be written")
|
||||
.output
|
||||
.version_id
|
||||
.expect("versioned PUT should return a version ID");
|
||||
let observed_current = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"replacement"), Method::PUT)))
|
||||
.await
|
||||
.expect("replacement version should be written")
|
||||
.output
|
||||
.version_id
|
||||
.expect("versioned PUT should return a version ID");
|
||||
|
||||
let copy = || {
|
||||
CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: bucket.clone().into(),
|
||||
key: object.to_string().into(),
|
||||
version_id: Some(original.clone().into()),
|
||||
})
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.unwrap()
|
||||
};
|
||||
let mut stale_req = build_request(copy(), Method::PUT);
|
||||
stale_req.headers.insert(
|
||||
RUSTFS_EXPECTED_CURRENT_VERSION_ID,
|
||||
HeaderValue::from_str(&Uuid::new_v4().to_string()).unwrap(),
|
||||
);
|
||||
let versions_before = live_object_version_count(&ecstore, bucket.as_str(), object).await;
|
||||
let err = Box::pin(usecase.execute_copy_object(stale_req))
|
||||
.await
|
||||
.expect_err("stale destination must fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::PreconditionFailed);
|
||||
assert_eq!(live_object_version_count(&ecstore, bucket.as_str(), object).await, versions_before);
|
||||
assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, b"replacement");
|
||||
|
||||
let mut matching_req = build_request(copy(), Method::PUT);
|
||||
matching_req
|
||||
.headers
|
||||
.insert(RUSTFS_EXPECTED_CURRENT_VERSION_ID, HeaderValue::from_str(&observed_current).unwrap());
|
||||
Box::pin(usecase.execute_copy_object(matching_req))
|
||||
.await
|
||||
.expect("matching destination version should permit undo copy");
|
||||
assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, b"original");
|
||||
assert_eq!(live_object_version_count(&ecstore, bucket.as_str(), object).await, versions_before + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "global-state object-version integration test: runs serialized in the CI ILM Integration (serial) lane"]
|
||||
async fn undo_copy_serializes_a_concurrent_put_until_after_commit() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("test-undo-race-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
let put = |payload: &'static [u8]| {
|
||||
PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.body(Some(streaming_blob_from_bytes(payload)))
|
||||
.content_length(Some(payload.len() as i64))
|
||||
.build()
|
||||
.unwrap()
|
||||
};
|
||||
let fs = FS::new();
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let original = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"original"), Method::PUT)))
|
||||
.await
|
||||
.expect("original should be written")
|
||||
.output
|
||||
.version_id
|
||||
.expect("original should be versioned");
|
||||
let observed = Box::pin(test_seams::put_object(&usecase, &fs, build_request(put(b"observed"), Method::PUT)))
|
||||
.await
|
||||
.expect("observed should be written")
|
||||
.output
|
||||
.version_id
|
||||
.expect("observed should be versioned");
|
||||
let encoded_object = encode_dir_object(object);
|
||||
let lock = ecstore
|
||||
.new_ns_lock(bucket.as_str(), &encoded_object)
|
||||
.await
|
||||
.expect("namespace lock should be available");
|
||||
let blocker = lock
|
||||
.get_write_lock(Duration::from_secs(10))
|
||||
.await
|
||||
.expect("test should acquire blocker");
|
||||
|
||||
let mut undo_req = build_request(
|
||||
CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: bucket.clone().into(),
|
||||
key: object.to_string().into(),
|
||||
version_id: Some(original.into()),
|
||||
})
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::PUT,
|
||||
);
|
||||
undo_req
|
||||
.headers
|
||||
.insert(RUSTFS_EXPECTED_CURRENT_VERSION_ID, HeaderValue::from_str(&observed).unwrap());
|
||||
|
||||
let undo = tokio::spawn(async move {
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
Box::pin(usecase.execute_copy_object(undo_req)).await
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
let put = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.body(Some(streaming_blob_from_bytes(b"concurrent")))
|
||||
.content_length(Some(10))
|
||||
.build()
|
||||
.unwrap();
|
||||
let concurrent_put = tokio::spawn(async move {
|
||||
let fs = FS::new();
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
Box::pin(test_seams::put_object(&usecase, &fs, build_request(put, Method::PUT))).await
|
||||
});
|
||||
|
||||
drop(blocker);
|
||||
let undo_result = undo.await.expect("undo task should not panic");
|
||||
let put_result = concurrent_put.await.expect("PUT task should not panic");
|
||||
assert!(put_result.is_ok(), "ordinary PUT should eventually succeed: {put_result:?}");
|
||||
if let Err(err) = undo_result {
|
||||
assert_eq!(err.code(), &S3ErrorCode::PreconditionFailed);
|
||||
}
|
||||
assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, b"concurrent");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn copy_object_allows_new_version_for_locked_destination_but_blocks_explicit_overwrite() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let fs = FS::new();
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-copy-object-lock-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let src_object = "test/source.txt";
|
||||
let dst_object = "test/destination.txt";
|
||||
let locked_payload = b"locked destination payload";
|
||||
let source_payload = b"copy source payload";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_object_lock_enabled(bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enable object lock for bucket");
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), src_object, source_payload).await;
|
||||
|
||||
let retain_until = Timestamp::from(time::OffsetDateTime::now_utc().saturating_add(time::Duration::days(1)));
|
||||
let locked_input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(dst_object.to_string())
|
||||
.body(Some(streaming_blob_from_bytes(locked_payload)))
|
||||
.content_length(Some(locked_payload.len() as i64))
|
||||
.object_lock_mode(Some(ObjectLockMode::from_static(ObjectLockMode::COMPLIANCE)))
|
||||
.object_lock_retain_until_date(Some(retain_until))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
Box::pin(test_seams::put_object(&usecase, &fs, build_request(locked_input, Method::PUT)))
|
||||
.await
|
||||
.expect("Failed to upload locked destination object");
|
||||
|
||||
let locked_info = ecstore
|
||||
.get_object_info(bucket.as_str(), dst_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to fetch locked destination object info");
|
||||
let locked_version_id = locked_info
|
||||
.version_id
|
||||
.expect("locked destination should have a version ID")
|
||||
.to_string();
|
||||
|
||||
let copy_input = CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: bucket.clone().into(),
|
||||
key: src_object.to_string().into(),
|
||||
version_id: None,
|
||||
})
|
||||
.bucket(bucket.clone())
|
||||
.key(dst_object.to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let copy_output = Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
|
||||
.await
|
||||
.expect("CopyObject should create a new version over a locked current version");
|
||||
let copied_version_id = copy_output
|
||||
.output
|
||||
.version_id
|
||||
.expect("versioned CopyObject should return the created version ID");
|
||||
|
||||
assert_ne!(copied_version_id, locked_version_id);
|
||||
assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), dst_object).await, source_payload);
|
||||
assert_eq!(live_object_version_count(&ecstore, bucket.as_str(), dst_object).await, 2);
|
||||
|
||||
let explicit_overwrite_input = CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: bucket.clone().into(),
|
||||
key: src_object.to_string().into(),
|
||||
version_id: None,
|
||||
})
|
||||
.bucket(bucket.clone())
|
||||
.key(dst_object.to_string())
|
||||
.version_id(Some(locked_version_id))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let err = Box::pin(usecase.execute_copy_object(build_request(explicit_overwrite_input, Method::PUT)))
|
||||
.await
|
||||
.expect_err("explicit CopyObject overwrite of a locked version should be blocked");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some("Invalid argument: A destination versionId requires replication authorization")
|
||||
);
|
||||
assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), dst_object).await, source_payload);
|
||||
assert_eq!(live_object_version_count(&ecstore, bucket.as_str(), dst_object).await, 2);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn concurrent_reverse_copy_object_does_not_deadlock_with_reader_locks() {
|
||||
temp_env::async_with_vars([(ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
|
||||
let bucket = format!("test-reverse-copy-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object_a = "test/a.bin";
|
||||
let object_b = "test/b.bin";
|
||||
let payload_a = vec![b'a'; 4 * 1024 * 1024];
|
||||
let payload_b = vec![b'b'; 4 * 1024 * 1024];
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object_a, &payload_a).await;
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object_b, &payload_b).await;
|
||||
|
||||
let a_to_b_input = CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: bucket.clone().into(),
|
||||
key: object_a.to_string().into(),
|
||||
version_id: None,
|
||||
})
|
||||
.bucket(bucket.clone())
|
||||
.key(object_b.to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let b_to_a_input = CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: bucket.clone().into(),
|
||||
key: object_b.to_string().into(),
|
||||
version_id: None,
|
||||
})
|
||||
.bucket(bucket.clone())
|
||||
.key(object_a.to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let start = Arc::new(Barrier::new(3));
|
||||
let start_a = start.clone();
|
||||
let a_to_b = tokio::spawn(async move {
|
||||
start_a.wait().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
Box::pin(usecase.execute_copy_object(build_request(a_to_b_input, Method::PUT))).await
|
||||
});
|
||||
let start_b = start.clone();
|
||||
let b_to_a = tokio::spawn(async move {
|
||||
start_b.wait().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
Box::pin(usecase.execute_copy_object(build_request(b_to_a_input, Method::PUT))).await
|
||||
});
|
||||
|
||||
start.wait().await;
|
||||
|
||||
let (a_to_b_result, b_to_a_result) = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let (a_to_b_result, b_to_a_result) = tokio::join!(a_to_b, b_to_a);
|
||||
(
|
||||
a_to_b_result.expect("A-to-B CopyObject task should not panic"),
|
||||
b_to_a_result.expect("B-to-A CopyObject task should not panic"),
|
||||
)
|
||||
})
|
||||
.await
|
||||
.expect("reverse CopyObject operations should not deadlock");
|
||||
|
||||
a_to_b_result.expect("A-to-B CopyObject should succeed");
|
||||
b_to_a_result.expect("B-to-A CopyObject should succeed");
|
||||
|
||||
let final_a = read_object_bytes(&ecstore, bucket.as_str(), object_a).await;
|
||||
let final_b = read_object_bytes(&ecstore, bucket.as_str(), object_b).await;
|
||||
assert!(
|
||||
final_a == payload_a || final_a == payload_b,
|
||||
"object A must contain a complete copied payload"
|
||||
);
|
||||
assert!(
|
||||
final_b == payload_a || final_b == payload_b,
|
||||
"object B must contain a complete copied payload"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn put_and_copy_object_transition_immediately_via_usecases() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let fs = FS::new();
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let put_bucket = format!("test-api-put-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let put_object = "test/object.txt";
|
||||
let put_payload = b"Hello, immediate transition through put API!";
|
||||
|
||||
create_test_bucket(&ecstore, put_bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(put_bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
let put_input = PutObjectInput::builder()
|
||||
.bucket(put_bucket.clone())
|
||||
.key(put_object.to_string())
|
||||
.body(Some(streaming_blob_from_bytes(put_payload)))
|
||||
.content_length(Some(put_payload.len() as i64))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
Box::pin(test_seams::put_object(&usecase, &fs, build_request(put_input, Method::PUT)))
|
||||
.await
|
||||
.expect("Failed to put object through usecase");
|
||||
|
||||
let put_info = wait_for_transition(&ecstore, put_bucket.as_str(), put_object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition immediately after put usecase");
|
||||
|
||||
assert_eq!(put_info.transitioned_object.status, "complete");
|
||||
assert_eq!(put_info.transitioned_object.tier, tier_name);
|
||||
assert!(backend.contains(&put_info.transitioned_object.name).await);
|
||||
|
||||
let src_bucket = format!("test-api-copy-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let dst_bucket = format!("test-api-copy-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let src_object = "test/source.txt";
|
||||
let dst_object = "test/copied.txt";
|
||||
let copy_payload = b"copy object immediate transition through copy API";
|
||||
|
||||
create_test_bucket(&ecstore, src_bucket.as_str()).await;
|
||||
create_test_bucket(&ecstore, dst_bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(dst_bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set destination lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, src_bucket.as_str(), src_object, copy_payload).await;
|
||||
|
||||
let copy_input = CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: src_bucket.clone().into(),
|
||||
key: src_object.to_string().into(),
|
||||
version_id: None,
|
||||
})
|
||||
.bucket(dst_bucket.clone())
|
||||
.key(dst_object.to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
|
||||
.await
|
||||
.expect("Failed to copy object through usecase");
|
||||
|
||||
let copy_info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("copied object should transition immediately after copy usecase");
|
||||
|
||||
assert_eq!(copy_info.transitioned_object.status, "complete");
|
||||
assert_eq!(copy_info.transitioned_object.tier, tier_name);
|
||||
assert!(backend.contains(©_info.transitioned_object.name).await);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn compensation_driven_copy_still_completes_transition() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let src_bucket = format!("test-comp-copy-src-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let dst_bucket = format!("test-comp-copy-dst-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let src_object = "test/source.txt";
|
||||
let dst_object = "test/copied.txt";
|
||||
let payload = b"copy object should still transition after compensation";
|
||||
|
||||
create_test_bucket(&ecstore, src_bucket.as_str()).await;
|
||||
create_test_bucket(&ecstore, dst_bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(dst_bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set destination lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, src_bucket.as_str(), src_object, payload).await;
|
||||
|
||||
let copy_input = CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: src_bucket.clone().into(),
|
||||
key: src_object.to_string().into(),
|
||||
version_id: None,
|
||||
})
|
||||
.bucket(dst_bucket.clone())
|
||||
.key(dst_object.to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
with_forced_immediate_enqueue_timeout(&backend, || async {
|
||||
Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
|
||||
.await
|
||||
.expect("Failed to copy object through usecase");
|
||||
})
|
||||
.await;
|
||||
|
||||
let info = wait_for_transition(&ecstore, dst_bucket.as_str(), dst_object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("copied object should eventually transition after compensation backfill");
|
||||
|
||||
assert_eq!(backend.put_count().await, 1, "compensation should upload the copied version once");
|
||||
assert_eq!(info.transitioned_object.status, "complete");
|
||||
assert_eq!(info.transitioned_object.tier, tier_name);
|
||||
assert!(backend.contains(&info.transitioned_object.name).await);
|
||||
assert_eq!(read_object_bytes(&ecstore, dst_bucket.as_str(), dst_object).await, payload);
|
||||
}
|
||||
|
||||
/// CopyObject creates a new object on the destination key, so it must join
|
||||
/// bucket replication exactly like PutObject: compute one replication decision
|
||||
/// that drives both the persisted pending marker and the post-commit schedule
|
||||
/// (MinIO CopyObjectHandler parity). A count of 0 means a copied object is
|
||||
/// never scheduled for replication and — because the copy path clones the
|
||||
/// source metadata wholesale — the destination silently inherits the source's
|
||||
/// replication bookkeeping, faking a COMPLETED/REPLICA state for an object
|
||||
/// that never replicated. The second half of this test pins the metadata
|
||||
/// cleanup (MinIO filterReplicationStatusMetadata parity).
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn copy_object_computes_replication_decision_and_strips_stale_status() {
|
||||
use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_REPLICA_STATUS, SUFFIX_REPLICA_TIMESTAMP, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, get_str,
|
||||
insert_str,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-copy-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let src_object = "copy/source.txt";
|
||||
let dst_object = "copy/destination.txt";
|
||||
let payload = b"copy replication decision payload";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
// Seed the source object with stale replication bookkeeping, as if it had
|
||||
// already replicated elsewhere (internal status/timestamp under both
|
||||
// compatibility prefixes, replica state, and the surfaced x-amz header).
|
||||
let mut stale_metadata = HashMap::new();
|
||||
insert_str(
|
||||
&mut stale_metadata,
|
||||
SUFFIX_REPLICATION_STATUS,
|
||||
"arn:minio:replication::stale:dst=COMPLETED;".to_string(),
|
||||
);
|
||||
insert_str(&mut stale_metadata, SUFFIX_REPLICATION_TIMESTAMP, "2024-01-01T00:00:00Z".to_string());
|
||||
insert_str(&mut stale_metadata, SUFFIX_REPLICA_STATUS, "REPLICA".to_string());
|
||||
insert_str(&mut stale_metadata, SUFFIX_REPLICA_TIMESTAMP, "2024-01-01T00:00:00Z".to_string());
|
||||
stale_metadata.insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), "COMPLETED".to_string());
|
||||
let mut reader = PutObjReader::from_vec(payload.to_vec());
|
||||
(*ecstore)
|
||||
.put_object(
|
||||
bucket.as_str(),
|
||||
src_object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
user_defined: stale_metadata,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("Failed to upload source object with stale replication metadata");
|
||||
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
|
||||
let copy_input = CopyObjectInput::builder()
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: bucket.clone().into(),
|
||||
key: src_object.into(),
|
||||
version_id: None,
|
||||
})
|
||||
.bucket(bucket.clone())
|
||||
.key(dst_object.to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
Box::pin(usecase.execute_copy_object(build_request(copy_input, Method::PUT)))
|
||||
.await
|
||||
.expect("Failed to copy object through usecase");
|
||||
|
||||
assert_eq!(
|
||||
MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
|
||||
1,
|
||||
"CopyObject must compute the replication decision exactly once; 0 means the copied \
|
||||
object never enters bucket replication (P0-6, MinIO CopyObjectHandler parity)"
|
||||
);
|
||||
|
||||
let copied = ecstore
|
||||
.get_object_info(bucket.as_str(), dst_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to read copied destination object info");
|
||||
assert!(
|
||||
get_str(&copied.user_defined, SUFFIX_REPLICATION_STATUS).is_none(),
|
||||
"destination must not inherit the source's replication status; got {:?}",
|
||||
get_str(&copied.user_defined, SUFFIX_REPLICATION_STATUS)
|
||||
);
|
||||
assert!(
|
||||
get_str(&copied.user_defined, SUFFIX_REPLICATION_TIMESTAMP).is_none(),
|
||||
"destination must not inherit the source's replication timestamp"
|
||||
);
|
||||
assert!(
|
||||
get_str(&copied.user_defined, SUFFIX_REPLICA_STATUS).is_none(),
|
||||
"destination must not inherit the source's replica status"
|
||||
);
|
||||
assert!(
|
||||
!copied
|
||||
.user_defined
|
||||
.keys()
|
||||
.any(|k| k.eq_ignore_ascii_case(AMZ_BUCKET_REPLICATION_STATUS)),
|
||||
"destination must not inherit the surfaced x-amz-replication-status header key"
|
||||
);
|
||||
}
|
||||
@@ -1,923 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! DeleteObject and DeleteObjects tests (batch E owns both).
|
||||
|
||||
use super::common::*;
|
||||
use super::*;
|
||||
use crate::app::test_seams;
|
||||
|
||||
async fn wait_for_object_absence(ecstore: &Arc<ECStore>, bucket: &str, object: &str, timeout: Duration) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
|
||||
loop {
|
||||
if ecstore
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return false;
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state object-version integration test: runs serialized in the CI ILM Integration (serial) lane"]
|
||||
async fn undo_delete_only_removes_the_current_matching_delete_marker() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let fs = FS::new();
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-undo-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
let put = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.body(Some(streaming_blob_from_bytes(b"visible again")))
|
||||
.content_length(Some(13))
|
||||
.build()
|
||||
.unwrap();
|
||||
Box::pin(test_seams::put_object(&usecase, &fs, build_request(put, Method::PUT)))
|
||||
.await
|
||||
.expect("object version should be written");
|
||||
let marker = Box::pin(
|
||||
usecase.execute_delete_object(build_request(
|
||||
DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::DELETE,
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("delete marker should be created")
|
||||
.output
|
||||
.version_id
|
||||
.expect("versioned delete should return the marker version");
|
||||
|
||||
let delete_marker = || {
|
||||
DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.version_id(Some(marker.clone()))
|
||||
.build()
|
||||
.unwrap()
|
||||
};
|
||||
let mut stale_req = build_request(delete_marker(), Method::DELETE);
|
||||
stale_req.headers.insert(
|
||||
RUSTFS_EXPECTED_CURRENT_VERSION_ID,
|
||||
HeaderValue::from_str(&Uuid::new_v4().to_string()).unwrap(),
|
||||
);
|
||||
let err = Box::pin(usecase.execute_delete_object(stale_req))
|
||||
.await
|
||||
.expect_err("mismatched delete marker must fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::PreconditionFailed);
|
||||
|
||||
let mut matching_req = build_request(delete_marker(), Method::DELETE);
|
||||
matching_req
|
||||
.headers
|
||||
.insert(RUSTFS_EXPECTED_CURRENT_VERSION_ID, HeaderValue::from_str(&marker).unwrap());
|
||||
Box::pin(usecase.execute_delete_object(matching_req))
|
||||
.await
|
||||
.expect("current matching delete marker should be removed");
|
||||
assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, b"visible again");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() {
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
let payload = b"delete transitioned object through delete API";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
|
||||
lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned object");
|
||||
|
||||
let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before delete usecase runs");
|
||||
let remote_object = transitioned.transitioned_object.name.clone();
|
||||
|
||||
assert!(backend.contains(&remote_object).await);
|
||||
|
||||
let mut req = build_request(
|
||||
DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::DELETE,
|
||||
);
|
||||
insert_header(&mut req.headers, SUFFIX_FORCE_DELETE, "true");
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
Box::pin(usecase.execute_delete_object(req))
|
||||
.await
|
||||
.expect("Failed to delete object through usecase");
|
||||
|
||||
assert!(
|
||||
wait_for_object_absence(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
|
||||
"object should be removed from hot tier after delete usecase"
|
||||
);
|
||||
|
||||
assert!(
|
||||
backend.wait_for_remote_absence(&remote_object, TRANSITION_WAIT_TIMEOUT).await,
|
||||
"transitioned object should be removed from remote tier after delete usecase"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn compensation_driven_transition_still_cleans_remote_tier_on_delete() {
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-compensation-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
let payload = b"compensation should still preserve delete cleanup";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
with_forced_immediate_enqueue_timeout(&backend, || async {
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
})
|
||||
.await;
|
||||
|
||||
let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should eventually transition after compensation backfill");
|
||||
assert_eq!(backend.put_count().await, 1, "compensation should upload the version once");
|
||||
let remote_object = transitioned.transitioned_object.name.clone();
|
||||
|
||||
assert!(backend.contains(&remote_object).await);
|
||||
|
||||
let mut req = build_request(
|
||||
DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::DELETE,
|
||||
);
|
||||
insert_header(&mut req.headers, SUFFIX_FORCE_DELETE, "true");
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
Box::pin(usecase.execute_delete_object(req))
|
||||
.await
|
||||
.expect("Failed to delete object through usecase after compensation-driven transition");
|
||||
|
||||
assert!(
|
||||
wait_for_object_absence(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
|
||||
"object should be removed from hot tier after delete usecase"
|
||||
);
|
||||
|
||||
assert!(
|
||||
backend.wait_for_remote_absence(&remote_object, TRANSITION_WAIT_TIMEOUT).await,
|
||||
"transitioned object should be removed from remote tier after delete usecase"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn compensation_driven_versioned_delete_still_creates_delete_marker() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-comp-versioned-delete-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
let payload = b"versioned delete should preserve transitioned remote version behind delete marker";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
with_forced_immediate_enqueue_timeout(&backend, || async {
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
})
|
||||
.await;
|
||||
|
||||
let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should eventually transition after compensation backfill");
|
||||
assert_eq!(backend.put_count().await, 1, "compensation should upload the version once");
|
||||
let remote_object = transitioned.transitioned_object.name.clone();
|
||||
|
||||
assert!(backend.contains(&remote_object).await);
|
||||
|
||||
let req = build_request(
|
||||
DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::DELETE,
|
||||
);
|
||||
|
||||
Box::pin(usecase.execute_delete_object(req))
|
||||
.await
|
||||
.expect("Failed to issue versioned delete after compensation-driven transition");
|
||||
|
||||
assert!(
|
||||
wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
|
||||
"versioned delete should create a delete marker after compensation-driven transition"
|
||||
);
|
||||
assert!(
|
||||
backend.contains(&remote_object).await,
|
||||
"creating a delete marker should not remove the transitioned remote object version"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression pin for the DeleteObjects hot path: the bucket's versioning
|
||||
/// configuration must be resolved exactly once per request — not once per
|
||||
/// key, which used to cost up to 1000 identical metadata lookups per call.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn delete_objects_resolves_bucket_versioning_once_per_request() {
|
||||
use super::super::storage_api::test::bucket::replication::DELETE_CONFIG_SNAPSHOT_LOADS;
|
||||
use super::super::storage_api::test::{ReqInfo, VERSIONING_CONFIG_LOOKUPS};
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-delobjs-vers-once-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
let keys = ["batch/a.txt", "batch/b.txt", "batch/c.txt"];
|
||||
for key in keys {
|
||||
upload_test_object(&ecstore, bucket.as_str(), key, b"delete objects versioning payload").await;
|
||||
}
|
||||
|
||||
let input = DeleteObjectsInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.delete(Delete {
|
||||
objects: keys
|
||||
.iter()
|
||||
.map(|key| ObjectIdentifier {
|
||||
key: key.to_string(),
|
||||
version_id: None,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
quiet: None,
|
||||
})
|
||||
.build()
|
||||
.expect("delete objects input should build");
|
||||
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let lookups_before = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst);
|
||||
let snapshots_before = DELETE_CONFIG_SNAPSHOT_LOADS.load(Ordering::SeqCst);
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.await
|
||||
.expect("delete objects should succeed");
|
||||
|
||||
let output = response.output;
|
||||
let errors = output.errors.unwrap_or_default();
|
||||
assert!(errors.is_empty(), "DeleteObjects reported per-key errors: {errors:?}");
|
||||
assert_eq!(
|
||||
output.deleted.unwrap_or_default().len(),
|
||||
keys.len(),
|
||||
"every requested key should be deleted"
|
||||
);
|
||||
|
||||
let lookups = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst) - lookups_before;
|
||||
let snapshots = DELETE_CONFIG_SNAPSHOT_LOADS.load(Ordering::SeqCst) - snapshots_before;
|
||||
assert_eq!(
|
||||
snapshots,
|
||||
1,
|
||||
"DeleteObjects must resolve delete admission config exactly once per request; got {snapshots} snapshots for {} keys",
|
||||
keys.len()
|
||||
);
|
||||
assert_eq!(lookups, 0, "DeleteObjects must not fall back to legacy per-key versioning lookups");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn delete_objects_treats_empty_directory_version_as_absent() {
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delobjs-empty-version-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "directory/";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, b"").await;
|
||||
|
||||
let input = DeleteObjectsInput::builder()
|
||||
.bucket(bucket)
|
||||
.delete(Delete {
|
||||
objects: vec![ObjectIdentifier {
|
||||
key: object.to_string(),
|
||||
version_id: Some(" \t ".to_string()),
|
||||
..Default::default()
|
||||
}],
|
||||
quiet: None,
|
||||
})
|
||||
.build()
|
||||
.expect("delete objects input should build");
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.await
|
||||
.expect("empty VersionId should be treated as an absent version");
|
||||
let deleted = response
|
||||
.output
|
||||
.deleted
|
||||
.expect("delete response should contain the directory entry");
|
||||
|
||||
assert_eq!(deleted.len(), 1);
|
||||
assert_eq!(
|
||||
deleted[0].delete_marker, None,
|
||||
"the synthetic null version must be purged, not hidden by a new marker"
|
||||
);
|
||||
assert_eq!(deleted[0].version_id, None, "the synthetic null sentinel must stay off the wire");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn delete_objects_preserves_explicit_null_directory_version() {
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delobjs-null-version-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "directory/";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, b"").await;
|
||||
|
||||
let input = DeleteObjectsInput::builder()
|
||||
.bucket(bucket)
|
||||
.delete(Delete {
|
||||
objects: vec![ObjectIdentifier {
|
||||
key: object.to_string(),
|
||||
version_id: Some("null".to_string()),
|
||||
..Default::default()
|
||||
}],
|
||||
quiet: None,
|
||||
})
|
||||
.build()
|
||||
.expect("delete objects input should build");
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let response = usecase
|
||||
.execute_delete_objects(req)
|
||||
.await
|
||||
.expect("explicit null VersionId should delete the directory's null version");
|
||||
let deleted = response
|
||||
.output
|
||||
.deleted
|
||||
.expect("delete response should contain the directory entry");
|
||||
|
||||
assert_eq!(deleted.len(), 1);
|
||||
assert_eq!(deleted[0].delete_marker, None);
|
||||
assert_eq!(deleted[0].version_id.as_deref(), Some("null"));
|
||||
}
|
||||
|
||||
async fn install_malformed_versioning_config(bucket: &str) {
|
||||
use super::super::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
|
||||
|
||||
let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized");
|
||||
let metadata = {
|
||||
let sys = sys.read().await;
|
||||
sys.get(bucket)
|
||||
.await
|
||||
.expect("bucket metadata should be cached before corruption injection")
|
||||
};
|
||||
let mut metadata = (*metadata).clone();
|
||||
metadata.versioning_config_xml = b"<VersioningConfiguration>".to_vec();
|
||||
metadata.versioning_config = None;
|
||||
set_bucket_metadata(bucket.to_string(), metadata)
|
||||
.await
|
||||
.expect("malformed versioning metadata should be installed for the request test");
|
||||
}
|
||||
|
||||
async fn install_malformed_replication_config(bucket: &str) {
|
||||
use super::super::storage_api::test::{get_global_bucket_metadata_sys, set_bucket_metadata};
|
||||
|
||||
let sys = get_global_bucket_metadata_sys().expect("bucket metadata system should be initialized");
|
||||
let metadata = {
|
||||
let sys = sys.read().await;
|
||||
sys.get(bucket)
|
||||
.await
|
||||
.expect("bucket metadata should be cached before corruption injection")
|
||||
};
|
||||
let mut metadata = (*metadata).clone();
|
||||
metadata.replication_config_xml = b"<ReplicationConfiguration>".to_vec();
|
||||
metadata.replication_config = None;
|
||||
set_bucket_metadata(bucket.to_string(), metadata)
|
||||
.await
|
||||
.expect("malformed replication metadata should be installed for the request test");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn delete_object_versioning_config_failure_leaves_latest_object_intact() {
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delete-versioning-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
let payload = b"local delete must not commit before versioning admission";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, payload).await;
|
||||
install_malformed_versioning_config(&bucket).await;
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.expect("delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_object(req)
|
||||
.await
|
||||
.expect_err("versioning config failure must reject DeleteObject");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn delete_object_replication_config_failure_leaves_latest_object_intact() {
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delete-repl-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
let payload = b"local delete must not commit before replication admission";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, payload).await;
|
||||
install_malformed_replication_config(&bucket).await;
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.expect("delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_object(req)
|
||||
.await
|
||||
.expect_err("replication config failure must reject DeleteObject");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn delete_object_uses_one_replication_config_generation() {
|
||||
use super::super::object_usecase::install_delete_snapshot_test_hook;
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
use super::super::storage_api::test::bucket::replication::take_scheduled_replication_deletes;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delete-config-generation-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"generation-bound delete".to_vec());
|
||||
let uploaded = ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned object should be uploaded");
|
||||
let version_id = uploaded.version_id.expect("versioned upload should return a version ID");
|
||||
let version_id_string = version_id.to_string();
|
||||
|
||||
install_delete_replication_config(&bucket, true, false, None).await;
|
||||
assert!(take_scheduled_replication_deletes().is_empty());
|
||||
|
||||
let loaded = Arc::new(Barrier::new(2));
|
||||
let resume = Arc::new(Barrier::new(2));
|
||||
install_delete_snapshot_test_hook(bucket.clone(), Arc::clone(&loaded), Arc::clone(&resume));
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.version_id(Some(version_id_string.clone()))
|
||||
.build()
|
||||
.expect("version delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let update_config = async {
|
||||
loaded.wait().await;
|
||||
install_delete_replication_config(&bucket, false, false, None).await;
|
||||
resume.wait().await;
|
||||
};
|
||||
let (response, ()) = tokio::join!(usecase.execute_delete_object(req), update_config);
|
||||
let response = response.expect("delete admitted by the first config generation should succeed");
|
||||
|
||||
assert_eq!(response.output.version_id.as_deref(), Some(version_id_string.as_str()));
|
||||
let scheduled = take_scheduled_replication_deletes();
|
||||
assert_eq!(scheduled.len(), 1, "the first config generation must control post-delete queueing");
|
||||
assert_eq!(scheduled[0].version_id, Some(version_id));
|
||||
let state = scheduled[0]
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.expect("admitted version purge should carry replication state");
|
||||
assert_eq!(
|
||||
state.version_purge_status_internal.as_deref(),
|
||||
Some("arn:aws:s3:::target-bucket=PENDING;")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replica_delete_uses_authorized_source_version_and_its_tags() {
|
||||
use super::super::object_usecase::install_delete_source_test_hook;
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
use super::super::storage_api::test::bucket::replication::{ReplicationStatusType, take_scheduled_replication_deletes};
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use rustfs_utils::http::{SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID};
|
||||
use std::collections::HashMap;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-replica-source-version-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
install_delete_replication_config(&bucket, true, true, Some(("generation", "updated"))).await;
|
||||
|
||||
let mut old_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
user_defined: HashMap::from([(AMZ_OBJECT_TAGGING.to_string(), "generation=old".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
old_opts.set_replica_status(ReplicationStatusType::Replica);
|
||||
let mut old_reader = PutObjReader::from_vec(b"old replica version".to_vec());
|
||||
let old = ecstore
|
||||
.put_object(&bucket, object, &mut old_reader, &old_opts)
|
||||
.await
|
||||
.expect("old replica version should be uploaded");
|
||||
let old_version_id = old.version_id.expect("old replica version should have a version ID");
|
||||
let old_version_id_string = old_version_id.to_string();
|
||||
|
||||
let mut latest_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
user_defined: HashMap::from([(AMZ_OBJECT_TAGGING.to_string(), "generation=latest".to_string())]),
|
||||
..Default::default()
|
||||
};
|
||||
latest_opts.set_replica_status(ReplicationStatusType::Replica);
|
||||
let mut latest_reader = PutObjReader::from_vec(b"latest replica version".to_vec());
|
||||
let latest = ecstore
|
||||
.put_object(&bucket, object, &mut latest_reader, &latest_opts)
|
||||
.await
|
||||
.expect("latest replica version should be uploaded");
|
||||
assert_ne!(latest.version_id, Some(old_version_id));
|
||||
assert!(take_scheduled_replication_deletes().is_empty());
|
||||
|
||||
let source_loaded = Arc::new(Barrier::new(2));
|
||||
let resume_delete = Arc::new(Barrier::new(2));
|
||||
install_delete_source_test_hook(bucket.clone(), Arc::clone(&source_loaded), Arc::clone(&resume_delete));
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.expect("replica delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.headers
|
||||
.insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA"));
|
||||
insert_header(&mut req.headers, SUFFIX_SOURCE_VERSION_ID, old_version_id_string.clone());
|
||||
insert_header(&mut req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let update_tags = async {
|
||||
source_loaded.wait().await;
|
||||
ecstore
|
||||
.put_object_tags(
|
||||
&bucket,
|
||||
object,
|
||||
"generation=updated",
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(old_version_id_string.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("tag update ordered before the locked delete should succeed");
|
||||
resume_delete.wait().await;
|
||||
};
|
||||
let (response, ()) = tokio::join!(usecase.execute_delete_object(req), update_tags);
|
||||
let response = response.expect("authorized replica version delete should succeed");
|
||||
|
||||
assert_eq!(response.output.version_id.as_deref(), Some(old_version_id_string.as_str()));
|
||||
let scheduled = take_scheduled_replication_deletes();
|
||||
assert_eq!(
|
||||
scheduled.len(),
|
||||
1,
|
||||
"the tag committed after the advisory pre-read must control downstream fanout"
|
||||
);
|
||||
assert_eq!(scheduled[0].version_id, Some(old_version_id));
|
||||
assert_eq!(
|
||||
scheduled[0]
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.and_then(|state| state.version_purge_status_internal.as_deref()),
|
||||
Some("arn:aws:s3:::target-bucket=PENDING;")
|
||||
);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, b"latest replica version");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn force_delete_replication_config_failure_leaves_prefix_intact() {
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-force-delete-repl-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let prefix = "prefix/";
|
||||
let object = "prefix/object.txt";
|
||||
let payload = b"force delete must validate replication metadata before deleting descendants";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, payload).await;
|
||||
install_malformed_replication_config(&bucket).await;
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(prefix.to_string())
|
||||
.build()
|
||||
.expect("delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
insert_header(&mut req.headers, SUFFIX_FORCE_DELETE, "true");
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_object(req)
|
||||
.await
|
||||
.expect_err("replication config failure must reject force delete");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn replica_delete_replication_config_failure_leaves_object_intact() {
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-replica-delete-repl-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
let payload = b"incoming replica delete must use the admitted request snapshot";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, payload).await;
|
||||
install_malformed_replication_config(&bucket).await;
|
||||
|
||||
let input = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.expect("delete input should build");
|
||||
let mut req = build_request(input, Method::DELETE);
|
||||
req.headers
|
||||
.insert(AMZ_BUCKET_REPLICATION_STATUS, HeaderValue::from_static("REPLICA"));
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_object(req)
|
||||
.await
|
||||
.expect_err("replication config failure must reject incoming replica delete");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn delete_objects_replication_config_failure_does_not_call_storage_delete() {
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket = format!("test-delete-repls-fail-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "object.txt";
|
||||
let payload = b"batch delete must fail closed on replication metadata errors";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, object, payload).await;
|
||||
install_malformed_replication_config(&bucket).await;
|
||||
|
||||
let input = DeleteObjectsInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.delete(Delete {
|
||||
objects: vec![ObjectIdentifier {
|
||||
key: object.to_string(),
|
||||
version_id: None,
|
||||
..Default::default()
|
||||
}],
|
||||
quiet: None,
|
||||
})
|
||||
.build()
|
||||
.expect("delete objects input should build");
|
||||
let mut req = build_request(input, Method::POST);
|
||||
req.extensions.insert(ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_objects(req)
|
||||
.await
|
||||
.expect_err("replication config failure must reject DeleteObjects");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(read_object_bytes(&ecstore, &bucket, object).await, payload);
|
||||
}
|
||||
|
||||
/// Regression pin for the nonexistent-bucket fast path: GET/HEAD/DeleteObject
|
||||
/// must decide NoSuchBucket from the bucket-existence check, before any bucket
|
||||
/// versioning lookup runs — and cheap request-shape validation still wins over
|
||||
/// bucket existence for GET.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial"]
|
||||
async fn nonexistent_bucket_fails_before_versioning_lookup_on_get_head_delete() {
|
||||
use super::super::storage_api::test::VERSIONING_CONFIG_LOOKUPS;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_disk_paths, _ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
// Never created.
|
||||
let bucket = format!("test-missing-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
|
||||
// Request-shape validation keeps precedence over bucket existence: GET with
|
||||
// both range and partNumber is InvalidArgument even on a missing bucket.
|
||||
let get_invalid = GetObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("k.txt".to_string())
|
||||
.range(Some(Range::Int { first: 0, last: Some(1) }))
|
||||
.part_number(Some(1))
|
||||
.build()
|
||||
.expect("get input should build");
|
||||
let err = Box::pin(test_seams::get_object(&usecase, build_request(get_invalid, Method::GET)))
|
||||
.await
|
||||
.expect_err("range+partNumber must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
|
||||
let lookups_before = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst);
|
||||
|
||||
let get = GetObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("k.txt".to_string())
|
||||
.build()
|
||||
.expect("get input should build");
|
||||
let err = Box::pin(test_seams::get_object(&usecase, build_request(get, Method::GET)))
|
||||
.await
|
||||
.expect_err("GET on a missing bucket must fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
|
||||
|
||||
// Bucket existence beats versionId-format validation (which lives in the
|
||||
// opts builders) for HEAD and DeleteObject.
|
||||
let head = HeadObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("k.txt".to_string())
|
||||
.version_id(Some("not-a-uuid".to_string()))
|
||||
.build()
|
||||
.expect("head input should build");
|
||||
let err = Box::pin(test_seams::head_object(&usecase, build_request(head, Method::HEAD)))
|
||||
.await
|
||||
.expect_err("HEAD on a missing bucket must fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
|
||||
|
||||
let del = DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("k.txt".to_string())
|
||||
.version_id(Some("not-a-uuid".to_string()))
|
||||
.build()
|
||||
.expect("delete input should build");
|
||||
let err = Box::pin(usecase.execute_delete_object(build_request(del, Method::DELETE)))
|
||||
.await
|
||||
.expect_err("DeleteObject on a missing bucket must fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::NoSuchBucket);
|
||||
|
||||
let lookups = VERSIONING_CONFIG_LOOKUPS.load(Ordering::SeqCst) - lookups_before;
|
||||
assert_eq!(
|
||||
lookups, 0,
|
||||
"requests naming a nonexistent bucket must fail before any bucket versioning lookup; got {lookups}"
|
||||
);
|
||||
}
|
||||
@@ -1,403 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! PutBucketLifecycleConfiguration tests (batch A3 owns the lifecycle operations).
|
||||
|
||||
use super::common::*;
|
||||
use super::*;
|
||||
|
||||
fn expiration_lifecycle_configuration(prefix: &str) -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
expiration: Some(LifecycleExpiration {
|
||||
date: Some(Timestamp::from(
|
||||
time::OffsetDateTime::now_utc()
|
||||
.replace_time(time::Time::MIDNIGHT)
|
||||
.saturating_sub(time::Duration::days(1)),
|
||||
)),
|
||||
days: None,
|
||||
expired_object_delete_marker: None,
|
||||
..Default::default()
|
||||
}),
|
||||
filter: Some(LifecycleRuleFilter {
|
||||
and: None,
|
||||
object_size_greater_than: None,
|
||||
object_size_less_than: None,
|
||||
prefix: Some(prefix.to_string()),
|
||||
tag: None,
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("expire-existing".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn expired_object_delete_marker_lifecycle_configuration() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
expiration: Some(LifecycleExpiration {
|
||||
expired_object_delete_marker: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
filter: Some(LifecycleRuleFilter {
|
||||
prefix: Some("test/".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("expired-delete-marker".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn del_marker_expiration_lifecycle_configuration(days: i32) -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: Some(DelMarkerExpiration { days: Some(days) }),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(30),
|
||||
..Default::default()
|
||||
}),
|
||||
filter: Some(LifecycleRuleFilter {
|
||||
prefix: Some("test/".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("del-marker-expiration".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let bucket_usecase = DefaultBucketUsecase::from_global();
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-comp-del-marker-cleanup-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
let payload = b"delete marker lifecycle should still clean up after compensation-driven transition";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set transition lifecycle configuration");
|
||||
|
||||
with_forced_immediate_enqueue_timeout(&backend, || async {
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
})
|
||||
.await;
|
||||
|
||||
let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should eventually transition after compensation backfill");
|
||||
assert_eq!(backend.put_count().await, 1, "compensation should upload the version once");
|
||||
let remote_object = transitioned.transitioned_object.name.clone();
|
||||
|
||||
assert!(backend.contains(&remote_object).await);
|
||||
|
||||
let req = build_request(
|
||||
DeleteObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::DELETE,
|
||||
);
|
||||
|
||||
Box::pin(usecase.execute_delete_object(req))
|
||||
.await
|
||||
.expect("Failed to issue versioned delete after compensation-driven transition");
|
||||
|
||||
assert!(
|
||||
wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
|
||||
"versioned delete should create a delete marker before lifecycle cleanup"
|
||||
);
|
||||
assert!(
|
||||
backend.contains(&remote_object).await,
|
||||
"delete marker creation should keep the transitioned remote object version"
|
||||
);
|
||||
|
||||
let req = build_request(
|
||||
PutBucketLifecycleConfigurationInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.lifecycle_configuration(Some(expiration_lifecycle_configuration("test/")))
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::PUT,
|
||||
);
|
||||
let prepared = req
|
||||
.input
|
||||
.lifecycle_configuration
|
||||
.clone()
|
||||
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
|
||||
.transpose();
|
||||
bucket_usecase
|
||||
.execute_put_bucket_lifecycle_configuration(req, prepared)
|
||||
.await
|
||||
.expect("Failed to update lifecycle configuration for delete marker cleanup");
|
||||
|
||||
assert!(
|
||||
wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
|
||||
"delete marker should remain visible after lifecycle update until cleanup completes"
|
||||
);
|
||||
assert!(
|
||||
backend.contains(&remote_object).await,
|
||||
"delete marker lifecycle cleanup should not remove the transitioned remote object version"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultBucketUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-api-expire-existing-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/existing.txt";
|
||||
let payload = b"expire existing object after lifecycle update";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
|
||||
let req = build_request(
|
||||
PutBucketLifecycleConfigurationInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.lifecycle_configuration(Some(expiration_lifecycle_configuration("test/")))
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
let prepared = req
|
||||
.input
|
||||
.lifecycle_configuration
|
||||
.clone()
|
||||
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
|
||||
.transpose();
|
||||
usecase
|
||||
.execute_put_bucket_lifecycle_configuration(req, prepared)
|
||||
.await
|
||||
.expect("Failed to update lifecycle configuration");
|
||||
|
||||
assert!(
|
||||
wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
|
||||
"existing object should be lifecycle-deleted after lifecycle update"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_object_lock_bucket() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultBucketUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-lock-expired-del-marker-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_object_lock_enabled(bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enable object lock for bucket");
|
||||
|
||||
let req = build_request(
|
||||
PutBucketLifecycleConfigurationInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.lifecycle_configuration(Some(expired_object_delete_marker_lifecycle_configuration()))
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
let prepared = req
|
||||
.input
|
||||
.lifecycle_configuration
|
||||
.clone()
|
||||
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
|
||||
.transpose();
|
||||
usecase
|
||||
.execute_put_bucket_lifecycle_configuration(req, prepared)
|
||||
.await
|
||||
.expect("ExpiredObjectDeleteMarker should be accepted on object-lock bucket");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_object_lock_bucket() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultBucketUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-lock-del-marker-exp-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_object_lock_enabled(bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enable object lock for bucket");
|
||||
|
||||
let req = build_request(
|
||||
PutBucketLifecycleConfigurationInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.lifecycle_configuration(Some(del_marker_expiration_lifecycle_configuration(1)))
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
let prepared = req
|
||||
.input
|
||||
.lifecycle_configuration
|
||||
.clone()
|
||||
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
|
||||
.transpose();
|
||||
let err = usecase
|
||||
.execute_put_bucket_lifecycle_configuration(req, prepared)
|
||||
.await
|
||||
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
let message = err.message().unwrap_or_default();
|
||||
assert!(
|
||||
message.contains(
|
||||
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"
|
||||
),
|
||||
"unexpected error message: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expiration_on_object_lock_bucket() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultBucketUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-lock-del-marker-exp0-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_object_lock_enabled(bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enable object lock for bucket");
|
||||
|
||||
let req = build_request(
|
||||
PutBucketLifecycleConfigurationInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.lifecycle_configuration(Some(del_marker_expiration_lifecycle_configuration(0)))
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
let prepared = req
|
||||
.input
|
||||
.lifecycle_configuration
|
||||
.clone()
|
||||
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
|
||||
.transpose();
|
||||
let err = usecase
|
||||
.execute_put_bucket_lifecycle_configuration(req, prepared)
|
||||
.await
|
||||
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
let message = err.message().unwrap_or_default();
|
||||
assert!(
|
||||
message.contains("Days must be a positive integer with DelMarkerExpiration"),
|
||||
"unexpected error message: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
|
||||
// S3 compatibility (ceph s3-tests `test_lifecycle_expiration_days0`): a rule with
|
||||
// Expiration{Days:0} must be rejected with InvalidArgument through the real
|
||||
// PutBucketLifecycleConfiguration handler, not merely at the crates/lifecycle
|
||||
// io::Error layer. No object lock required.
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultBucketUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-zero-day-exp-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
let lifecycle = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(0),
|
||||
..Default::default()
|
||||
}),
|
||||
filter: Some(LifecycleRuleFilter {
|
||||
prefix: Some("days0/".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("expire-zero-days".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
};
|
||||
|
||||
let req = build_request(
|
||||
PutBucketLifecycleConfigurationInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.lifecycle_configuration(Some(lifecycle))
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
let prepared = req
|
||||
.input
|
||||
.lifecycle_configuration
|
||||
.clone()
|
||||
.map(crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s)
|
||||
.transpose();
|
||||
let err = usecase
|
||||
.execute_put_bucket_lifecycle_configuration(req, prepared)
|
||||
.await
|
||||
.expect_err("zero-day expiration must be rejected with InvalidArgument");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
let message = err.message().unwrap_or_default();
|
||||
assert!(
|
||||
message.contains("'Days' for Expiration action must be a positive integer"),
|
||||
"unexpected error message: {message}"
|
||||
);
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! CompleteMultipartUpload tests (batch E owns CompleteMultipartUpload).
|
||||
|
||||
use super::common::*;
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn complete_multipart_upload_transitions_immediately_via_usecase() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/multipart.txt";
|
||||
let payload = b"multipart immediate transition through complete API";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
let upload = ecstore
|
||||
.new_multipart_upload(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to create multipart upload");
|
||||
|
||||
let mut reader = PutObjReader::from_vec(payload.to_vec());
|
||||
let uploaded_part = ecstore
|
||||
.put_object_part(bucket.as_str(), object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to upload multipart part");
|
||||
|
||||
let complete_input = CompleteMultipartUploadInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.multipart_upload(Some(CompletedMultipartUpload {
|
||||
parts: Some(vec![CompletedPart {
|
||||
part_number: Some(1),
|
||||
e_tag: uploaded_part.etag.clone().map(|etag| to_s3s_etag(&etag)),
|
||||
..Default::default()
|
||||
}]),
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let complete_output = Box::pin(usecase.execute_complete_multipart_upload(build_request(complete_input, Method::POST)))
|
||||
.await
|
||||
.expect("Failed to complete multipart upload through usecase");
|
||||
assert!(
|
||||
complete_output.output.version_id.is_some(),
|
||||
"versioned CompleteMultipartUpload should return the created version ID"
|
||||
);
|
||||
|
||||
let info = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("multipart object should transition immediately after complete usecase");
|
||||
|
||||
assert_eq!(info.transitioned_object.status, "complete");
|
||||
assert_eq!(info.transitioned_object.tier, tier_name);
|
||||
assert!(backend.contains(&info.transitioned_object.name).await);
|
||||
}
|
||||
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
#[test]
|
||||
fn compensation_driven_complete_multipart_upload_still_transitions() {
|
||||
std::thread::Builder::new()
|
||||
.name("lifecycle-compensation-complete-multipart".to_string())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("compensation multipart test runtime should build");
|
||||
|
||||
runtime.block_on(compensation_driven_complete_multipart_upload_still_transitions_inner());
|
||||
})
|
||||
.expect("compensation multipart test thread should spawn")
|
||||
.join()
|
||||
.expect("compensation multipart test thread should finish");
|
||||
}
|
||||
|
||||
async fn compensation_driven_complete_multipart_upload_still_transitions_inner() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-comp-mpu-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/multipart.txt";
|
||||
let payload = b"multipart should still transition after compensation";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
let upload = ecstore
|
||||
.new_multipart_upload(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to create multipart upload");
|
||||
|
||||
let mut reader = PutObjReader::from_vec(payload.to_vec());
|
||||
let uploaded_part = ecstore
|
||||
.put_object_part(bucket.as_str(), object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to upload multipart part");
|
||||
|
||||
let complete_input = CompleteMultipartUploadInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.multipart_upload(Some(CompletedMultipartUpload {
|
||||
parts: Some(vec![CompletedPart {
|
||||
part_number: Some(1),
|
||||
e_tag: uploaded_part.etag.clone().map(|etag| to_s3s_etag(&etag)),
|
||||
..Default::default()
|
||||
}]),
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
with_forced_immediate_enqueue_timeout(&backend, || async {
|
||||
Box::pin(usecase.execute_complete_multipart_upload(build_request(complete_input, Method::POST)))
|
||||
.await
|
||||
.expect("Failed to complete multipart upload through usecase");
|
||||
})
|
||||
.await;
|
||||
|
||||
let info = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("multipart object should eventually transition after compensation backfill");
|
||||
|
||||
assert_eq!(backend.put_count().await, 1, "compensation should upload the version once");
|
||||
assert_eq!(info.transitioned_object.status, "complete");
|
||||
assert_eq!(info.transitioned_object.tier, tier_name);
|
||||
assert!(backend.contains(&info.transitioned_object.name).await);
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! PutObjectRetention and PutObjectLegalHold tests (batch B1 owns both).
|
||||
|
||||
use super::common::*;
|
||||
use super::*;
|
||||
|
||||
/// The object-lock handlers do not go through the object PUT path, so they must schedule
|
||||
/// replication themselves. Without it a retention or legal hold applied after upload stays
|
||||
/// local and the replica keeps its previous, unprotected state (a WORM object that is still
|
||||
/// deletable on the peer). This observes the shared decision counter: each handler must
|
||||
/// compute a replication decision exactly once when replication is configured, which is
|
||||
/// the step that drives both the persisted pending marker and the schedule.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn object_lock_handlers_schedule_replication() {
|
||||
use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
|
||||
use crate::app::storage_api::s3::S3;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let fs = FS::new();
|
||||
|
||||
let bucket = format!("test-lock-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "locked/object.txt";
|
||||
|
||||
(*ecstore)
|
||||
.make_bucket(
|
||||
bucket.as_str(),
|
||||
&MakeBucketOptions {
|
||||
lock_enabled: true,
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("Failed to create object-lock bucket");
|
||||
upload_test_object(&ecstore, bucket.as_str(), object, b"worm payload").await;
|
||||
|
||||
// PutObjectRetention must compute a replication decision.
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
let retention_input = PutObjectRetentionInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.retention(Some(ObjectLockRetention {
|
||||
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
|
||||
retain_until_date: Some(time::macros::datetime!(2030-01-01 00:00:00 UTC).into()),
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
fs.put_object_retention(build_request(retention_input, Method::PUT))
|
||||
.await
|
||||
.expect("put_object_retention should succeed");
|
||||
assert_eq!(
|
||||
MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
|
||||
1,
|
||||
"PutObjectRetention must compute the replication decision exactly once; 0 means the \
|
||||
retention change never replicates and the peer copy stays unprotected"
|
||||
);
|
||||
|
||||
// A bucket without a replication configuration can skip the decision.
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
let no_replication_input = PutObjectLegalHoldInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.legal_hold(Some(ObjectLockLegalHold {
|
||||
status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF)),
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
fs.put_object_legal_hold(build_request(no_replication_input, Method::PUT))
|
||||
.await
|
||||
.expect("legal hold without replication should succeed");
|
||||
assert_eq!(
|
||||
MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
|
||||
0,
|
||||
"a bucket without replication must skip the decision"
|
||||
);
|
||||
|
||||
// A configured bucket must still evaluate the metadata replication decision.
|
||||
install_delete_replication_config(&bucket, true, false, None).await;
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
let legal_hold_input = PutObjectLegalHoldInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.legal_hold(Some(ObjectLockLegalHold {
|
||||
status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON)),
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
fs.put_object_legal_hold(build_request(legal_hold_input, Method::PUT))
|
||||
.await
|
||||
.expect("put_object_legal_hold should succeed");
|
||||
assert_eq!(
|
||||
MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
|
||||
1,
|
||||
"PutObjectLegalHold must compute the replication decision exactly once; 0 means the \
|
||||
legal hold never replicates and the peer copy stays deletable"
|
||||
);
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! PutObject tests (batch D owns PutObject).
|
||||
|
||||
use super::common::*;
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn put_object_if_none_match_existing_object_returns_precondition_failed() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let fs = FS::new();
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-put-if-none-match-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
let initial_payload = b"initial conditional put payload";
|
||||
let replacement_payload = b"replacement conditional put payload";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
let initial_input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.body(Some(streaming_blob_from_bytes(initial_payload)))
|
||||
.content_length(Some(initial_payload.len() as i64))
|
||||
.build()
|
||||
.unwrap();
|
||||
Box::pin(usecase.execute_put_object(&fs, build_request(initial_input, Method::PUT)))
|
||||
.await
|
||||
.expect("Failed to upload initial object through usecase");
|
||||
|
||||
let existing_info = ecstore
|
||||
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to fetch existing object info");
|
||||
let existing_etag = existing_info.etag.expect("existing object should have an ETag");
|
||||
let initial_versions = live_object_version_count(&ecstore, bucket.as_str(), object).await;
|
||||
|
||||
for (condition, expected_code) in [
|
||||
(ETagCondition::Any, S3ErrorCode::PreconditionFailed),
|
||||
(ETagCondition::ETag(ETag::Strong(existing_etag)), S3ErrorCode::NotImplemented),
|
||||
(
|
||||
ETagCondition::ETag(ETag::Strong("different-etag".to_string())),
|
||||
S3ErrorCode::NotImplemented,
|
||||
),
|
||||
] {
|
||||
let header = condition.to_http_header().expect("conditional PUT header should encode");
|
||||
// Cover header-only callers and the DTO-plus-header shape of parsed HTTP requests.
|
||||
for if_none_match in [None, Some(condition)] {
|
||||
let has_dto = if_none_match.is_some();
|
||||
let replacement_input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.body(Some(streaming_blob_from_bytes(replacement_payload)))
|
||||
.content_length(Some(replacement_payload.len() as i64))
|
||||
.if_none_match(if_none_match)
|
||||
.build()
|
||||
.expect("conditional replacement PUT input should build");
|
||||
let mut req = build_request(replacement_input, Method::PUT);
|
||||
req.headers.insert(IF_NONE_MATCH, header.clone());
|
||||
|
||||
let err = Box::pin(usecase.execute_put_object(&fs, req))
|
||||
.await
|
||||
.expect_err("conditional PUT must reject the replacement");
|
||||
assert_eq!(err.code(), &expected_code, "header={header:?}, has_dto={has_dto}");
|
||||
assert_eq!(
|
||||
read_object_bytes(&ecstore, bucket.as_str(), object).await,
|
||||
initial_payload,
|
||||
"failed conditional PutObject must not overwrite the current object"
|
||||
);
|
||||
assert_eq!(
|
||||
live_object_version_count(&ecstore, bucket.as_str(), object).await,
|
||||
initial_versions,
|
||||
"failed conditional PutObject must not create a version"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1320: a single PUT must compute the replication decision
|
||||
/// exactly once. Historically the PUT path computed `must_replicate_object`
|
||||
/// twice with the same inputs — once before commit to persist the pending
|
||||
/// replication metadata, and again after commit to drive the schedule. Besides
|
||||
/// repeating the versioning/config/target traversal on the hot path, a
|
||||
/// replication-config hot update between the two computations could split the
|
||||
/// two phases (pending-without-schedule or schedule-without-pending). The fix
|
||||
/// reuses one immutable decision for both phases; this test observes the
|
||||
/// computation counter and fails if a second computation is reintroduced.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn put_object_computes_replication_decision_exactly_once() {
|
||||
use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let fs = FS::new();
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-repl-decision-once-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
let payload = b"replication decision single-compute payload";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
// Reset immediately before the single PUT so the counter reflects only this
|
||||
// request. `#[serial]` guarantees no other usecase PUT runs concurrently, so
|
||||
// no unrelated `must_replicate_object` call can perturb the count.
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.body(Some(streaming_blob_from_bytes(payload)))
|
||||
.content_length(Some(payload.len() as i64))
|
||||
.build()
|
||||
.unwrap();
|
||||
Box::pin(usecase.execute_put_object(&fs, build_request(input, Method::PUT)))
|
||||
.await
|
||||
.expect("Failed to upload object through usecase");
|
||||
|
||||
let calls = MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst);
|
||||
assert_eq!(
|
||||
calls, 1,
|
||||
"a single PUT must compute the replication decision exactly once; reverting to a \
|
||||
pre-commit + post-commit recompute makes this observe 2 (rustfs/backlog#1320)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Snowball auto-extract writes each archive member as an independent object,
|
||||
/// so each member must join bucket replication like a regular PUT (MinIO
|
||||
/// PutObjectExtract parity). The archive holds two file entries; a count other
|
||||
/// than 2 means extracted objects bypass the replication decision entirely and
|
||||
/// stay local forever.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn put_object_extract_computes_replication_decision_per_entry() {
|
||||
use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
|
||||
use super::super::storage_api::test::ReqInfo;
|
||||
use super::super::storage_api::test::bucket::metadata::BUCKET_POLICY_CONFIG;
|
||||
use std::io::Cursor;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-extract-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
// The extract path re-authorizes every extracted entry internally. The test
|
||||
// harness has no IAM system, so allow anonymous PutObject via bucket policy.
|
||||
let policy_json = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/*")]
|
||||
}
|
||||
]
|
||||
})
|
||||
.to_string();
|
||||
metadata_sys::update(bucket.as_str(), BUCKET_POLICY_CONFIG, policy_json.into_bytes())
|
||||
.await
|
||||
.expect("Failed to install anonymous put bucket policy");
|
||||
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
for (path, data) in [
|
||||
("member-one.txt", b"first member payload".as_slice()),
|
||||
("nested/member-two.txt", b"second member payload".as_slice()),
|
||||
] {
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(data.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, path, Cursor::new(data))
|
||||
.await
|
||||
.expect("Failed to append archive member");
|
||||
}
|
||||
let archive = builder.into_inner().await.expect("Failed to finish archive").into_inner();
|
||||
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
|
||||
let input = PutObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("archive.tar".to_string())
|
||||
.body(Some(streaming_blob_from_bytes(&archive)))
|
||||
.content_length(Some(archive.len() as i64))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut req = build_request(input, Method::PUT);
|
||||
req.extensions.insert(ReqInfo::default());
|
||||
Box::pin(usecase.execute_put_object_extract(req))
|
||||
.await
|
||||
.expect("Failed to extract archive through usecase");
|
||||
|
||||
assert_eq!(
|
||||
MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
|
||||
2,
|
||||
"snowball auto-extract must compute one replication decision per extracted member \
|
||||
object; 0 means extracted objects bypass bucket replication (P0-6 companion, MinIO \
|
||||
PutObjectExtract parity)"
|
||||
);
|
||||
}
|
||||
@@ -1,492 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! RestoreObject tests (batch E owns RestoreObject).
|
||||
|
||||
use super::common::*;
|
||||
use super::*;
|
||||
|
||||
const RESTORE_SUSPENDED_WAIT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
async fn suspend_test_bucket(bucket: &str) {
|
||||
DefaultBucketUsecase::from_global()
|
||||
.execute_put_bucket_versioning(
|
||||
build_request(
|
||||
PutBucketVersioningInput::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.versioning_configuration(VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
})
|
||||
.build()
|
||||
.expect("suspended versioning request should build"),
|
||||
Method::PUT,
|
||||
),
|
||||
Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Suspended".to_owned()),
|
||||
..Default::default()
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("bucket versioning should be suspended");
|
||||
}
|
||||
|
||||
async fn transition_uploaded_object_directly(
|
||||
ecstore: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
tier_name: &str,
|
||||
uploaded: &ObjectInfo,
|
||||
) -> ObjectInfo {
|
||||
let transition_opts = ObjectOptions {
|
||||
transition: lifecycle::lifecycle_contract::TransitionOptions {
|
||||
status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: uploaded.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: uploaded.version_id.map(|version| version.to_string()),
|
||||
versioned: uploaded.version_id.is_some(),
|
||||
mod_time: uploaded.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
ecstore
|
||||
.transition_object(bucket, object, &transition_opts)
|
||||
.await
|
||||
.expect("Failed to transition object directly");
|
||||
wait_for_transition(ecstore, bucket, object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before restore assertions")
|
||||
}
|
||||
|
||||
async fn wait_for_restore_completion(
|
||||
ecstore: &Arc<ECStore>,
|
||||
backend: &MockWarmBackend,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: Duration,
|
||||
) -> Result<ObjectInfo, String> {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
let mut last_state = None;
|
||||
|
||||
loop {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
let tier_gets = backend.get_count().await;
|
||||
let op_log = backend.op_log().await;
|
||||
return Err(format!(
|
||||
"restore copy-back should complete within {timeout:?}; tier_gets={tier_gets}, op_log={op_log:?}; last observed state: {}",
|
||||
last_state.unwrap_or_else(|| "no object info observed".to_string())
|
||||
));
|
||||
}
|
||||
|
||||
match (**ecstore).get_object_info(bucket, object, &ObjectOptions::default()).await {
|
||||
Ok(info) => {
|
||||
if !info.restore_ongoing && info.restore_expires.is_some() {
|
||||
return Ok(info);
|
||||
}
|
||||
last_state = Some(format!(
|
||||
"restore_ongoing={}, restore_expires={:?}, transitioned_status={}",
|
||||
info.restore_ongoing, info.restore_expires, info.transitioned_object.status
|
||||
));
|
||||
}
|
||||
Err(err) => {
|
||||
last_state = Some(format!("get_object_info failed: {err}"));
|
||||
}
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// backlog#1148 ilm-8: the RestoreObject API surface on a transitioned object.
|
||||
///
|
||||
/// POST restore(days=1) is accepted and flips the object to
|
||||
/// `x-amz-restore: ongoing-request="true"` while the mock tier GET barrier
|
||||
/// proves the background copy-back has reached the remote read; a second POST
|
||||
/// during that window is rejected with 409 `RestoreAlreadyInProgress`.
|
||||
/// Synchronous SetDisks transition tests cover copy-back completion, restore
|
||||
/// metadata, and local byte-identical reads.
|
||||
///
|
||||
/// Re-enabled in the serial lane by backlog#1304: the accept path now flips
|
||||
/// the ongoing flag under a short compare-and-set guard and the copy-back
|
||||
/// runs without the #4877 whole-copy-back lock, so the mid-restore
|
||||
/// `ongoing-request="true"` read and the fast 409 rejection asserted here are
|
||||
/// the implemented contract.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
|
||||
async fn restore_object_usecase_reports_ongoing_conflict() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-restore-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
// Keep the object under the shared ILM test prefix even though this setup
|
||||
// transitions it directly; it keeps diagnostics aligned with sibling tests.
|
||||
let object = "test/restore/api-object.bin";
|
||||
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
assert!(uploaded.version_id.is_none(), "fixture must create the null version");
|
||||
let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
|
||||
backend.clear_op_log().await;
|
||||
let get_barrier = backend.arm_get_barrier().await;
|
||||
|
||||
let restore_request = || RestoreRequest {
|
||||
days: Some(1),
|
||||
description: None,
|
||||
glacier_job_parameters: None,
|
||||
output_location: None,
|
||||
select_parameters: None,
|
||||
tier: None,
|
||||
type_: None,
|
||||
};
|
||||
let restore_input = || {
|
||||
RestoreObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.restore_request(Some(restore_request()))
|
||||
.build()
|
||||
.unwrap()
|
||||
};
|
||||
|
||||
Box::pin(usecase.execute_restore_object(build_request(restore_input(), Method::POST)))
|
||||
.await
|
||||
.expect("restore request should be accepted");
|
||||
|
||||
get_barrier.wait_until_paused().await;
|
||||
|
||||
// The barrier proves the detached copy-back reached the tier GET and is
|
||||
// still paused, so the ongoing state and conflict rejection are not timing
|
||||
// assumptions about task scheduling.
|
||||
let ongoing = ecstore
|
||||
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("Failed to load object info during restore");
|
||||
assert!(
|
||||
ongoing.restore_ongoing,
|
||||
"x-amz-restore must report ongoing-request=true while the copy-back tier GET is paused"
|
||||
);
|
||||
|
||||
// A second restore while one is in flight is rejected.
|
||||
let err = Box::pin(usecase.execute_restore_object(build_request(restore_input(), Method::POST)))
|
||||
.await
|
||||
.expect_err("a second restore while one is ongoing must be rejected");
|
||||
assert_eq!(
|
||||
err.code(),
|
||||
&S3ErrorCode::RestoreAlreadyInProgress,
|
||||
"unexpected rejection for a repeated restore: {err:?}"
|
||||
);
|
||||
|
||||
get_barrier.release();
|
||||
}
|
||||
|
||||
/// rustfs/backlog#1337: cancellation after the ongoing metadata commit but
|
||||
/// before detached worker creation must not strand the object forever. The
|
||||
/// replacement POST proves the abandoned v1 generation has no live worker,
|
||||
/// atomically supersedes it, and performs exactly one remote copy-back.
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1337"]
|
||||
#[test]
|
||||
fn restore_object_usecase_recovers_cancelled_post_commit_generation() {
|
||||
std::thread::Builder::new()
|
||||
.name("lifecycle-restore-orphan-recovery".to_string())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("orphan restore recovery test runtime should build");
|
||||
runtime.block_on(restore_object_usecase_recovers_cancelled_post_commit_generation_inner());
|
||||
})
|
||||
.expect("orphan restore recovery test thread should spawn")
|
||||
.join()
|
||||
.expect("orphan restore recovery test thread should finish");
|
||||
}
|
||||
|
||||
async fn restore_object_usecase_recovers_cancelled_post_commit_generation_inner() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
let bucket = format!("test-api-restore-orphan-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/restore/orphaned-generation.bin";
|
||||
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
|
||||
backend.clear_op_log().await;
|
||||
let tier_gets_before_restore = backend.get_count().await;
|
||||
|
||||
let restore_input = || {
|
||||
RestoreObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.restore_request(Some(RestoreRequest {
|
||||
days: Some(1),
|
||||
description: None,
|
||||
glacier_job_parameters: None,
|
||||
output_location: None,
|
||||
select_parameters: None,
|
||||
tier: None,
|
||||
type_: None,
|
||||
}))
|
||||
.build()
|
||||
.expect("restore request should build")
|
||||
};
|
||||
|
||||
let commit_barrier = crate::app::object::RestoreStatusCommitBarrier::install(bucket.as_str(), object);
|
||||
let first_input = restore_input();
|
||||
let first = tokio::spawn(async move {
|
||||
DefaultObjectUsecase::from_global()
|
||||
.execute_restore_object(build_request(first_input, Method::POST))
|
||||
.await
|
||||
});
|
||||
commit_barrier.wait_until_paused().await;
|
||||
|
||||
first.abort();
|
||||
assert!(first.await.expect_err("the first request must be cancelled").is_cancelled());
|
||||
drop(commit_barrier);
|
||||
|
||||
let orphaned = ecstore
|
||||
.get_object_info(bucket.as_str(), object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the committed orphan generation should remain readable");
|
||||
assert!(orphaned.restore_ongoing, "the interrupted request must have committed ongoing=true");
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_consistent_str(orphaned.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_WORKER_LOCK,),
|
||||
Some(rustfs_utils::http::RESTORE_WORKER_LOCK_PROTOCOL_V1),
|
||||
"recoverable generations must advertise the worker-lock protocol"
|
||||
);
|
||||
let orphaned_operation_id =
|
||||
rustfs_utils::http::get_consistent_str(orphaned.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_OPERATION_ID)
|
||||
.and_then(|value| Uuid::parse_str(value).ok())
|
||||
.expect("recoverable generations must persist a non-nil operation id");
|
||||
assert!(!orphaned_operation_id.is_nil());
|
||||
|
||||
DefaultObjectUsecase::from_global()
|
||||
.execute_restore_object(build_request(restore_input(), Method::POST))
|
||||
.await
|
||||
.expect("a new POST must supersede the committed generation whose worker lock was released");
|
||||
|
||||
let completed = wait_for_restore_completion(&ecstore, &backend, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("{err}"));
|
||||
assert!(!completed.restore_ongoing);
|
||||
assert!(completed.restore_expires.is_some());
|
||||
assert!(
|
||||
rustfs_utils::http::get_str(completed.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_OPERATION_ID,).is_none(),
|
||||
"completed replacement restore must consume its operation id"
|
||||
);
|
||||
assert!(
|
||||
rustfs_utils::http::get_str(completed.user_defined.as_ref(), rustfs_utils::http::SUFFIX_RESTORE_WORKER_LOCK,).is_none(),
|
||||
"completed replacement restore must consume its liveness marker"
|
||||
);
|
||||
assert_eq!(
|
||||
backend.get_count().await - tier_gets_before_restore,
|
||||
1,
|
||||
"the cancelled pre-spawn generation must not issue a tier GET"
|
||||
);
|
||||
}
|
||||
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#4879"]
|
||||
#[test]
|
||||
fn restore_object_usecase_completes_suspended_null_version_in_place() {
|
||||
std::thread::Builder::new()
|
||||
.name("lifecycle-restore-suspended-null".to_string())
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("suspended null-version restore test runtime should build");
|
||||
|
||||
runtime.block_on(restore_object_usecase_completes_suspended_null_version_in_place_inner());
|
||||
})
|
||||
.expect("suspended null-version restore test thread should spawn")
|
||||
.join()
|
||||
.expect("suspended null-version restore test thread should finish");
|
||||
}
|
||||
|
||||
async fn restore_object_usecase_completes_suspended_null_version_in_place_inner() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
let bucket = format!("test-api-restore-suspended-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/restore/suspended-null.bin";
|
||||
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
assert!(uploaded.version_id.is_none(), "fixture must create the null version");
|
||||
let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
|
||||
suspend_test_bucket(bucket.as_str()).await;
|
||||
backend.clear_op_log().await;
|
||||
let tier_gets_before_restore = backend.get_count().await;
|
||||
let get_barrier = backend.arm_get_barrier().await;
|
||||
|
||||
Box::pin(
|
||||
usecase.execute_restore_object(build_request(
|
||||
RestoreObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.restore_request(Some(RestoreRequest {
|
||||
days: Some(1),
|
||||
description: None,
|
||||
glacier_job_parameters: None,
|
||||
output_location: None,
|
||||
select_parameters: None,
|
||||
tier: None,
|
||||
type_: None,
|
||||
}))
|
||||
.build()
|
||||
.expect("restore request should build"),
|
||||
Method::POST,
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.expect("suspended null-version restore should be accepted");
|
||||
|
||||
get_barrier.wait_until_paused().await;
|
||||
get_barrier.release();
|
||||
let completed = wait_for_restore_completion(&ecstore, &backend, bucket.as_str(), object, RESTORE_SUSPENDED_WAIT_TIMEOUT)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("{err}"));
|
||||
|
||||
assert!(!completed.restore_ongoing, "the original null version must complete in place");
|
||||
assert!(completed.restore_expires.is_some(), "completed restore must carry an expiry");
|
||||
assert!(
|
||||
completed.version_id.is_none() || completed.version_id.is_some_and(|version_id| version_id.is_nil()),
|
||||
"suspended restore must remain on the null version"
|
||||
);
|
||||
assert_eq!(
|
||||
live_object_version_count(&ecstore, bucket.as_str(), object).await,
|
||||
1,
|
||||
"suspended restore must not create a UUID version"
|
||||
);
|
||||
assert_eq!(
|
||||
backend.get_count().await - tier_gets_before_restore,
|
||||
1,
|
||||
"suspended restore copy-back must fetch the tier exactly once"
|
||||
);
|
||||
}
|
||||
|
||||
/// backlog#1304: the restore-accept compare-and-set itself, under real
|
||||
/// concurrency. Two POST ?restore for the same transitioned object race each
|
||||
/// other from separate tasks; the accept guard must let exactly one through —
|
||||
/// the winner flips `ongoing-request="true"`, the loser must read that flag
|
||||
/// and be rejected with 409 `RestoreAlreadyInProgress` — and the tier must
|
||||
/// see exactly one copy-back GET, never two. Without the accept guard both
|
||||
/// requests can observe ongoing=false and double-start the copy-back (the
|
||||
/// #4859 race), which this test catches via the tier GET count.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-8)"]
|
||||
async fn restore_object_usecase_accepts_exactly_one_of_two_concurrent_restores() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-restore-cas-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
// Keep the object under the shared ILM test prefix for diagnostics parity.
|
||||
let object = "test/restore/cas-object.bin";
|
||||
let payload: Vec<u8> = (0..128 * 1024).map(|i| (i % 251) as u8).collect();
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
let _ = transition_uploaded_object_directly(&ecstore, bucket.as_str(), object, &tier_name, &uploaded).await;
|
||||
backend.clear_op_log().await;
|
||||
|
||||
// Hold the accepted copy-back at the tier GET until both accept attempts
|
||||
// return, so the loser cannot observe an already-restored object.
|
||||
let get_barrier = backend.arm_get_barrier().await;
|
||||
|
||||
let tier_gets_before_restore = backend.get_count().await;
|
||||
|
||||
let restore_input = || {
|
||||
RestoreObjectInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.restore_request(Some(RestoreRequest {
|
||||
days: Some(1),
|
||||
description: None,
|
||||
glacier_job_parameters: None,
|
||||
output_location: None,
|
||||
select_parameters: None,
|
||||
tier: None,
|
||||
type_: None,
|
||||
}))
|
||||
.build()
|
||||
.unwrap()
|
||||
};
|
||||
let spawn_restore = |input: RestoreObjectInput| {
|
||||
tokio::spawn(async move {
|
||||
let usecase = DefaultObjectUsecase::from_global();
|
||||
Box::pin(usecase.execute_restore_object(build_request(input, Method::POST))).await
|
||||
})
|
||||
};
|
||||
|
||||
let (first, second) = tokio::join!(spawn_restore(restore_input()), spawn_restore(restore_input()));
|
||||
let results = [
|
||||
first.expect("first concurrent restore task panicked"),
|
||||
second.expect("second concurrent restore task panicked"),
|
||||
];
|
||||
|
||||
let accepted = results.iter().filter(|result| result.is_ok()).count();
|
||||
assert_eq!(accepted, 1, "exactly one of two concurrent restores must be accepted, got {accepted}");
|
||||
let rejection = results
|
||||
.iter()
|
||||
.find_map(|result| result.as_ref().err())
|
||||
.expect("the losing concurrent restore must surface an error");
|
||||
assert_eq!(
|
||||
rejection.code(),
|
||||
&S3ErrorCode::RestoreAlreadyInProgress,
|
||||
"the losing concurrent restore must be rejected as already in progress: {rejection:?}"
|
||||
);
|
||||
|
||||
get_barrier.wait_until_paused().await;
|
||||
get_barrier.release();
|
||||
|
||||
// This test is scoped to the accept CAS: completion, expiry metadata, and
|
||||
// local restored GET service are covered by the single-request restore test
|
||||
// above. Here it is enough to prove exactly one copy-back was admitted.
|
||||
let expected_tier_gets = tier_gets_before_restore + 1;
|
||||
let deadline = tokio::time::Instant::now() + TRANSITION_WAIT_TIMEOUT;
|
||||
loop {
|
||||
let actual_tier_gets = backend.get_count().await;
|
||||
if actual_tier_gets >= expected_tier_gets {
|
||||
assert_eq!(
|
||||
actual_tier_gets - tier_gets_before_restore,
|
||||
1,
|
||||
"two concurrent restore requests must trigger exactly one tier copy-back GET"
|
||||
);
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
let op_log = backend.op_log().await;
|
||||
panic!(
|
||||
"mock tier should record exactly one restore GET within {TRANSITION_WAIT_TIMEOUT:?}; \
|
||||
tier_gets_before_restore={tier_gets_before_restore}, actual_tier_gets={actual_tier_gets}, op_log={op_log:?}"
|
||||
);
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
@@ -1,498 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! ILM transition-engine tests that call no S3 use-case; owned by no operation batch.
|
||||
|
||||
use super::common::*;
|
||||
use super::*;
|
||||
|
||||
const ENV_GET_CODEC_STREAMING_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_ENABLE";
|
||||
|
||||
const ENV_GET_CODEC_STREAMING_ROLLOUT: &str = "RUSTFS_GET_CODEC_STREAMING_ROLLOUT";
|
||||
|
||||
const ENV_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED: &str = "RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED";
|
||||
|
||||
const ENV_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED: &str = "RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED";
|
||||
|
||||
const ENV_GET_CODEC_STREAMING_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_MIN_SIZE";
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn transition_wait_timeout_reports_metadata_state_and_read_failure() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("test-transition-wait-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
upload_test_object(&ecstore, &bucket, "plain.txt", b"not transitioned").await;
|
||||
|
||||
let pending = wait_for_transition(&ecstore, &bucket, "plain.txt", Duration::ZERO)
|
||||
.await
|
||||
.expect_err("an untransitioned object must still fail the observation deadline");
|
||||
assert!(pending.contains("transitioned_status=\"\""), "{pending}");
|
||||
assert!(pending.contains("compensation_pending="), "{pending}");
|
||||
assert!(pending.contains("compensation_running="), "{pending}");
|
||||
|
||||
let missing = wait_for_transition(&ecstore, &bucket, "missing.txt", Duration::ZERO)
|
||||
.await
|
||||
.expect_err("metadata read failures must not become a completed transition");
|
||||
assert!(missing.contains("metadata_error=ObjectNotFound"), "{missing}");
|
||||
}
|
||||
|
||||
async fn with_get_codec_streaming_remote_probe_env<F, Fut>(test_fn: F)
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = ()>,
|
||||
{
|
||||
let metrics_was_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
let result = std::panic::AssertUnwindSafe(temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_GET_CODEC_STREAMING_ENABLE, Some("true")),
|
||||
(ENV_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")),
|
||||
(ENV_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
],
|
||||
test_fn(),
|
||||
))
|
||||
.catch_unwind()
|
||||
.await;
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(metrics_was_enabled);
|
||||
if let Err(err) = result {
|
||||
std::panic::resume_unwind(err);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
fn get_transitioned_object_uses_remote_codec_fallback_path() {
|
||||
// CI's default test stack overflows this deep async path; run it on a dedicated large-stack thread.
|
||||
std::thread::Builder::new()
|
||||
.name("lifecycle-transition-remote-codec-fallback".to_string())
|
||||
.stack_size(16 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("remote codec fallback test runtime should build");
|
||||
|
||||
runtime.block_on(async {
|
||||
with_get_codec_streaming_remote_probe_env(|| async {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-get-remote-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/remote-codec-fallback.txt";
|
||||
let payload: Vec<u8> = (0..(1024 * 1024))
|
||||
.map(|index| u8::try_from(index % 251).expect("payload byte fits in u8"))
|
||||
.collect();
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
let transition_opts = ObjectOptions {
|
||||
transition: lifecycle::lifecycle_contract::TransitionOptions {
|
||||
status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.clone(),
|
||||
etag: uploaded.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: uploaded.version_id.map(|version| version.to_string()),
|
||||
versioned: true,
|
||||
mod_time: uploaded.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
ecstore
|
||||
.transition_object(bucket.as_str(), object, &transition_opts)
|
||||
.await
|
||||
.expect("Failed to transition object directly");
|
||||
|
||||
let transitioned = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before remote fallback GET");
|
||||
|
||||
assert_eq!(transitioned.transitioned_object.status, "complete");
|
||||
assert_eq!(transitioned.transitioned_object.tier, tier_name);
|
||||
assert!(!transitioned.transitioned_object.name.is_empty());
|
||||
assert!(backend.contains(&transitioned.transitioned_object.name).await);
|
||||
|
||||
let actual = read_object_bytes(&ecstore, bucket.as_str(), object).await;
|
||||
assert_eq!(actual, payload);
|
||||
})
|
||||
.await;
|
||||
});
|
||||
})
|
||||
.expect("remote codec fallback test thread should spawn")
|
||||
.join()
|
||||
.expect("remote codec fallback test thread should not panic");
|
||||
}
|
||||
|
||||
/// Regression test for rustfs/rustfs#4827: a duplicate transition task that runs
|
||||
/// after the winning transition completed must return early without a second
|
||||
/// tier upload and without clobbering the winner's remote object / metadata.
|
||||
///
|
||||
/// The duplicate models the residual race left after the in-flight enqueue dedup
|
||||
/// (#4839): a stale-snapshot task admitted after the winner released its claim.
|
||||
/// On an unversioned bucket the duplicate's metadata re-read is eligible for the
|
||||
/// GET metadata cache; before the fix the cache still held the pre-transition
|
||||
/// entry (transition_object never invalidated it after persisting), so the
|
||||
/// TRANSITION_COMPLETE early-return never fired and the duplicate streamed the
|
||||
/// already-deleted local data to the tier again (NotFound reader errors +
|
||||
/// rejected duplicate tier PUT).
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn duplicate_transition_task_skips_already_transitioned_version() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-api-dup-transition-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/duplicate-transition.txt";
|
||||
let payload = b"duplicate transition tasks must not re-upload";
|
||||
|
||||
// Unversioned bucket: the duplicate's re-read goes through the GET metadata
|
||||
// cache (versioned reads bypass it), which is the stale-read window under test.
|
||||
(*ecstore)
|
||||
.make_bucket(bucket.as_str(), &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("Failed to create unversioned test bucket");
|
||||
|
||||
let uploaded = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
let transition_opts = ObjectOptions {
|
||||
transition: lifecycle::lifecycle_contract::TransitionOptions {
|
||||
status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.clone(),
|
||||
etag: uploaded.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: uploaded.version_id.map(|version| version.to_string()),
|
||||
mod_time: uploaded.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Winner: transitions the object, persists transition_status=complete, and
|
||||
// deletes the local data.
|
||||
ecstore
|
||||
.transition_object(bucket.as_str(), object, &transition_opts)
|
||||
.await
|
||||
.expect("winning transition should succeed");
|
||||
|
||||
let puts_after_winner: Vec<String> = backend
|
||||
.op_log()
|
||||
.await
|
||||
.into_iter()
|
||||
.filter_map(|op| match op {
|
||||
MockWarmOp::Put { object, .. } => Some(object),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(puts_after_winner.len(), 1, "winner must upload exactly one tier object");
|
||||
let winner_remote_object = puts_after_winner[0].clone();
|
||||
|
||||
// Duplicate task with the same stale snapshot (same mod_time / etag), run
|
||||
// immediately so the pre-fix stale metadata-cache entry is still live. It
|
||||
// must observe transition_status=complete and return Ok without uploading.
|
||||
ecstore
|
||||
.transition_object(bucket.as_str(), object, &transition_opts)
|
||||
.await
|
||||
.expect("duplicate transition must return early without error");
|
||||
|
||||
let puts_after_duplicate = backend
|
||||
.op_log()
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|op| matches!(op, MockWarmOp::Put { .. }))
|
||||
.count();
|
||||
assert_eq!(puts_after_duplicate, 1, "duplicate transition must not attempt a second tier upload");
|
||||
|
||||
// The winner's remote object and local metadata must be intact.
|
||||
assert!(backend.contains(&winner_remote_object).await);
|
||||
let info = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object must remain transitioned after duplicate task");
|
||||
assert_eq!(info.transitioned_object.status, "complete");
|
||||
assert_eq!(info.transitioned_object.tier, tier_name);
|
||||
assert_eq!(
|
||||
info.transitioned_object.name, winner_remote_object,
|
||||
"duplicate transition must not repoint metadata at a new remote object"
|
||||
);
|
||||
assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn transition_rejects_stale_etag_before_remote_upload() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
let bucket = format!("test-api-transition-stale-etag-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/transition-stale-etag.txt";
|
||||
(*ecstore)
|
||||
.make_bucket(bucket.as_str(), &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("unversioned test bucket should be created");
|
||||
let original = upload_test_object(&ecstore, bucket.as_str(), object, b"original transition candidate").await;
|
||||
let replacement_payload = b"replacement committed before the stale transition starts";
|
||||
let replacement = upload_test_object(&ecstore, bucket.as_str(), object, replacement_payload).await;
|
||||
let transition_opts = ObjectOptions {
|
||||
transition: lifecycle::lifecycle_contract::TransitionOptions {
|
||||
status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name,
|
||||
etag: original.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: replacement.version_id.map(|version| version.to_string()),
|
||||
mod_time: replacement.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
ecstore
|
||||
.transition_object(bucket.as_str(), object, &transition_opts)
|
||||
.await
|
||||
.expect_err("stale ETag must reject the transition before remote upload");
|
||||
assert_eq!(backend.put_count().await, 0, "rejected stale work must not upload a remote candidate");
|
||||
assert_eq!(
|
||||
read_object_bytes(&ecstore, bucket.as_str(), object).await,
|
||||
replacement_payload,
|
||||
"the replacement object must remain local and readable"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn transition_commit_rejects_replaced_source() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
let bucket = format!("test-api-transition-replace-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/transition-replaced-source.txt";
|
||||
let original_payload = b"transition source that must not replace a concurrent overwrite";
|
||||
let replacement_payload = b"concurrent replacement must remain the authoritative local object";
|
||||
|
||||
(*ecstore)
|
||||
.make_bucket(bucket.as_str(), &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("unversioned test bucket should be created");
|
||||
let original = upload_test_object(&ecstore, bucket.as_str(), object, original_payload).await;
|
||||
let transition_opts = ObjectOptions {
|
||||
transition: lifecycle::lifecycle_contract::TransitionOptions {
|
||||
status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.clone(),
|
||||
etag: original.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let put_barrier = backend.arm_put_barrier().await;
|
||||
let transition_store = Arc::clone(&ecstore);
|
||||
let transition_bucket = bucket.clone();
|
||||
let transition = tokio::spawn(async move {
|
||||
transition_store
|
||||
.transition_object(&transition_bucket, object, &transition_opts)
|
||||
.await
|
||||
});
|
||||
put_barrier.wait_until_paused().await;
|
||||
|
||||
upload_test_object(&ecstore, bucket.as_str(), object, replacement_payload).await;
|
||||
put_barrier.release();
|
||||
|
||||
transition
|
||||
.await
|
||||
.expect("transition task should not panic")
|
||||
.expect_err("transition must reject a source replaced during remote upload");
|
||||
assert_eq!(
|
||||
backend.put_count().await,
|
||||
1,
|
||||
"the stale transition should upload exactly one remote candidate"
|
||||
);
|
||||
assert_eq!(
|
||||
backend.remove_count().await,
|
||||
1,
|
||||
"the uncommitted remote candidate must be removed after identity revalidation fails"
|
||||
);
|
||||
assert_eq!(
|
||||
backend.remove_versions().await,
|
||||
backend.put_versions().await,
|
||||
"cleanup must delete the exact object version returned by the tier PUT"
|
||||
);
|
||||
assert_eq!(
|
||||
backend.object_count().await,
|
||||
0,
|
||||
"the rejected transition must not leave a remote candidate"
|
||||
);
|
||||
assert_eq!(
|
||||
read_object_bytes(&ecstore, bucket.as_str(), object).await,
|
||||
replacement_payload,
|
||||
"the replacement object must remain readable and unmodified"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn cancelled_transition_waiting_for_prepared_reader_cleans_remote() {
|
||||
temp_env::async_with_vars([(ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
let bucket = format!("test-api-transition-reader-lock-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/transition-reader-lock.txt";
|
||||
let payload = b"prepared reader must keep transition commit behind the namespace lock".repeat(1024);
|
||||
|
||||
(*ecstore)
|
||||
.make_bucket(bucket.as_str(), &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("unversioned test bucket should be created");
|
||||
let original = upload_test_object(&ecstore, bucket.as_str(), object, &payload).await;
|
||||
let transition_opts = ObjectOptions {
|
||||
transition: lifecycle::lifecycle_contract::TransitionOptions {
|
||||
status: lifecycle::lifecycle_contract::TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name,
|
||||
etag: original.etag.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
version_id: original.version_id.map(|version| version.to_string()),
|
||||
mod_time: original.mod_time,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let put_barrier = backend.arm_put_barrier().await;
|
||||
let transition_store = Arc::clone(&ecstore);
|
||||
let transition_bucket = bucket.clone();
|
||||
let mut transition = tokio::spawn(async move {
|
||||
transition_store
|
||||
.transition_object(&transition_bucket, object, &transition_opts)
|
||||
.await
|
||||
});
|
||||
tokio::select! {
|
||||
result = &mut transition => panic!("transition ended before reaching the tier PUT barrier: {result:?}"),
|
||||
() = put_barrier.wait_until_paused() => {}
|
||||
}
|
||||
|
||||
let prepared_reader = ecstore
|
||||
.prepare_get_object_reader(bucket.as_str(), object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("prepared reader should resolve while the local source still exists");
|
||||
put_barrier.release();
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(200), &mut transition)
|
||||
.await
|
||||
.is_err(),
|
||||
"transition commit must wait while a prepared reader owns the namespace read lock"
|
||||
);
|
||||
|
||||
transition.abort();
|
||||
assert!(
|
||||
transition
|
||||
.await
|
||||
.expect_err("transition task should be cancelled")
|
||||
.is_cancelled(),
|
||||
"transition cancellation should not be reported as a task panic"
|
||||
);
|
||||
drop(prepared_reader);
|
||||
assert_eq!(backend.put_count().await, 1, "the transition should upload exactly one remote candidate");
|
||||
assert!(
|
||||
backend.wait_for_object_count(0, TRANSITION_WAIT_TIMEOUT).await,
|
||||
"cancelling while the commit waits for the namespace lock must clean the exact uploaded candidate"
|
||||
);
|
||||
assert_eq!(backend.remove_versions().await, backend.put_versions().await);
|
||||
assert_eq!(read_object_bytes(&ecstore, bucket.as_str(), object).await, payload);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let manager = create_isolated_manager(HybridStrategyConfig::default());
|
||||
let _ = manager.get_dirty_disks().await;
|
||||
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let _backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-capacity-transition-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
let payload = b"transition should mark dirty scope";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
|
||||
lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(ecstore.clone(), bucket.as_str())
|
||||
.await
|
||||
.expect("Failed to enqueue transitioned object");
|
||||
|
||||
let _ = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should transition before dirty scope assertion");
|
||||
|
||||
let dirty_disks = manager.get_dirty_disks().await;
|
||||
assert_eq!(dirty_disks.len(), disk_paths.len());
|
||||
|
||||
let actual_paths: std::collections::HashSet<_> = dirty_disks
|
||||
.into_iter()
|
||||
.map(|disk| stdfs::canonicalize(&disk.drive_path).unwrap().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
let expected_paths: std::collections::HashSet<_> = disk_paths
|
||||
.iter()
|
||||
.map(|path| stdfs::canonicalize(path).unwrap().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
assert_eq!(actual_paths, expected_paths);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn immediate_transition_timeout_eventually_completes_via_compensation() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
|
||||
let backend = register_mock_tier(&tier_name).await;
|
||||
|
||||
let bucket = format!("test-compensation-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "test/object.txt";
|
||||
let payload = b"transition compensation should eventually complete";
|
||||
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
set_bucket_lifecycle_transition_with_tier(bucket.as_str(), &tier_name)
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
with_forced_immediate_enqueue_timeout(&backend, || async {
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
})
|
||||
.await;
|
||||
|
||||
let info = wait_for_transition(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT)
|
||||
.await
|
||||
.expect("object should eventually transition after compensation backfill");
|
||||
|
||||
assert_eq!(backend.put_count().await, 1, "compensation should upload the version once");
|
||||
assert_eq!(info.transitioned_object.status, "complete");
|
||||
assert_eq!(info.transitioned_object.tier, tier_name);
|
||||
assert!(backend.contains(&info.transitioned_object.name).await);
|
||||
}
|
||||
@@ -49,5 +49,3 @@ pub(crate) mod gating_test_env;
|
||||
mod lifecycle_transition_api_test;
|
||||
#[cfg(test)]
|
||||
mod put_prelookup_gating_test;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_seams;
|
||||
|
||||
@@ -2011,10 +2011,12 @@ mod tests {
|
||||
.version_id(requested_version)
|
||||
.build()
|
||||
.expect("HEAD request");
|
||||
let get_error = crate::app::test_seams::get_object(&usecase, build_request(get, Method::GET))
|
||||
let get_error = usecase
|
||||
.execute_get_object(build_request(get, Method::GET))
|
||||
.await
|
||||
.expect_err("marker GET");
|
||||
let head_error = crate::app::test_seams::head_object(&usecase, build_request(head, Method::HEAD))
|
||||
let head_error = usecase
|
||||
.execute_head_object(build_request(head, Method::HEAD))
|
||||
.await
|
||||
.expect_err("marker HEAD");
|
||||
for (method, error) in [("GET", get_error), ("HEAD", head_error)] {
|
||||
@@ -2061,7 +2063,8 @@ mod tests {
|
||||
.version_id(Some(original_id.clone()))
|
||||
.build()
|
||||
.expect("historical GET request");
|
||||
let response = crate::app::test_seams::get_object(&usecase, build_request(get, Method::GET))
|
||||
let response = usecase
|
||||
.execute_get_object(build_request(get, Method::GET))
|
||||
.await
|
||||
.expect("historical GET");
|
||||
assert!(!response.headers.contains_key("x-amz-delete-marker"));
|
||||
@@ -2077,7 +2080,8 @@ mod tests {
|
||||
.version_id(Some(original_id.clone()))
|
||||
.build()
|
||||
.expect("historical HEAD request");
|
||||
let response = crate::app::test_seams::head_object(&usecase, build_request(head, Method::HEAD))
|
||||
let response = usecase
|
||||
.execute_head_object(build_request(head, Method::HEAD))
|
||||
.await
|
||||
.expect("historical HEAD");
|
||||
assert_eq!(response.output.content_length, Some(i64::try_from(payload.len()).unwrap()));
|
||||
@@ -2097,10 +2101,12 @@ mod tests {
|
||||
.version_id(version)
|
||||
.build()
|
||||
.expect("missing HEAD request");
|
||||
let get_error = crate::app::test_seams::get_object(&usecase, build_request(get, Method::GET))
|
||||
let get_error = usecase
|
||||
.execute_get_object(build_request(get, Method::GET))
|
||||
.await
|
||||
.expect_err("missing GET");
|
||||
let head_error = crate::app::test_seams::head_object(&usecase, build_request(head, Method::HEAD))
|
||||
let head_error = usecase
|
||||
.execute_head_object(build_request(head, Method::HEAD))
|
||||
.await
|
||||
.expect_err("missing HEAD");
|
||||
for error in [get_error, head_error] {
|
||||
@@ -2195,7 +2201,7 @@ mod tests {
|
||||
.version_id(Some(marker_id.clone()))
|
||||
.build()
|
||||
.unwrap();
|
||||
let get_err = Box::pin(crate::app::test_seams::get_object(&usecase, build_request(get, Method::GET)))
|
||||
let get_err = Box::pin(usecase.execute_get_object(build_request(get, Method::GET)))
|
||||
.await
|
||||
.expect_err("GET of a marker remains forbidden");
|
||||
assert_eq!(get_err.code(), &S3ErrorCode::MethodNotAllowed);
|
||||
@@ -2205,7 +2211,7 @@ mod tests {
|
||||
.version_id(Some(marker_id.clone()))
|
||||
.build()
|
||||
.unwrap();
|
||||
let head_err = Box::pin(crate::app::test_seams::head_object(&usecase, build_request(head, Method::HEAD)))
|
||||
let head_err = Box::pin(usecase.execute_head_object(build_request(head, Method::HEAD)))
|
||||
.await
|
||||
.expect_err("HEAD of a marker remains forbidden");
|
||||
assert_eq!(head_err.code(), &S3ErrorCode::MethodNotAllowed);
|
||||
@@ -2281,7 +2287,7 @@ mod tests {
|
||||
.version_id(Some(marker_id))
|
||||
.build()
|
||||
.unwrap();
|
||||
let missing = Box::pin(crate::app::test_seams::get_object(&usecase, build_request(removed, Method::GET)))
|
||||
let missing = Box::pin(usecase.execute_get_object(build_request(removed, Method::GET)))
|
||||
.await
|
||||
.expect_err("the marker version must actually be gone");
|
||||
assert_eq!(
|
||||
@@ -2294,7 +2300,7 @@ mod tests {
|
||||
.key(key.to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let response = Box::pin(crate::app::test_seams::get_object(&usecase, build_request(get, Method::GET)))
|
||||
let response = Box::pin(usecase.execute_get_object(build_request(get, Method::GET)))
|
||||
.await
|
||||
.expect("GET restores the historical payload");
|
||||
let mut body = response.output.body.expect("GET body");
|
||||
|
||||
@@ -2674,7 +2674,9 @@ mod tests {
|
||||
.expect("metadata GET input must build");
|
||||
let metadata_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
let metadata_get = tokio::spawn(async move {
|
||||
crate::app::test_seams::get_object(&metadata_usecase, build_request(metadata_input, Method::GET)).await
|
||||
metadata_usecase
|
||||
.execute_get_object(build_request(metadata_input, Method::GET))
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(2), metadata_entered.wait())
|
||||
.await
|
||||
@@ -2703,10 +2705,7 @@ mod tests {
|
||||
.build()
|
||||
.expect("GET input must build");
|
||||
let get_usecase = DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
let get =
|
||||
tokio::spawn(
|
||||
async move { crate::app::test_seams::get_object(&get_usecase, build_request(get_input, Method::GET)).await },
|
||||
);
|
||||
let get = tokio::spawn(async move { get_usecase.execute_get_object(build_request(get_input, Method::GET)).await });
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while !object_traffic_health.read_storage_stalled_for_test() {
|
||||
tokio::task::yield_now().await;
|
||||
@@ -4226,7 +4225,7 @@ mod tests {
|
||||
.expect("SSE-C HEAD input must build");
|
||||
let mut head_request = build_request(head_input, Method::HEAD);
|
||||
head_request.headers = read_headers.clone();
|
||||
let head = Box::pin(crate::app::test_seams::head_object(&usecase, head_request))
|
||||
let head = Box::pin(usecase.execute_head_object(head_request))
|
||||
.await
|
||||
.expect("matching SSE-C key must authorize HEAD");
|
||||
assert_eq!(head.output.server_side_encryption, None);
|
||||
@@ -4243,7 +4242,7 @@ mod tests {
|
||||
.expect("SSE-C GET input must build");
|
||||
let mut get_request = build_request(get_input, Method::GET);
|
||||
get_request.headers = read_headers;
|
||||
let get = Box::pin(crate::app::test_seams::get_object(&usecase, get_request))
|
||||
let get = Box::pin(usecase.execute_get_object(get_request))
|
||||
.await
|
||||
.expect("matching SSE-C key must authorize GET");
|
||||
assert_eq!(get.output.server_side_encryption, None);
|
||||
|
||||
@@ -59,8 +59,6 @@ pub(crate) mod s3 {
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::dto::{GetObjectInput, HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::dto::{GetObjectOutput, HeadObjectOutput, PutObjectInput, PutObjectOutput};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::dto::{ListObjectsInput, StreamingBlob, UploadPartInput, UploadPartOutput};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::service::{S3Service, S3ServiceBuilder};
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::app::object_usecase::DefaultObjectUsecase;
|
||||
use crate::app::storage_api::s3::{GetObjectInput, GetObjectOutput, S3Request, S3Response, S3Result};
|
||||
use std::future::Future;
|
||||
|
||||
/// Calls `DefaultObjectUsecase::execute_get_object` with today's arguments.
|
||||
///
|
||||
/// Owned by the GetObject batch (C). Tests outside that batch's files call this seam instead of the use-case method;
|
||||
/// when batch C changes the use-case signature it changes only this body.
|
||||
pub(crate) fn get_object(
|
||||
usecase: &DefaultObjectUsecase,
|
||||
req: S3Request<GetObjectInput>,
|
||||
) -> impl Future<Output = S3Result<S3Response<GetObjectOutput>>> + '_ {
|
||||
usecase.execute_get_object(req)
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::app::object_usecase::DefaultObjectUsecase;
|
||||
use crate::app::storage_api::s3::{HeadObjectInput, HeadObjectOutput, S3Request, S3Response, S3Result};
|
||||
use std::future::Future;
|
||||
|
||||
/// Calls `DefaultObjectUsecase::execute_head_object` with today's arguments.
|
||||
///
|
||||
/// Owned by the HeadObject batch (C). Tests outside that batch's files call this seam instead of the use-case method;
|
||||
/// when batch C changes the use-case signature it changes only this body.
|
||||
pub(crate) fn head_object(
|
||||
usecase: &DefaultObjectUsecase,
|
||||
req: S3Request<HeadObjectInput>,
|
||||
) -> impl Future<Output = S3Result<S3Response<HeadObjectOutput>>> + '_ {
|
||||
usecase.execute_head_object(req)
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::app::storage_api::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response, S3Result};
|
||||
use std::future::Future;
|
||||
|
||||
/// Calls `DefaultBucketUsecase::execute_list_objects_v2` with today's arguments.
|
||||
///
|
||||
/// Owned by the ListObjectsV2 batch (B2). Tests outside that batch's files call this seam instead of the use-case
|
||||
/// method; when batch B2 changes the use-case signature it changes only this body.
|
||||
pub(crate) fn list_objects_v2(
|
||||
usecase: &DefaultBucketUsecase,
|
||||
req: S3Request<ListObjectsV2Input>,
|
||||
) -> impl Future<Output = S3Result<S3Response<ListObjectsV2Output>>> + '_ {
|
||||
usecase.execute_list_objects_v2(req)
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Test-only seams over the use-case entry points that tests of other operation families call as an oracle.
|
||||
//!
|
||||
//! Each seam is owned by the batch that owns its operation. A test outside that family calls the seam, never the
|
||||
//! use-case method, so a signature change touches one seam body instead of another family's tests.
|
||||
//! `scripts/check_test_seams.py` enforces this.
|
||||
|
||||
mod get_object;
|
||||
mod head_object;
|
||||
mod list_objects_v2;
|
||||
mod put_object;
|
||||
|
||||
pub(crate) use get_object::get_object;
|
||||
pub(crate) use head_object::head_object;
|
||||
pub(crate) use list_objects_v2::list_objects_v2;
|
||||
pub(crate) use put_object::put_object;
|
||||
@@ -1,30 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::app::object_usecase::DefaultObjectUsecase;
|
||||
use crate::app::storage_api::s3::{PutObjectInput, PutObjectOutput, S3Request, S3Response, S3Result};
|
||||
use crate::app::storage_api::test::ecfs::FS;
|
||||
use std::future::Future;
|
||||
|
||||
/// Calls `DefaultObjectUsecase::execute_put_object` with today's arguments.
|
||||
///
|
||||
/// Owned by the PutObject batch (D). Tests outside that batch's files call this seam instead of the use-case method;
|
||||
/// when batch D changes the use-case signature it changes only this body.
|
||||
pub(crate) fn put_object<'a>(
|
||||
usecase: &'a DefaultObjectUsecase,
|
||||
fs: &'a FS,
|
||||
req: S3Request<PutObjectInput>,
|
||||
) -> impl Future<Output = S3Result<S3Response<PutObjectOutput>>> + 'a {
|
||||
usecase.execute_put_object(fs, req)
|
||||
}
|
||||
@@ -0,0 +1,752 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Test-only source reader for text guards that must survive a mechanical file split.
|
||||
//!
|
||||
//! A text guard asserts that some literal appears in production source. When it reads one file
|
||||
//! with `include_str!`, moving the guarded function into another file either breaks the guard or,
|
||||
//! worse, makes it stop seeing the code it guards. A guard built on [`SourceSet`] instead names
|
||||
//! the file it reads today plus the sibling directory a split moves that file's code into, and
|
||||
//! reads every `.rs` file of both at test time.
|
||||
//!
|
||||
//! The named file is required: a missing file panics with its path, so a guard never passes on
|
||||
//! absent input. The directory is optional while the split has not happened; once it exists,
|
||||
//! every `.rs` file under it (recursively) is read in byte order of its manifest-relative path,
|
||||
//! after the named file.
|
||||
//!
|
||||
//! [`SourceSet::production`] keeps production text only, because a guard about production code
|
||||
//! must not be satisfied by a test that merely mentions the guarded call: it leaves out files
|
||||
//! that are test-only by path (a `tests` directory, `tests.rs`, `*_test.rs`, `*_tests.rs`,
|
||||
//! `test_support.rs`, a file with `#![cfg(test)]`, or a child of a `#[cfg(test)] mod x;`) and
|
||||
//! removes every item annotated `#[cfg(test)]` or `#[cfg(all(test, ...))]`. The stripper relies on
|
||||
//! rustfmt's layout and panics, naming the file and line, on any shape it does not recognise
|
||||
//! instead of guessing which text to keep.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// One source file of a [`SourceSet`].
|
||||
pub(crate) struct SourceFile {
|
||||
/// Path relative to the `rustfs` package root, with `/` separators.
|
||||
pub(crate) path: String,
|
||||
pub(crate) text: String,
|
||||
}
|
||||
|
||||
/// The ordered files a text guard reads: one required file, then the split directory.
|
||||
pub(crate) struct SourceSet {
|
||||
files: Vec<SourceFile>,
|
||||
}
|
||||
|
||||
impl SourceSet {
|
||||
/// Production text of `file` and of every `.rs` file under `split_dir`, both relative to the
|
||||
/// `rustfs` package root.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics, naming the path, when `file` is missing, unreadable or test-only, when `split_dir`
|
||||
/// exists but is not a directory, when a file cannot be read, or when a `#[cfg(test)]` item
|
||||
/// has a shape the stripper does not recognise.
|
||||
pub(crate) fn production(file: &str, split_dir: &str) -> Self {
|
||||
Self::production_under(Path::new(env!("CARGO_MANIFEST_DIR")), file, split_dir)
|
||||
}
|
||||
|
||||
/// [`SourceSet::production`] under another package root, for guard fixtures.
|
||||
pub(crate) fn production_under(root: &Path, file: &str, split_dir: &str) -> Self {
|
||||
assert!(
|
||||
!is_test_only_path(file),
|
||||
"guard source {file} is a test-only file; a production guard cannot be about it"
|
||||
);
|
||||
let mut stripped = Vec::new();
|
||||
let mut test_module_paths = Vec::new();
|
||||
for source in Self::read_under(root, file, split_dir).files {
|
||||
if is_test_only_path(&source.path) {
|
||||
continue;
|
||||
}
|
||||
let Some((text, test_modules)) = strip_test_items(&source.path, &source.text) else {
|
||||
assert!(source.path != file, "guard source {file} is test-only (#![cfg(test)])");
|
||||
continue;
|
||||
};
|
||||
let children = child_module_dir(&source.path);
|
||||
test_module_paths.extend(test_modules.into_iter().map(|name| format!("{children}/{name}")));
|
||||
stripped.push(SourceFile { path: source.path, text });
|
||||
}
|
||||
stripped.retain(|source| {
|
||||
!test_module_paths
|
||||
.iter()
|
||||
.any(|module| source.path == format!("{module}.rs") || source.path.starts_with(&format!("{module}/")))
|
||||
});
|
||||
Self { files: stripped }
|
||||
}
|
||||
|
||||
fn read_under(root: &Path, file: &str, split_dir: &str) -> Self {
|
||||
let required = root.join(file);
|
||||
let text = std::fs::read_to_string(&required)
|
||||
.unwrap_or_else(|err| panic!("guard source {} must exist and be readable: {err}", required.display()));
|
||||
let mut files = vec![SourceFile {
|
||||
path: file.to_owned(),
|
||||
text,
|
||||
}];
|
||||
|
||||
let dir = root.join(split_dir);
|
||||
if dir.exists() {
|
||||
assert!(dir.is_dir(), "guard split path {} must be a directory", dir.display());
|
||||
let mut paths = Vec::new();
|
||||
collect_rust_files(&dir, &mut paths);
|
||||
let mut split_files: Vec<SourceFile> = paths
|
||||
.into_iter()
|
||||
.map(|path| {
|
||||
let text = std::fs::read_to_string(&path)
|
||||
.unwrap_or_else(|err| panic!("guard source {} must be readable: {err}", path.display()));
|
||||
let relative = path
|
||||
.strip_prefix(root)
|
||||
.unwrap_or_else(|_| panic!("guard source {} must be under {}", path.display(), root.display()));
|
||||
let path = relative
|
||||
.components()
|
||||
.map(|component| component.as_os_str().to_string_lossy())
|
||||
.collect::<Vec<_>>()
|
||||
.join("/");
|
||||
SourceFile { path, text }
|
||||
})
|
||||
.collect();
|
||||
split_files.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
files.extend(split_files);
|
||||
}
|
||||
|
||||
Self { files }
|
||||
}
|
||||
|
||||
/// Whether any one file of the set contains `needle`.
|
||||
pub(crate) fn contains(&self, needle: &str) -> bool {
|
||||
self.files.iter().any(|file| file.text.contains(needle))
|
||||
}
|
||||
|
||||
/// Non-overlapping occurrences of `needle`, summed over the files; a match never spans two
|
||||
/// files.
|
||||
pub(crate) fn count(&self, needle: &str) -> usize {
|
||||
self.files.iter().map(|file| file.text.matches(needle).count()).sum()
|
||||
}
|
||||
|
||||
/// The text of the one `async fn name` method of the set: from its definition line, which
|
||||
/// must be indented by exactly four spaces (a method of a top-level `impl`), through the first
|
||||
/// following line that is exactly ` }`. A neighbouring method of any visibility, a test
|
||||
/// module, or the rest of the file is never part of it.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics unless exactly one such definition exists across the set and it has that closing
|
||||
/// line.
|
||||
pub(crate) fn method_body(&self, name: &str) -> &str {
|
||||
let definitions: Vec<(&SourceFile, usize)> = self
|
||||
.files
|
||||
.iter()
|
||||
.flat_map(|file| {
|
||||
line_starts(&file.text)
|
||||
.into_iter()
|
||||
.filter(move |&start| is_method_definition(line_at(&file.text, start), name))
|
||||
.map(move |start| (file, start))
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
definitions.len(),
|
||||
1,
|
||||
"method {name} must be defined exactly once, as a four-space-indented `async fn`, across {}",
|
||||
self.describe()
|
||||
);
|
||||
let (file, start) = definitions[0];
|
||||
let end = line_starts(&file.text)
|
||||
.into_iter()
|
||||
.filter(|&line| line > start)
|
||||
.find(|&line| line_at(&file.text, line) == " }")
|
||||
.unwrap_or_else(|| panic!("method {name} in {} has no closing ` }}` line", file.path));
|
||||
let close = end + " }".len();
|
||||
&file.text[start..close]
|
||||
}
|
||||
|
||||
/// Manifest-relative paths of the set, for panic messages.
|
||||
pub(crate) fn describe(&self) -> String {
|
||||
self.files
|
||||
.iter()
|
||||
.map(|file| file.path.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
}
|
||||
|
||||
/// Byte offsets at which each line of `text` starts.
|
||||
fn line_starts(text: &str) -> Vec<usize> {
|
||||
std::iter::once(0)
|
||||
.chain(text.match_indices('\n').map(|(index, _)| index + 1))
|
||||
.filter(|&start| start < text.len())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The line starting at `start`, without its line ending.
|
||||
fn line_at(text: &str, start: usize) -> &str {
|
||||
let rest = &text[start..];
|
||||
let line = rest.split('\n').next().unwrap_or(rest);
|
||||
line.strip_suffix('\r').unwrap_or(line)
|
||||
}
|
||||
|
||||
fn is_method_definition(line: &str, name: &str) -> bool {
|
||||
let Some(item) = line.strip_prefix(" ") else {
|
||||
return false;
|
||||
};
|
||||
if item.starts_with(' ') {
|
||||
return false;
|
||||
}
|
||||
let item = ["pub(crate) ", "pub(super) ", "pub "]
|
||||
.iter()
|
||||
.find_map(|visibility| item.strip_prefix(visibility))
|
||||
.unwrap_or(item);
|
||||
item.strip_prefix("async fn ")
|
||||
.and_then(|rest| rest.strip_prefix(name))
|
||||
.is_some_and(|rest| rest.starts_with(['(', '<']))
|
||||
}
|
||||
|
||||
fn is_test_only_path(path: &str) -> bool {
|
||||
let components: Vec<&str> = path.split('/').collect();
|
||||
let Some((file, directories)) = components.split_last() else {
|
||||
return false;
|
||||
};
|
||||
let stem = file.strip_suffix(".rs").unwrap_or(*file);
|
||||
directories.contains(&"tests")
|
||||
|| stem == "tests"
|
||||
|| stem == "test_support"
|
||||
|| stem.ends_with("_test")
|
||||
|| stem.ends_with("_tests")
|
||||
}
|
||||
|
||||
/// The directory that holds the child modules of the module in `path`.
|
||||
fn child_module_dir(path: &str) -> String {
|
||||
match path.strip_suffix("/mod.rs") {
|
||||
Some(dir) => dir.to_owned(),
|
||||
None => path.strip_suffix(".rs").unwrap_or(path).to_owned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn indentation(line: &str) -> usize {
|
||||
line.len() - line.trim_start_matches(' ').len()
|
||||
}
|
||||
|
||||
fn is_test_cfg_attribute(trimmed: &str) -> bool {
|
||||
trimmed == "#[cfg(test)]" || (trimmed.starts_with("#[cfg(all(test,") && trimmed.ends_with(")]"))
|
||||
}
|
||||
|
||||
/// `text` without its `#[cfg(test)]` items, plus the names of the `#[cfg(test)] mod name;`
|
||||
/// declarations it removed; `None` when the whole file is `#![cfg(test)]`.
|
||||
fn strip_test_items(path: &str, text: &str) -> Option<(String, Vec<String>)> {
|
||||
let lines: Vec<&str> = text.split_inclusive('\n').collect();
|
||||
let mut kept = String::with_capacity(text.len());
|
||||
let mut test_modules = Vec::new();
|
||||
let mut index = 0;
|
||||
while index < lines.len() {
|
||||
let trimmed = lines[index].trim();
|
||||
if trimmed == "#![cfg(test)]" {
|
||||
return None;
|
||||
}
|
||||
if !is_test_cfg_attribute(trimmed) {
|
||||
assert!(
|
||||
!(trimmed.starts_with("#[cfg(test)]") || trimmed.starts_with("#[cfg(all(test,")),
|
||||
"{path}:{}: a test-only item on its attribute line is not a shape the guard stripper reads",
|
||||
index + 1
|
||||
);
|
||||
kept.push_str(lines[index]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
let indent = indentation(lines[index]);
|
||||
let mut item = index + 1;
|
||||
while item < lines.len() && is_attribute_or_comment(lines[item].trim()) {
|
||||
item += 1;
|
||||
}
|
||||
assert!(item < lines.len(), "{path}:{}: #[cfg(test)] is not followed by an item", index + 1);
|
||||
assert_eq!(
|
||||
indentation(lines[item]),
|
||||
indent,
|
||||
"{path}:{}: the #[cfg(test)] item is not at its attribute's indentation",
|
||||
item + 1
|
||||
);
|
||||
if let Some(name) = declared_module(lines[item].trim()) {
|
||||
test_modules.push(name.to_owned());
|
||||
}
|
||||
index = test_item_end(path, &lines, item, indent) + 1;
|
||||
}
|
||||
Some((kept, test_modules))
|
||||
}
|
||||
|
||||
fn is_attribute_or_comment(trimmed: &str) -> bool {
|
||||
(trimmed.starts_with("#[") && trimmed.ends_with(']')) || trimmed.starts_with("//")
|
||||
}
|
||||
|
||||
fn declared_module(item: &str) -> Option<&str> {
|
||||
let item = ["pub(crate) ", "pub(super) ", "pub "]
|
||||
.iter()
|
||||
.find_map(|visibility| item.strip_prefix(visibility))
|
||||
.unwrap_or(item);
|
||||
let name = item.strip_prefix("mod ")?.strip_suffix(';')?;
|
||||
(!name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')).then_some(name)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Lexeme {
|
||||
Code,
|
||||
Str,
|
||||
RawStr(usize),
|
||||
BlockComment,
|
||||
}
|
||||
|
||||
fn is_ident(c: char) -> bool {
|
||||
c.is_alphanumeric() || c == '_'
|
||||
}
|
||||
|
||||
/// Index of the last line of the item that starts at `lines[start]` with indentation `indent`.
|
||||
fn test_item_end(path: &str, lines: &[&str], start: usize, indent: usize) -> usize {
|
||||
let mut depth: i64 = 0;
|
||||
let mut state = Lexeme::Code;
|
||||
// Inside a `where` clause a trailing comma separates bounds instead of ending the item.
|
||||
let mut in_where_clause = false;
|
||||
for (offset, line) in lines[start..].iter().enumerate() {
|
||||
let number = start + offset + 1;
|
||||
if state == Lexeme::Code && depth == 0 && indentation(line) == indent && line.trim().starts_with("where") {
|
||||
in_where_clause = true;
|
||||
}
|
||||
let chars: Vec<char> = line.chars().collect();
|
||||
let mut last_code = None;
|
||||
let mut i = 0;
|
||||
while i < chars.len() {
|
||||
let c = chars[i];
|
||||
let next = chars.get(i + 1).copied();
|
||||
match state {
|
||||
Lexeme::Code => match c {
|
||||
'/' if next == Some('/') => break,
|
||||
'/' if next == Some('*') => {
|
||||
state = Lexeme::BlockComment;
|
||||
i += 1;
|
||||
}
|
||||
'"' => state = Lexeme::Str,
|
||||
'r' if matches!(next, Some('"' | '#'))
|
||||
&& (i == 0
|
||||
|| !is_ident(chars[i - 1])
|
||||
|| (chars[i - 1] == 'b' && (i == 1 || !is_ident(chars[i - 2])))) =>
|
||||
{
|
||||
let mut quote = i + 1;
|
||||
while chars.get(quote) == Some(&'#') {
|
||||
quote += 1;
|
||||
}
|
||||
if chars.get(quote) == Some(&'"') {
|
||||
state = Lexeme::RawStr(quote - i - 1);
|
||||
i = quote;
|
||||
} else {
|
||||
last_code = Some(c);
|
||||
}
|
||||
}
|
||||
'\'' => {
|
||||
if next == Some('\\') {
|
||||
let close = chars
|
||||
.get(i + 3..)
|
||||
.and_then(|rest| rest.iter().position(|&candidate| candidate == '\''))
|
||||
.unwrap_or_else(|| panic!("{path}:{number}: unterminated character literal"));
|
||||
i += 3 + close;
|
||||
last_code = Some('\'');
|
||||
} else if chars.get(i + 2) == Some(&'\'') {
|
||||
i += 2;
|
||||
last_code = Some('\'');
|
||||
} else {
|
||||
last_code = Some(c);
|
||||
}
|
||||
}
|
||||
'{' | '(' | '[' => {
|
||||
depth += 1;
|
||||
last_code = Some(c);
|
||||
}
|
||||
'}' | ')' | ']' => {
|
||||
depth -= 1;
|
||||
assert!(depth >= 0, "{path}:{number}: unbalanced `{c}` in a #[cfg(test)] item");
|
||||
last_code = Some(c);
|
||||
}
|
||||
c if !c.is_whitespace() => last_code = Some(c),
|
||||
_ => {}
|
||||
},
|
||||
Lexeme::Str => match c {
|
||||
'\\' => i += 1,
|
||||
'"' => {
|
||||
state = Lexeme::Code;
|
||||
last_code = Some('"');
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Lexeme::RawStr(hashes) => {
|
||||
if c == '"' && chars.len() > i + hashes && chars[i + 1..=i + hashes].iter().all(|&h| h == '#') {
|
||||
i += hashes;
|
||||
state = Lexeme::Code;
|
||||
last_code = Some('"');
|
||||
}
|
||||
}
|
||||
Lexeme::BlockComment => {
|
||||
if c == '*' && next == Some('/') {
|
||||
state = Lexeme::Code;
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
if state != Lexeme::Code || depth != 0 {
|
||||
continue;
|
||||
}
|
||||
let line_indent = indentation(line);
|
||||
assert!(
|
||||
offset == 0 || line_indent >= indent,
|
||||
"{path}:{number}: a #[cfg(test)] item continues below its own indentation"
|
||||
);
|
||||
match last_code {
|
||||
Some(';') => return start + offset,
|
||||
Some(',') if !in_where_clause => return start + offset,
|
||||
Some('}') => {
|
||||
assert!(
|
||||
offset == 0 || line_indent == indent,
|
||||
"{path}:{number}: a #[cfg(test)] item block closes at indentation {line_indent}, expected {indent}"
|
||||
);
|
||||
return start + offset;
|
||||
}
|
||||
_ => {
|
||||
let following = lines
|
||||
.get(start + offset + 1)
|
||||
.unwrap_or_else(|| panic!("{path}:{number}: a #[cfg(test)] item never ends"));
|
||||
let following_trimmed = following.trim();
|
||||
assert!(
|
||||
indentation(following) > indent
|
||||
|| (indentation(following) == indent
|
||||
&& ["where", "{", "->", ".", ")", "]"]
|
||||
.iter()
|
||||
.any(|prefix| following_trimmed.starts_with(prefix))),
|
||||
"{path}:{}: the line after an unterminated #[cfg(test)] item line does not continue it",
|
||||
number + 1
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
panic!("{path}:{}: a #[cfg(test)] item never ends", start + 1);
|
||||
}
|
||||
|
||||
fn collect_rust_files(dir: &Path, out: &mut Vec<PathBuf>) {
|
||||
let entries =
|
||||
std::fs::read_dir(dir).unwrap_or_else(|err| panic!("guard split directory {} must be readable: {err}", dir.display()));
|
||||
for entry in entries {
|
||||
let entry = entry.unwrap_or_else(|err| panic!("guard split directory {} must be listable: {err}", dir.display()));
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
collect_rust_files(&path, out);
|
||||
} else if path.extension().is_some_and(|extension| extension == "rs") {
|
||||
out.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes `files` under a fresh temporary package root, for guard fixtures.
|
||||
pub(crate) fn fixture_root(files: &[(&str, &str)]) -> tempfile::TempDir {
|
||||
let root = tempfile::tempdir().expect("fixture root");
|
||||
for (relative, text) in files {
|
||||
let path = root.path().join(relative);
|
||||
std::fs::create_dir_all(path.parent().expect("fixture path has a parent")).expect("create fixture dir");
|
||||
std::fs::write(path, text).expect("write fixture");
|
||||
}
|
||||
root
|
||||
}
|
||||
|
||||
/// Runs `run`, which must panic, and returns its panic message.
|
||||
pub(crate) fn panic_message(run: impl FnOnce() + std::panic::UnwindSafe) -> String {
|
||||
let payload = std::panic::catch_unwind(run).expect_err("the guard input must be rejected");
|
||||
payload
|
||||
.downcast_ref::<String>()
|
||||
.cloned()
|
||||
.or_else(|| payload.downcast_ref::<&str>().map(|message| (*message).to_owned()))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{SourceSet, fixture_root, panic_message};
|
||||
|
||||
const MAIN: &str = "src/app/usecase.rs";
|
||||
const SPLIT: &str = "src/app/usecase";
|
||||
|
||||
#[test]
|
||||
fn absent_split_directory_reads_only_the_required_file() {
|
||||
let root = fixture_root(&[(MAIN, "guarded();\n")]);
|
||||
|
||||
let set = SourceSet::read_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
assert_eq!(set.describe(), MAIN);
|
||||
assert!(set.contains("guarded();"));
|
||||
assert_eq!(set.count("guarded();"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_directory_files_follow_the_required_file_in_path_order() {
|
||||
let root = fixture_root(&[
|
||||
(MAIN, "guarded();\n"),
|
||||
("src/app/usecase/zeta.rs", "guarded();\n"),
|
||||
("src/app/usecase/alpha.rs", "guarded();\nguarded();\n"),
|
||||
("src/app/usecase/tests/nested.rs", "guarded();\n"),
|
||||
("src/app/usecase/notes.txt", "guarded();\n"),
|
||||
]);
|
||||
|
||||
let set = SourceSet::read_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
assert_eq!(
|
||||
set.describe(),
|
||||
"src/app/usecase.rs, src/app/usecase/alpha.rs, src/app/usecase/tests/nested.rs, src/app/usecase/zeta.rs"
|
||||
);
|
||||
assert_eq!(set.count("guarded();"), 5, "non-.rs files are not source");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_required_file_fails_even_when_the_split_directory_exists() {
|
||||
let root = fixture_root(&[("src/app/usecase/moved.rs", "guarded();\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains(MAIN), "panic must name the missing file: {message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_path_that_is_a_file_fails() {
|
||||
let root = fixture_root(&[(MAIN, "guarded();\n"), (SPLIT, "not a directory\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains("must be a directory"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_does_not_join_a_needle_across_two_files() {
|
||||
let root = fixture_root(&[(MAIN, "guard"), ("src/app/usecase/tail.rs", "ed();\n")]);
|
||||
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
assert_eq!(set.count("guarded();"), 0);
|
||||
assert!(!set.contains("guarded();"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_text_keeps_production_items_and_drops_every_test_only_one() {
|
||||
let main = "\
|
||||
use crate::kept;
|
||||
#[cfg(test)]
|
||||
use crate::test_only_import;
|
||||
#[cfg(any(test, feature = \"x\"))]
|
||||
fn kept_in_some_production_build() { guarded(); }
|
||||
|
||||
impl Usecase {
|
||||
#[cfg(test)]
|
||||
field_for_tests: Option<u8>,
|
||||
#[cfg(test)]
|
||||
pub fn without_context() -> Self {
|
||||
guarded();
|
||||
}
|
||||
fn run(&self) {
|
||||
#[cfg(test)]
|
||||
let value = injected()
|
||||
.map_or(value, |err| guarded());
|
||||
#[cfg(test)]
|
||||
if let Some(hook) = self.hook {
|
||||
guarded();
|
||||
}
|
||||
guarded();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn helper<F>(f: F) -> u8
|
||||
where
|
||||
F: Fn(),
|
||||
{
|
||||
guarded();
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
const CANARY: &str = \"guarded();\";
|
||||
|
||||
#[cfg(test)]
|
||||
mod declared;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
const BRACES: &str = \"}}} {{\";
|
||||
const RAW: &[u8] = br#\"{\"a\": \"}\"}\"#;
|
||||
fn t() { let c = '}'; let d = '\\''; guarded(); }
|
||||
}
|
||||
";
|
||||
let root = fixture_root(&[
|
||||
(MAIN, main),
|
||||
("src/app/usecase/declared.rs", "guarded();\n"),
|
||||
("src/app/usecase/declared/deeper.rs", "guarded();\n"),
|
||||
("src/app/usecase/ops.rs", "fn op() { guarded(); }\n"),
|
||||
("src/app/usecase/tests/op_tests.rs", "guarded();\n"),
|
||||
("src/app/usecase/ops_test.rs", "guarded();\n"),
|
||||
("src/app/usecase/test_support.rs", "guarded();\n"),
|
||||
("src/app/usecase/gated.rs", "#![cfg(test)]\nguarded();\n"),
|
||||
]);
|
||||
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
assert_eq!(set.describe(), "src/app/usecase.rs, src/app/usecase/ops.rs");
|
||||
assert_eq!(set.count("guarded();"), 3, "production calls: kept_in_some_production_build, run, op");
|
||||
assert!(set.contains("use crate::kept;") && set.contains("impl Usecase {") && set.contains("fn run(&self) {"));
|
||||
assert!(!set.contains("test_only_import") && !set.contains("without_context") && !set.contains("mod tests"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_test_only_item_on_its_attribute_line_fails_loudly() {
|
||||
let root = fixture_root(&[(MAIN, "#[cfg(test)] use crate::hidden;\nguarded();\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains("src/app/usecase.rs:1"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_test_only_item_that_never_closes_fails_loudly() {
|
||||
let root = fixture_root(&[(MAIN, "#[cfg(test)]\nmod tests {\n fn t() {}\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains("never ends"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_test_only_block_closing_at_another_indentation_fails_loudly() {
|
||||
let root = fixture_root(&[(MAIN, "impl X {\n #[cfg(test)]\n fn t() {\n}\n fn kept() {}\n}\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains("src/app/usecase.rs:4"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unterminated_test_only_line_followed_by_production_fails_loudly() {
|
||||
let root = fixture_root(&[(MAIN, "fn f() {\n #[cfg(test)]\n let hook = injected()\n guarded();\n}\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, MAIN, SPLIT);
|
||||
});
|
||||
|
||||
assert!(message.contains("does not continue it"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_test_only_required_file_is_rejected() {
|
||||
let root = fixture_root(&[("src/app/usecase_test.rs", "guarded();\n")]);
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::production_under(&root_path, "src/app/usecase_test.rs", "src/app/usecase_test");
|
||||
});
|
||||
|
||||
assert!(message.contains("test-only"), "{message}");
|
||||
}
|
||||
|
||||
const METHODS: &str = "\
|
||||
impl Usecase {
|
||||
pub async fn guarded(&self) {
|
||||
first();
|
||||
}
|
||||
|
||||
pub(crate) async fn neighbour(&self) {
|
||||
needle();
|
||||
}
|
||||
|
||||
pub(super) async fn other(&self) {
|
||||
needle();
|
||||
}
|
||||
}
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn method_body_stops_at_its_own_closing_line_before_any_neighbour() {
|
||||
let root = fixture_root(&[
|
||||
(MAIN, METHODS),
|
||||
("src/app/usecase/later.rs", "impl Usecase {\n fn x() { needle(); }\n}\n"),
|
||||
]);
|
||||
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
let body = set.method_body("guarded");
|
||||
|
||||
assert_eq!(body, " pub async fn guarded(&self) {\n first();\n }");
|
||||
assert!(!body.contains("needle"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_body_of_the_last_method_stops_before_the_test_module() {
|
||||
let text = "impl Usecase {\n async fn last(&self) {\n first();\n }\n}\n\n#[cfg(test)]\nmod tests {\n fn t() { needle(); }\n}\n";
|
||||
let root = fixture_root(&[("src/app/usecase/moved.rs", text), (MAIN, "\n")]);
|
||||
|
||||
let set = SourceSet::read_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
assert!(!set.method_body("last").contains("needle"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_body_does_not_match_a_longer_name_or_a_nested_definition() {
|
||||
let text = "impl Usecase {\n pub async fn guarded_more(&self) {}\n async fn guarded(&self) {}\n}\n";
|
||||
let root = fixture_root(&[(MAIN, text)]);
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
let message = panic_message(move || {
|
||||
set.method_body("guarded");
|
||||
});
|
||||
|
||||
assert!(message.contains("exactly once"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_body_defined_in_two_files_fails() {
|
||||
let root = fixture_root(&[(MAIN, METHODS), ("src/app/usecase/copy.rs", METHODS)]);
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
let message = panic_message(move || {
|
||||
set.method_body("guarded");
|
||||
});
|
||||
|
||||
assert!(message.contains("src/app/usecase/copy.rs"), "panic must name the set: {message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn method_body_without_a_closing_line_fails() {
|
||||
let root = fixture_root(&[(MAIN, "impl Usecase {\n async fn guarded(&self) {\n first();\n }\n}\n")]);
|
||||
let set = SourceSet::production_under(root.path(), MAIN, SPLIT);
|
||||
|
||||
let message = panic_message(move || {
|
||||
set.method_body("guarded");
|
||||
});
|
||||
|
||||
assert!(message.contains("no closing"), "{message}");
|
||||
}
|
||||
}
|
||||
@@ -89,6 +89,8 @@ pub mod diagnose;
|
||||
pub mod embedded;
|
||||
pub mod error;
|
||||
pub mod foreground_observation;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod guard_sources;
|
||||
pub mod init;
|
||||
pub mod inspect;
|
||||
pub(crate) mod kms_deletion_gate;
|
||||
|
||||
@@ -1334,12 +1334,16 @@ fn test_delete_bucket_broadcast_fences_target_membership_through_delivery() {
|
||||
"a failed destructive delivery must remain visible for operator repair"
|
||||
);
|
||||
|
||||
let usecase = include_str!("../app/bucket_usecase.rs");
|
||||
let delete = usecase
|
||||
.split("async fn execute_delete_bucket_inner")
|
||||
.nth(1)
|
||||
.and_then(|rest| rest.split("pub async fn execute_head_bucket").next())
|
||||
.expect("delete bucket usecase");
|
||||
assert_delete_bucket_reserves_before_local_delete(&crate::guard_sources::SourceSet::production(
|
||||
"src/app/bucket_usecase.rs",
|
||||
"src/app/bucket_usecase",
|
||||
));
|
||||
}
|
||||
|
||||
/// The reservation and the local delete must both be inside `execute_delete_bucket_inner`, in
|
||||
/// that order; the slice ends at that method's own closing line.
|
||||
fn assert_delete_bucket_reserves_before_local_delete(usecase: &crate::guard_sources::SourceSet) {
|
||||
let delete = usecase.method_body("execute_delete_bucket_inner");
|
||||
assert!(
|
||||
delete
|
||||
.find("prepare_site_replication_delete_bucket")
|
||||
@@ -1349,6 +1353,67 @@ fn test_delete_bucket_broadcast_fences_target_membership_through_delivery() {
|
||||
);
|
||||
}
|
||||
|
||||
const DELETE_BUCKET_INNER_FIXTURE: &str = "\
|
||||
impl DefaultBucketUsecase {
|
||||
async fn execute_delete_bucket_inner(&self) {
|
||||
let intent = prepare_site_replication_delete_bucket(&bucket, force).await?;
|
||||
LOCAL_DELETE
|
||||
}
|
||||
}
|
||||
";
|
||||
|
||||
fn delete_bucket_fixture(core: &str) -> crate::guard_sources::SourceSet {
|
||||
let root = crate::guard_sources::fixture_root(&[
|
||||
("src/app/bucket_usecase.rs", "pub struct DefaultBucketUsecase;\n"),
|
||||
("src/app/bucket_usecase/core.rs", core),
|
||||
]);
|
||||
crate::guard_sources::SourceSet::production_under(root.path(), "src/app/bucket_usecase.rs", "src/app/bucket_usecase")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_bucket_order_guard_passes_after_the_method_moves_into_the_split_directory() {
|
||||
let core = DELETE_BUCKET_INNER_FIXTURE.replace("LOCAL_DELETE", "store.delete_bucket(&bucket).await;");
|
||||
assert_delete_bucket_reserves_before_local_delete(&delete_bucket_fixture(&core));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_bucket_order_guard_fails_when_the_local_delete_leaves_the_method() {
|
||||
let core = format!(
|
||||
"{}\nimpl DefaultBucketUsecase {{\n pub(crate) async fn delete_helper(&self) {{\n store.delete_bucket(&bucket).await;\n }}\n}}\n",
|
||||
DELETE_BUCKET_INNER_FIXTURE.replace("LOCAL_DELETE", "")
|
||||
);
|
||||
let usecase = delete_bucket_fixture(&core);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_delete_bucket_reserves_before_local_delete(&usecase));
|
||||
|
||||
assert!(message.contains("local delete"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_bucket_order_guard_fails_when_the_delete_precedes_the_reservation() {
|
||||
let core = DELETE_BUCKET_INNER_FIXTURE.replace(
|
||||
" let intent = prepare_site_replication_delete_bucket(&bucket, force).await?;\n LOCAL_DELETE",
|
||||
" store.delete_bucket(&bucket).await;\n let intent = prepare_site_replication_delete_bucket(&bucket, force).await?;",
|
||||
);
|
||||
let usecase = delete_bucket_fixture(&core);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_delete_bucket_reserves_before_local_delete(&usecase));
|
||||
|
||||
assert!(message.contains("destructive peer liabilities"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_bucket_order_guard_fails_when_the_method_has_no_closing_line() {
|
||||
let core = DELETE_BUCKET_INNER_FIXTURE
|
||||
.replace("LOCAL_DELETE", "store.delete_bucket(&bucket).await;")
|
||||
.replace(" }\n}\n", "");
|
||||
let usecase = delete_bucket_fixture(&core);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_delete_bucket_reserves_before_local_delete(&usecase));
|
||||
|
||||
assert!(message.contains("no closing"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_retry_settlement_preserves_a_newer_same_path_failure() {
|
||||
let peer = peer("remote", "https://remote.example.com");
|
||||
|
||||
@@ -6900,27 +6900,23 @@ mod tests {
|
||||
));
|
||||
let usecase = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
assert_eq!(
|
||||
crate::app::test_seams::get_object(&usecase, get)
|
||||
.await
|
||||
.expect_err("bad GET part number")
|
||||
.code(),
|
||||
usecase.execute_get_object(get).await.expect_err("bad GET part number").code(),
|
||||
&S3ErrorCode::InvalidArgument
|
||||
);
|
||||
assert_eq!(
|
||||
crate::app::test_seams::head_object(&usecase, head)
|
||||
usecase
|
||||
.execute_head_object(head)
|
||||
.await
|
||||
.expect_err("range and part number conflict")
|
||||
.code(),
|
||||
&S3ErrorCode::InvalidArgument
|
||||
);
|
||||
assert_eq!(
|
||||
crate::app::test_seams::list_objects_v2(
|
||||
&crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context))),
|
||||
list
|
||||
)
|
||||
.await
|
||||
.expect_err("negative max keys")
|
||||
.code(),
|
||||
crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
|
||||
.execute_list_objects_v2(list)
|
||||
.await
|
||||
.expect_err("negative max keys")
|
||||
.code(),
|
||||
&S3ErrorCode::InvalidArgument
|
||||
);
|
||||
}
|
||||
@@ -7008,77 +7004,78 @@ mod tests {
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("created identity");
|
||||
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
|
||||
let usecase = crate::app::object_usecase::DefaultObjectUsecase::with_context(Some(Arc::clone(&context)));
|
||||
crate::app::test_seams::get_object(&usecase, get.clone())
|
||||
usecase
|
||||
.execute_get_object(get.clone())
|
||||
.await
|
||||
.expect("a local GET hit ignores source capture failure");
|
||||
crate::app::test_seams::head_object(&usecase, head.clone())
|
||||
usecase
|
||||
.execute_head_object(head.clone())
|
||||
.await
|
||||
.expect("a local HEAD hit ignores source capture failure");
|
||||
get.input.key = "missing".into();
|
||||
head.input.key = "missing".into();
|
||||
assert_eq!(
|
||||
crate::app::test_seams::get_object(&usecase, get.clone())
|
||||
usecase
|
||||
.execute_get_object(get.clone())
|
||||
.await
|
||||
.expect_err("failed capture cannot rebind on GET miss")
|
||||
.code(),
|
||||
&S3ErrorCode::NoSuchBucket
|
||||
);
|
||||
assert_eq!(
|
||||
crate::app::test_seams::head_object(&usecase, head.clone())
|
||||
usecase
|
||||
.execute_head_object(head.clone())
|
||||
.await
|
||||
.expect_err("failed capture cannot rebind on HEAD miss")
|
||||
.code(),
|
||||
&S3ErrorCode::NoSuchBucket
|
||||
);
|
||||
assert_eq!(
|
||||
crate::app::test_seams::list_objects_v2(
|
||||
&crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context))),
|
||||
list.clone()
|
||||
)
|
||||
.await
|
||||
.expect_err("failed capture cannot rebind on LIST")
|
||||
.code(),
|
||||
crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
|
||||
.execute_list_objects_v2(list.clone())
|
||||
.await
|
||||
.expect_err("failed capture cannot rebind on LIST")
|
||||
.code(),
|
||||
&S3ErrorCode::NoSuchBucket
|
||||
);
|
||||
config.filter.prefix = Some("remote/".into());
|
||||
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
|
||||
assert_eq!(
|
||||
crate::app::test_seams::get_object(&usecase, get.clone())
|
||||
usecase
|
||||
.execute_get_object(get.clone())
|
||||
.await
|
||||
.expect_err("filtered GET remains local")
|
||||
.code(),
|
||||
&S3ErrorCode::NoSuchKey
|
||||
);
|
||||
assert_eq!(
|
||||
crate::app::test_seams::head_object(&usecase, head.clone())
|
||||
usecase
|
||||
.execute_head_object(head.clone())
|
||||
.await
|
||||
.expect_err("filtered HEAD remains local")
|
||||
.code(),
|
||||
&S3ErrorCode::NoSuchKey
|
||||
);
|
||||
list.input.prefix = Some("local/".into());
|
||||
let listing = crate::app::test_seams::list_objects_v2(
|
||||
&crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context))),
|
||||
list.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("disjoint prefix needs no source identity");
|
||||
let listing = crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
|
||||
.execute_list_objects_v2(list.clone())
|
||||
.await
|
||||
.expect("disjoint prefix needs no source identity");
|
||||
assert_eq!(listing.output.key_count, Some(0));
|
||||
list.input.prefix = Some("remote/".into());
|
||||
list.input.max_keys = Some(0);
|
||||
let listing = crate::app::test_seams::list_objects_v2(
|
||||
&crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context))),
|
||||
list,
|
||||
)
|
||||
.await
|
||||
.expect("an empty page needs no source identity");
|
||||
let listing = crate::app::bucket_usecase::DefaultBucketUsecase::with_context(Some(Arc::clone(&context)))
|
||||
.execute_list_objects_v2(list)
|
||||
.await
|
||||
.expect("an empty page needs no source identity");
|
||||
assert_eq!(listing.output.key_count, Some(0));
|
||||
|
||||
config.filter.prefix = None;
|
||||
config.policy.head = crate::on_demand_migration::HeadPolicy::LocalOnly;
|
||||
sys.apply_for_incarnation(&bucket, incarnation, Some(&config)).await;
|
||||
assert_eq!(
|
||||
crate::app::test_seams::head_object(&usecase, head.clone())
|
||||
usecase
|
||||
.execute_head_object(head.clone())
|
||||
.await
|
||||
.expect_err("local-only HEAD needs no source identity")
|
||||
.code(),
|
||||
@@ -7098,14 +7095,16 @@ mod tests {
|
||||
.await
|
||||
.expect("create local delete marker");
|
||||
assert_eq!(
|
||||
crate::app::test_seams::get_object(&usecase, get)
|
||||
usecase
|
||||
.execute_get_object(get)
|
||||
.await
|
||||
.expect_err("GET respects the delete marker before capture failure")
|
||||
.code(),
|
||||
&S3ErrorCode::NoSuchKey
|
||||
);
|
||||
assert_eq!(
|
||||
crate::app::test_seams::head_object(&usecase, head)
|
||||
usecase
|
||||
.execute_head_object(head)
|
||||
.await
|
||||
.expect_err("HEAD respects the delete marker before capture failure")
|
||||
.code(),
|
||||
|
||||
+91
-1756
File diff suppressed because it is too large
Load Diff
@@ -53,7 +53,7 @@ mod tests {
|
||||
remove_object_lock_retention_metadata, set_bucket_metadata, validate_bucket_object_lock_enabled,
|
||||
validate_list_object_unordered_with_delimiter,
|
||||
};
|
||||
use http::{Extensions, HeaderMap, HeaderValue, Method, StatusCode, Uri};
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
|
||||
@@ -62,108 +62,14 @@ mod tests {
|
||||
};
|
||||
use rustfs_zip::CompressionFormat;
|
||||
use s3s::dto::{
|
||||
DefaultRetention, DeleteObjectTaggingInput, Delimiter, FilterRule, FilterRuleName, GetBucketAclInput, GetObjectAclInput,
|
||||
GetObjectLegalHoldInput, GetObjectRetentionInput, GetObjectTaggingInput, LambdaFunctionConfiguration,
|
||||
NotificationConfigurationFilter, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold,
|
||||
ObjectLockLegalHoldStatus, ObjectLockRetention, ObjectLockRetentionMode, ObjectLockRule, PutBucketAclInput,
|
||||
PutObjectAclInput, PutObjectLegalHoldInput, PutObjectLockConfigurationInput, PutObjectRetentionInput,
|
||||
PutObjectTaggingInput, QueueConfiguration, S3KeyFilter, Tag, Tagging, TopicConfiguration,
|
||||
DefaultRetention, Delimiter, FilterRule, FilterRuleName, LambdaFunctionConfiguration, NotificationConfigurationFilter,
|
||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention,
|
||||
ObjectLockRetentionMode, ObjectLockRule, QueueConfiguration, S3KeyFilter, TopicConfiguration,
|
||||
};
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request, s3_error};
|
||||
use s3s::{S3Error, S3ErrorCode, s3_error};
|
||||
use std::collections::HashMap;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
input,
|
||||
method,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_apis_report_storage_read_quorum_as_unavailable() {
|
||||
crate::app::gating_test_env::run_large_stack_test("list-quorum-surface", || async {
|
||||
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use s3s::dto::{ListObjectVersionsInput, ListObjectsInput, ListObjectsV2Input};
|
||||
crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let bucket = format!("list-quorum-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create listing bucket");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let saved_disks = set.disks.read().await.clone();
|
||||
{
|
||||
let mut disks = set.disks.write().await;
|
||||
for disk in disks.iter_mut().skip(1) {
|
||||
*disk = None;
|
||||
}
|
||||
}
|
||||
let fs = FS::new();
|
||||
let errors = [
|
||||
S3::list_objects(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectsInput {
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("V1 cannot list below storage read quorum"),
|
||||
S3::list_objects_v2(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectsV2Input {
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("V2 cannot list below storage read quorum"),
|
||||
S3::list_object_versions(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectVersionsInput {
|
||||
bucket,
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("versions cannot list below storage read quorum"),
|
||||
];
|
||||
*set.disks.write().await = saved_disks;
|
||||
for error in errors {
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
|
||||
let source = error.source().expect("retain storage diagnostics");
|
||||
assert!(
|
||||
matches!(
|
||||
source.downcast_ref::<StorageError>(),
|
||||
Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
|
||||
),
|
||||
"the production LIST boundary must expose a typed read-quorum failure: {source}"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fs_creation() {
|
||||
let _fs = FS::new();
|
||||
@@ -293,164 +199,6 @@ mod tests {
|
||||
assert_eq!(gz_format.extension(), "gz");
|
||||
}
|
||||
|
||||
/// Premise guard for the `*_returns_internal_error_when_store_uninitialized`
|
||||
/// tests (backlog#1830): they assert the error path taken while the global
|
||||
/// object layer is absent. Under nextest — the authoritative runner — every
|
||||
/// test owns its process, so the premise always holds and the assertion
|
||||
/// always runs. Under the documented shared-process `cargo test` fallback a
|
||||
/// sibling test may have initialized the store first; the premise is then
|
||||
/// unattainable, so the test skips instead of asserting against a scenario
|
||||
/// it does not describe.
|
||||
fn store_uninitialized_premise_holds() -> bool {
|
||||
if crate::runtime_sources::current_object_store_handle().is_some() {
|
||||
eprintln!("skipping store-uninitialized assertion: a sibling test already initialized the global object layer");
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_acl(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_bucket_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetBucketAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_bucket_acl(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_legal_hold_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectLegalHoldInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_legal_hold(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_retention_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetObjectRetentionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_retention(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_legal_hold_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectLegalHoldInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_legal_hold(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_bucket_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutBucketAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_bucket_acl(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_acl(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_retention_returns_internal_error_when_store_uninitialized() {
|
||||
let input = PutObjectRetentionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_retention(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectLockConfigurationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.object_lock_configuration(Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs
|
||||
.put_object_lock_configuration(build_request(input, Method::PUT))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
/// The bucket default retention path reads the persisted shape; the request
|
||||
/// literals in these tests stay on the wire DTO and convert once here.
|
||||
fn persisted_lock(cfg: &ObjectLockConfiguration) -> rustfs_gateway_types::persistence::PersistedObjectLockConfiguration {
|
||||
@@ -777,104 +525,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_tagging(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_rejects_too_many_tags() {
|
||||
let tag_set = (0..11)
|
||||
.map(|index| Tag {
|
||||
key: Some(format!("k{index}")),
|
||||
value: Some(format!("v{index}")),
|
||||
})
|
||||
.collect();
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging { tag_set })
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Cannot have more than 10 tags per object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_rejects_empty_tag_key_before_store_lookup() {
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some(String::new()),
|
||||
value: Some("v1".to_string()),
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Tag key cannot be empty"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some("k".to_string()),
|
||||
value: Some("v".to_string()),
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delete_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = DeleteObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs
|
||||
.delete_object_tagging(build_request(input, Method::DELETE))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_adaptive_buffer_size_with_profile() {
|
||||
const KB: i64 = 1024;
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of bucket policy, policy status, versioning, notification and ACL: batch A2 of
|
||||
//! the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api::{self, acl};
|
||||
use crate::storage::storage_api::ecfs_consumer::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage::{persisted_from_s3s, persisted_to_s3s};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
/// Handler bodies that [`S3`] runs through `audit_only_operation`.
|
||||
impl FS {
|
||||
async fn put_bucket_acl_inner(&self, req: S3Request<PutBucketAclInput>) -> S3Result<S3Response<PutBucketAclOutput>> {
|
||||
let PutBucketAclInput {
|
||||
bucket,
|
||||
access_control_policy,
|
||||
..
|
||||
} = req.input;
|
||||
record_s3_op(S3Operation::PutBucketAcl);
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
if access_control_policy.is_some() {
|
||||
return Err(s3_error!(
|
||||
NotImplemented,
|
||||
"ACL XML grants are not supported; use canned ACL headers or omit ACL"
|
||||
));
|
||||
}
|
||||
|
||||
Ok(S3Response::new(PutBucketAclOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
impl FS {
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_policy_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketPolicyInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketPolicyOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketPolicy, |req| usecase.execute_delete_bucket_policy(req)).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_acl_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketAclInput>,
|
||||
) -> S3Result<S3Response<GetBucketAclOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketAcl);
|
||||
let GetBucketAclInput { bucket, .. } = req.input;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(acl::build_get_bucket_acl_output()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_notification_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketNotificationConfigurationInput>,
|
||||
) -> S3Result<S3Response<GetBucketNotificationConfigurationOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketNotificationConfiguration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let notification_configuration = usecase
|
||||
.execute_get_bucket_notification_configuration(req)
|
||||
.await?
|
||||
.as_ref()
|
||||
.map(persisted_to_s3s::<_, NotificationConfiguration>)
|
||||
.transpose()?;
|
||||
match notification_configuration {
|
||||
Some(NotificationConfiguration {
|
||||
event_bridge_configuration,
|
||||
lambda_function_configurations,
|
||||
queue_configurations,
|
||||
topic_configurations,
|
||||
}) => Ok(S3Response::new(GetBucketNotificationConfigurationOutput {
|
||||
event_bridge_configuration,
|
||||
lambda_function_configurations,
|
||||
queue_configurations,
|
||||
topic_configurations,
|
||||
})),
|
||||
None => Ok(S3Response::new(GetBucketNotificationConfigurationOutput::default())),
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_policy_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketPolicyInput>,
|
||||
) -> S3Result<S3Response<GetBucketPolicyOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketPolicy);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_get_bucket_policy(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_policy_status_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketPolicyStatusInput>,
|
||||
) -> S3Result<S3Response<GetBucketPolicyStatusOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketPolicyStatus);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_get_bucket_policy_status(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "get_bucket_versioning", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_versioning_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketVersioningInput>,
|
||||
) -> S3Result<S3Response<GetBucketVersioningOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketVersioning);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let config = usecase.execute_get_bucket_versioning(req).await?;
|
||||
Ok(S3Response::new(GetBucketVersioningOutput {
|
||||
status: config.status.map(BucketVersioningStatus::from),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_acl_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketAclInput>,
|
||||
) -> S3Result<S3Response<PutBucketAclOutput>> {
|
||||
audit_only_operation(req, S3Operation::PutBucketAcl, |req| self.put_bucket_acl_inner(req)).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_notification_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketNotificationConfigurationInput>,
|
||||
) -> S3Result<S3Response<PutBucketNotificationConfigurationOutput>> {
|
||||
let prepared = persisted_from_s3s(&req.input.notification_configuration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketNotificationConfiguration, |req| {
|
||||
usecase.execute_put_bucket_notification_configuration(req, prepared)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_policy_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketPolicyInput>,
|
||||
) -> S3Result<S3Response<PutBucketPolicyOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketPolicy, |req| usecase.execute_put_bucket_policy(req)).await
|
||||
}
|
||||
|
||||
#[instrument(name = "put_bucket_versioning", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_versioning_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketVersioningInput>,
|
||||
) -> S3Result<S3Response<PutBucketVersioningOutput>> {
|
||||
let prepared = persisted_from_s3s(&req.input.versioning_configuration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketVersioning, |req| {
|
||||
usecase.execute_put_bucket_versioning(req, prepared)
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of the bucket CORS, tagging, public access block and encryption configurations:
|
||||
//! batch A1 of the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use crate::storage::{persisted_from_s3s, persisted_to_s3s, tagging_from_s3s, tagging_to_s3s};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "delete_bucket_cors", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_cors_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketCorsInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketCorsOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketCors, |req| usecase.execute_delete_bucket_cors(req)).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_encryption_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketEncryptionInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketEncryptionOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketEncryption, |req| {
|
||||
usecase.execute_delete_bucket_encryption(req)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(name = "delete_bucket_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketTaggingInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketTaggingOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketTagging, |req| usecase.execute_delete_bucket_tagging(req)).await
|
||||
}
|
||||
|
||||
#[instrument(name = "delete_public_access_block", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_public_access_block_edge(
|
||||
&self,
|
||||
req: S3Request<DeletePublicAccessBlockInput>,
|
||||
) -> S3Result<S3Response<DeletePublicAccessBlockOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeletePublicAccessBlock, |req| {
|
||||
usecase.execute_delete_public_access_block(req)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(name = "get_bucket_cors", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_cors_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketCorsInput>,
|
||||
) -> S3Result<S3Response<GetBucketCorsOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketCors);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let cors_configuration = usecase.execute_get_bucket_cors(req).await?;
|
||||
let cors_configuration: CORSConfiguration = persisted_to_s3s(&cors_configuration)?;
|
||||
Ok(S3Response::new(GetBucketCorsOutput {
|
||||
cors_rules: Some(cors_configuration.cors_rules),
|
||||
}))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_encryption_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketEncryptionInput>,
|
||||
) -> S3Result<S3Response<GetBucketEncryptionOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketEncryption);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let server_side_encryption_configuration = usecase
|
||||
.execute_get_bucket_encryption(req)
|
||||
.await?
|
||||
.as_ref()
|
||||
.map(persisted_to_s3s::<_, ServerSideEncryptionConfiguration>)
|
||||
.transpose()?;
|
||||
Ok(S3Response::new(GetBucketEncryptionOutput {
|
||||
server_side_encryption_configuration,
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "get_bucket_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketTaggingInput>,
|
||||
) -> S3Result<S3Response<GetBucketTaggingOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketTagging);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let tagging = usecase.execute_get_bucket_tagging(req).await?;
|
||||
Ok(S3Response::new(GetBucketTaggingOutput {
|
||||
tag_set: tagging_to_s3s(&tagging).tag_set,
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "get_public_access_block", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_public_access_block_edge(
|
||||
&self,
|
||||
req: S3Request<GetPublicAccessBlockInput>,
|
||||
) -> S3Result<S3Response<GetPublicAccessBlockOutput>> {
|
||||
record_s3_op(S3Operation::GetPublicAccessBlock);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let config = usecase.execute_get_public_access_block(req).await?;
|
||||
Ok(S3Response::new(GetPublicAccessBlockOutput {
|
||||
public_access_block_configuration: Some(persisted_to_s3s(&config)?),
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "put_bucket_cors", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_cors_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketCorsInput>,
|
||||
) -> S3Result<S3Response<PutBucketCorsOutput>> {
|
||||
let prepared = persisted_from_s3s(&req.input.cors_configuration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketCors, |req| usecase.execute_put_bucket_cors(req, prepared)).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_encryption_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketEncryptionInput>,
|
||||
) -> S3Result<S3Response<PutBucketEncryptionOutput>> {
|
||||
let prepared = persisted_from_s3s(&req.input.server_side_encryption_configuration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketEncryption, |req| {
|
||||
usecase.execute_put_bucket_encryption(req, prepared)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(name = "put_public_access_block", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_public_access_block_edge(
|
||||
&self,
|
||||
req: S3Request<PutPublicAccessBlockInput>,
|
||||
) -> S3Result<S3Response<PutPublicAccessBlockOutput>> {
|
||||
let prepared = persisted_from_s3s(&req.input.public_access_block_configuration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutPublicAccessBlock, |req| {
|
||||
usecase.execute_put_public_access_block(req, prepared)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(name = "put_bucket_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketTaggingInput>,
|
||||
) -> S3Result<S3Response<PutBucketTaggingOutput>> {
|
||||
let tagging = tagging_from_s3s(&req.input.tagging);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::PutBucketTagging, |req| usecase.execute_put_bucket_tagging(req, tagging)).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of bucket create, delete, head and location, and the website, accelerate,
|
||||
//! logging and request-payment configurations: batch A4 of the gateway migration
|
||||
//! (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::access::bucket_config_mutation_incarnation;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use crate::storage::storage_api::ecfs_consumer::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage::storage_api::ecstore_bucket::utils::serialize as serialize_bucket_config;
|
||||
use crate::storage::validate_website_configuration;
|
||||
use crate::storage::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_WEBSITE_CONFIG, StorageError,
|
||||
delete_bucket_metadata_config_if_incarnation, get_bucket_accelerate_config, get_bucket_logging_config,
|
||||
get_bucket_request_payment_config, get_bucket_website_config, update_bucket_metadata_config_if_incarnation,
|
||||
};
|
||||
use crate::storage::{persisted_from_s3s, persisted_to_s3s};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
/// Handler bodies that [`S3`] runs through `audit_only_operation`.
|
||||
impl FS {
|
||||
async fn put_bucket_logging_inner(
|
||||
&self,
|
||||
req: S3Request<PutBucketLoggingInput>,
|
||||
) -> S3Result<S3Response<PutBucketLoggingOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
record_s3_op(S3Operation::PutBucketLogging);
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
let bucket_logging_status = persisted_from_s3s::<_, rustfs_gateway_types::persistence::PersistedBucketLoggingStatus>(
|
||||
&req.input.bucket_logging_status,
|
||||
)?;
|
||||
let logging_config = serialize_bucket_config(&bucket_logging_status)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?;
|
||||
update_bucket_metadata_config_if_incarnation(
|
||||
&req.input.bucket,
|
||||
BUCKET_LOGGING_CONFIG,
|
||||
logging_config,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(PutBucketLoggingOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
impl FS {
|
||||
#[instrument(
|
||||
name = "create_bucket",
|
||||
level = "debug",
|
||||
skip(self, req),
|
||||
fields(start_time=?time::OffsetDateTime::now_utc())
|
||||
)]
|
||||
pub(in crate::storage::ecfs) async fn create_bucket_edge(
|
||||
&self,
|
||||
req: S3Request<CreateBucketInput>,
|
||||
) -> S3Result<S3Response<CreateBucketOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_create_bucket(req).await
|
||||
}
|
||||
|
||||
/// Delete a bucket
|
||||
#[instrument(name = "delete_bucket", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_delete_bucket(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_website_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketWebsiteInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketWebsiteOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
delete_bucket_metadata_config_if_incarnation(&req.input.bucket, BUCKET_WEBSITE_CONFIG, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(DeleteBucketWebsiteOutput::default()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_accelerate_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketAccelerateConfigurationInput>,
|
||||
) -> S3Result<S3Response<GetBucketAccelerateConfigurationOutput>> {
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
match get_bucket_accelerate_config(&req.input.bucket).await {
|
||||
Ok((accelerate, _)) => Ok(S3Response::new(GetBucketAccelerateConfigurationOutput {
|
||||
status: accelerate.status.map(BucketAccelerateStatus::from),
|
||||
..Default::default()
|
||||
})),
|
||||
Err(StorageError::ConfigNotFound) => Ok(S3Response::new(GetBucketAccelerateConfigurationOutput::default())),
|
||||
Err(err) => Err(crate::error::ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get bucket location
|
||||
#[instrument(name = "get_bucket_location", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_location_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketLocationInput>,
|
||||
) -> S3Result<S3Response<GetBucketLocationOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketLocation);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_get_bucket_location(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_request_payment_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketRequestPaymentInput>,
|
||||
) -> S3Result<S3Response<GetBucketRequestPaymentOutput>> {
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
match get_bucket_request_payment_config(&req.input.bucket).await {
|
||||
Ok((payment, _)) => Ok(S3Response::new(GetBucketRequestPaymentOutput {
|
||||
payer: Some(Payer::from(payment.payer)),
|
||||
})),
|
||||
Err(StorageError::ConfigNotFound) => Ok(S3Response::new(GetBucketRequestPaymentOutput {
|
||||
payer: Some(Payer::from_static(Payer::BUCKET_OWNER)),
|
||||
})),
|
||||
Err(err) => Err(crate::error::ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_website_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketWebsiteInput>,
|
||||
) -> S3Result<S3Response<GetBucketWebsiteOutput>> {
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
match get_bucket_website_config(&req.input.bucket).await {
|
||||
Ok((website, _)) => {
|
||||
let website: WebsiteConfiguration = persisted_to_s3s(&website)?;
|
||||
Ok(S3Response::new(GetBucketWebsiteOutput {
|
||||
error_document: website.error_document,
|
||||
index_document: website.index_document,
|
||||
redirect_all_requests_to: website.redirect_all_requests_to,
|
||||
routing_rules: website.routing_rules,
|
||||
}))
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => Err(s3_error!(NoSuchWebsiteConfiguration)),
|
||||
Err(err) => Err(crate::error::ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(name = "head_bucket", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn head_bucket_edge(
|
||||
&self,
|
||||
req: S3Request<HeadBucketInput>,
|
||||
) -> S3Result<S3Response<HeadBucketOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_head_bucket(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_accelerate_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketAccelerateConfigurationInput>,
|
||||
) -> S3Result<S3Response<PutBucketAccelerateConfigurationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
let accelerate_configuration = persisted_from_s3s::<
|
||||
_,
|
||||
rustfs_gateway_types::persistence::PersistedAccelerateConfiguration,
|
||||
>(&req.input.accelerate_configuration)?;
|
||||
let accelerate_config = serialize_bucket_config(&accelerate_configuration)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?;
|
||||
update_bucket_metadata_config_if_incarnation(
|
||||
&req.input.bucket,
|
||||
BUCKET_ACCELERATE_CONFIG,
|
||||
accelerate_config,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(PutBucketAccelerateConfigurationOutput::default()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_logging_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketLoggingInput>,
|
||||
) -> S3Result<S3Response<GetBucketLoggingOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketLogging);
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
match get_bucket_logging_config(&req.input.bucket).await {
|
||||
Ok((logging, _)) => {
|
||||
let logging: BucketLoggingStatus = persisted_to_s3s(&logging)?;
|
||||
Ok(S3Response::new(GetBucketLoggingOutput {
|
||||
logging_enabled: logging.logging_enabled,
|
||||
}))
|
||||
}
|
||||
Err(StorageError::ConfigNotFound) => Ok(S3Response::new(GetBucketLoggingOutput::default())),
|
||||
Err(err) => Err(crate::error::ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_logging_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketLoggingInput>,
|
||||
) -> S3Result<S3Response<PutBucketLoggingOutput>> {
|
||||
audit_only_operation(req, S3Operation::PutBucketLogging, |req| self.put_bucket_logging_inner(req)).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_request_payment_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketRequestPaymentInput>,
|
||||
) -> S3Result<S3Response<PutBucketRequestPaymentOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
let request_payment_configuration = persisted_from_s3s::<
|
||||
_,
|
||||
rustfs_gateway_types::persistence::PersistedRequestPaymentConfiguration,
|
||||
>(&req.input.request_payment_configuration)?;
|
||||
let payment_config = serialize_bucket_config(&request_payment_configuration)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?;
|
||||
update_bucket_metadata_config_if_incarnation(
|
||||
&req.input.bucket,
|
||||
BUCKET_REQUEST_PAYMENT_CONFIG,
|
||||
payment_config,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(PutBucketRequestPaymentOutput::default()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_website_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketWebsiteInput>,
|
||||
) -> S3Result<S3Response<PutBucketWebsiteOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
store
|
||||
.get_bucket_info(&req.input.bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
let website_configuration = persisted_from_s3s::<_, rustfs_gateway_types::persistence::PersistedWebsiteConfiguration>(
|
||||
&req.input.website_configuration,
|
||||
)?;
|
||||
validate_website_configuration(&website_configuration)?;
|
||||
|
||||
let website_config = serialize_bucket_config(&website_configuration)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::MalformedXML, format!("{err}")))?;
|
||||
update_bucket_metadata_config_if_incarnation(
|
||||
&req.input.bucket,
|
||||
BUCKET_WEBSITE_CONFIG,
|
||||
website_config,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(crate::error::ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(PutBucketWebsiteOutput::default()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of the bucket lifecycle, replication and Object Lock configurations: batch A3 of
|
||||
//! the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::admin::handlers::site_replication::site_replication_bucket_meta_hook;
|
||||
use crate::app::storage_api::object_usecase::bucket::replication::OperatorRuleContract;
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::StorageVersioningConfigExt as _;
|
||||
use crate::storage::access::bucket_config_mutation_incarnation;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use crate::storage::s3_compat::lifecycle::{lifecycle_configuration_from_s3s, lifecycle_configuration_to_s3s};
|
||||
use crate::storage::s3_compat::replication::{replication_configuration_from_s3s, replication_configuration_to_s3s};
|
||||
use crate::storage::storage_api::ecfs_consumer::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::storage::storage_api::ecstore_bucket::utils::serialize as serialize_bucket_config;
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
use crate::storage::{
|
||||
BUCKET_VERSIONING_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, StorageError, get_bucket_object_lock_config,
|
||||
update_bucket_metadata_config_if_incarnation,
|
||||
};
|
||||
use crate::storage::{persisted_from_s3s, persisted_to_s3s};
|
||||
use http::StatusCode;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::{instrument, warn};
|
||||
|
||||
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) static SITE_REPLICATION_GATE_TEST_OVERRIDE: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) const SITE_REPLICATION_GATE_FORCE_DISABLED: u8 = 1;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) const SITE_REPLICATION_GATE_FORCE_ENABLED: u8 = 2;
|
||||
|
||||
async fn site_replication_gate_enabled() -> S3Result<bool> {
|
||||
#[cfg(test)]
|
||||
match SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED => return Ok(false),
|
||||
SITE_REPLICATION_GATE_FORCE_ENABLED => return Ok(true),
|
||||
_ => {}
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_enabled()
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Remote site-replication peer deployment ids and the operator-rule contract
|
||||
/// the peers support, handed to the bucket usecase so an S3 replication-config
|
||||
/// edit keeps exactly the reconciler-owned rules and merges the way every
|
||||
/// peer will (issue #1948). Read here, in the interface layer, because the
|
||||
/// usecase must not import the admin handlers (layer guard); a state-read
|
||||
/// failure propagates so the edit fails closed.
|
||||
async fn site_replication_edit_context() -> S3Result<(std::collections::HashSet<String>, OperatorRuleContract)> {
|
||||
// While the gate override is in effect the test exercises the deny/allow
|
||||
// branch, not the peer set; there is no persisted state to read.
|
||||
#[cfg(test)]
|
||||
if SITE_REPLICATION_GATE_TEST_OVERRIDE.load(std::sync::atomic::Ordering::SeqCst) != 0 {
|
||||
return Ok((std::collections::HashSet::new(), OperatorRuleContract::Derived));
|
||||
}
|
||||
crate::admin::handlers::site_replication::site_replication_edit_context()
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// MinIO `ErrReplicationDenyEditError`.
|
||||
fn replication_deny_edit_error() -> S3Error {
|
||||
let mut err = S3Error::with_message(
|
||||
S3ErrorCode::Custom("XMinioReplicationDenyEdit".into()),
|
||||
"Sub-User is not allowed to edit Replication configuration",
|
||||
);
|
||||
err.set_status_code(StatusCode::BAD_REQUEST);
|
||||
err
|
||||
}
|
||||
|
||||
/// Site-replication gate for S3 replication-config edits (issue #1948).
|
||||
///
|
||||
/// On a site-replication deployment the bucket's replication config carries
|
||||
/// the operator-managed `site-repl-*` rules that keep every peer in sync, and
|
||||
/// a successful edit is broadcast to all peers — so a user holding only
|
||||
/// bucket-scoped `s3:PutReplicationConfiguration` could rewrite or erase
|
||||
/// replication net-wide. MinIO parity (`ErrReplicationDenyEditError`): only
|
||||
/// owner credentials (root or root-parented) may edit. Runs after the policy
|
||||
/// authorization in the access layer and only on the external S3 path — the
|
||||
/// reconciler and peer bucket-meta ingestion never route through these
|
||||
/// handlers.
|
||||
async fn deny_replication_config_edit_for_non_owner<T>(req: &S3Request<T>) -> S3Result<()> {
|
||||
if crate::storage::access::req_info_ref(req)?.is_owner {
|
||||
return Ok(());
|
||||
}
|
||||
if site_replication_gate_enabled().await? {
|
||||
return Err(replication_deny_edit_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handler bodies that [`S3`] runs through `audit_only_operation`.
|
||||
impl FS {
|
||||
async fn delete_bucket_replication_inner(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let (site_peers, contract) = site_replication_edit_context().await?;
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_delete_bucket_replication(req, site_peers, contract).await
|
||||
}
|
||||
|
||||
async fn put_bucket_replication_inner(
|
||||
&self,
|
||||
mut req: S3Request<PutBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
deny_replication_config_edit_for_non_owner(&req).await?;
|
||||
let (site_peers, contract) = site_replication_edit_context().await?;
|
||||
let replication_configuration =
|
||||
replication_configuration_from_s3s(std::mem::take(&mut req.input.replication_configuration));
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase
|
||||
.execute_put_bucket_replication(req, replication_configuration, site_peers, contract)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_object_lock_configuration_inner(
|
||||
&self,
|
||||
req: S3Request<PutObjectLockConfigurationInput>,
|
||||
) -> S3Result<S3Response<PutObjectLockConfigurationOutput>> {
|
||||
let expected_incarnation_id = bucket_config_mutation_incarnation(&req, &req.input.bucket)?;
|
||||
let PutObjectLockConfigurationInput {
|
||||
bucket,
|
||||
object_lock_configuration,
|
||||
..
|
||||
} = req.input;
|
||||
|
||||
let Some(input_cfg) = object_lock_configuration else { return Err(s3_error!(InvalidArgument)) };
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
validate_object_lock_configuration_input(&input_cfg)?;
|
||||
|
||||
match get_bucket_object_lock_config(&bucket).await {
|
||||
Ok(_) => {}
|
||||
Err(err) => {
|
||||
if err == StorageError::ConfigNotFound {
|
||||
if !BucketVersioningSys::enabled(&bucket).await {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidBucketState,
|
||||
"Object Lock configuration cannot be enabled on existing buckets".to_string(),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
warn!("get_object_lock_config err {:?}", err);
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
"Failed to get bucket ObjectLockConfiguration".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let persisted_cfg =
|
||||
persisted_from_s3s::<_, rustfs_gateway_types::persistence::PersistedObjectLockConfiguration>(&input_cfg)?;
|
||||
let data = serialize_bucket_config(&persisted_cfg)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")))?;
|
||||
let object_lock_config =
|
||||
String::from_utf8(data.clone()).map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")))?;
|
||||
|
||||
let updated_at = update_bucket_metadata_config_if_incarnation(&bucket, OBJECT_LOCK_CONFIG, data, expected_incarnation_id)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
// When Object Lock is enabled, automatically enable versioning if not already enabled.
|
||||
// This matches S3-compatible behavior.
|
||||
let versioning_config = BucketVersioningSys::get(&bucket).await.map_err(ApiError::from)?;
|
||||
if !versioning_config.enabled() {
|
||||
let enable_versioning_config = rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
let versioning_data = serialize_bucket_config(&enable_versioning_config)
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")))?;
|
||||
update_bucket_metadata_config_if_incarnation(
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
versioning_data,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
}
|
||||
|
||||
if let Some(notification_sys) =
|
||||
runtime_sources::current_notification_system_for_context(self.server_ctx.app_context().as_deref())
|
||||
{
|
||||
propagate_object_lock_peer_reload(notification_sys.load_bucket_metadata(&bucket).await)?;
|
||||
}
|
||||
|
||||
if let Err(err) = site_replication_bucket_meta_hook(SRBucketMeta {
|
||||
bucket: bucket.clone(),
|
||||
r#type: "object-lock-config".to_string(),
|
||||
object_lock_config: Some(object_lock_config),
|
||||
updated_at: Some(updated_at),
|
||||
api_version: Some(SITE_REPL_API_VERSION.to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT_LOCK,
|
||||
event = "put_object_lock_configuration",
|
||||
bucket = %bucket,
|
||||
result = "site_replication_hook_failed",
|
||||
error = ?err,
|
||||
"storage object lock state"
|
||||
);
|
||||
}
|
||||
|
||||
rustfs_scanner::record_dirty_usage_bucket_from_producer(
|
||||
&bucket,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::BucketMetadata,
|
||||
);
|
||||
Ok(S3Response::new(PutObjectLockConfigurationOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "delete_bucket_lifecycle", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_lifecycle_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketLifecycleInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketLifecycleOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::DeleteBucketLifecycle, |req| {
|
||||
usecase.execute_delete_bucket_lifecycle(req)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn delete_bucket_replication_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<DeleteBucketReplicationOutput>> {
|
||||
audit_only_operation(req, S3Operation::DeleteBucketReplication, |req| self.delete_bucket_replication_inner(req)).await
|
||||
}
|
||||
|
||||
#[instrument(name = "get_bucket_lifecycle_configuration", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_lifecycle_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketLifecycleConfigurationInput>,
|
||||
) -> S3Result<S3Response<GetBucketLifecycleConfigurationOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketLifecycleConfiguration);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let config = usecase.execute_get_bucket_lifecycle_configuration(req).await?;
|
||||
let config = lifecycle_configuration_to_s3s(config)
|
||||
.map_err(|error| S3Error::with_message(S3ErrorCode::InternalError, error.to_string()))?;
|
||||
Ok(S3Response::new(GetBucketLifecycleConfigurationOutput {
|
||||
rules: Some(config.rules),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_bucket_replication_edge(
|
||||
&self,
|
||||
req: S3Request<GetBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<GetBucketReplicationOutput>> {
|
||||
record_s3_op(S3Operation::GetBucketReplication);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let replication_configuration = usecase.execute_get_bucket_replication(req).await?;
|
||||
Ok(S3Response::new(GetBucketReplicationOutput {
|
||||
replication_configuration: Some(replication_configuration_to_s3s(replication_configuration)),
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "get_object_lock_configuration", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_object_lock_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectLockConfigurationInput>,
|
||||
) -> S3Result<S3Response<GetObjectLockConfigurationOutput>> {
|
||||
record_s3_op(S3Operation::GetObjectLockConfiguration);
|
||||
let GetObjectLockConfigurationInput { bucket, .. } = req.input;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let object_lock_configuration = match get_bucket_object_lock_config(&bucket).await {
|
||||
Ok((cfg, _created)) => Some(persisted_to_s3s::<_, ObjectLockConfiguration>(&cfg)?),
|
||||
Err(err) => {
|
||||
if err == StorageError::ConfigNotFound {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::ObjectLockConfigurationNotFoundError,
|
||||
"Object Lock configuration does not exist for this bucket".to_string(),
|
||||
));
|
||||
}
|
||||
warn!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT_LOCK,
|
||||
event = "object_lock_config_load_failed",
|
||||
bucket = %bucket,
|
||||
error = ?err,
|
||||
"Failed to load bucket object lock configuration"
|
||||
);
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
"Failed to load Object Lock configuration".to_string(),
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
Ok(S3Response::new(GetObjectLockConfigurationOutput {
|
||||
object_lock_configuration,
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "put_bucket_lifecycle_configuration", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_lifecycle_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketLifecycleConfigurationInput>,
|
||||
) -> S3Result<S3Response<PutBucketLifecycleConfigurationOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
let prepared = req
|
||||
.input
|
||||
.lifecycle_configuration
|
||||
.clone()
|
||||
.map(lifecycle_configuration_from_s3s)
|
||||
.transpose();
|
||||
audit_only_operation(req, S3Operation::PutBucketLifecycleConfiguration, |req| {
|
||||
usecase.execute_put_bucket_lifecycle_configuration(req, prepared)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_replication_edge(
|
||||
&self,
|
||||
req: S3Request<PutBucketReplicationInput>,
|
||||
) -> S3Result<S3Response<PutBucketReplicationOutput>> {
|
||||
audit_only_operation(req, S3Operation::PutBucketReplication, |req| self.put_bucket_replication_inner(req)).await
|
||||
}
|
||||
|
||||
#[instrument(name = "put_object_lock_configuration", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_object_lock_configuration_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectLockConfigurationInput>,
|
||||
) -> S3Result<S3Response<PutObjectLockConfigurationOutput>> {
|
||||
audit_only_operation(req, S3Operation::PutObjectLockConfiguration, |req| {
|
||||
self.put_object_lock_configuration_inner(req)
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of the object and multipart-upload listings, CreateMultipartUpload and
|
||||
//! AbortMultipartUpload: batch B2 of the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "abort_multipart_upload", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn abort_multipart_upload_edge(
|
||||
&self,
|
||||
req: S3Request<AbortMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<AbortMultipartUploadOutput>> {
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::AbortMultipartUpload, |req| usecase.execute_abort_multipart_upload(req)).await
|
||||
}
|
||||
|
||||
#[instrument(name = "create_multipart_upload", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn create_multipart_upload_edge(
|
||||
&self,
|
||||
req: S3Request<CreateMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CreateMultipartUploadOutput>> {
|
||||
crate::hp_guard!("S3::create_multipart_upload");
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
usecase.execute_create_multipart_upload(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn list_multipart_uploads_edge(
|
||||
&self,
|
||||
req: S3Request<ListMultipartUploadsInput>,
|
||||
) -> S3Result<S3Response<ListMultipartUploadsOutput>> {
|
||||
record_s3_op(S3Operation::ListMultipartUploads);
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
usecase.execute_list_multipart_uploads(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn list_object_versions_edge(
|
||||
&self,
|
||||
req: S3Request<ListObjectVersionsInput>,
|
||||
) -> S3Result<S3Response<ListObjectVersionsOutput>> {
|
||||
record_s3_op(S3Operation::ListObjectVersions);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_list_object_versions(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "list_objects", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn list_objects_edge(
|
||||
&self,
|
||||
req: S3Request<ListObjectsInput>,
|
||||
) -> S3Result<S3Response<ListObjectsOutput>> {
|
||||
record_s3_op(S3Operation::ListObjects);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_list_objects(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "list_objects_v2", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn list_objects_v2_edge(
|
||||
&self,
|
||||
req: S3Request<ListObjectsV2Input>,
|
||||
) -> S3Result<S3Response<ListObjectsV2Output>> {
|
||||
crate::hp_guard!("S3::list_objects_v2");
|
||||
record_s3_op(S3Operation::ListObjectsV2);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_list_objects_v2(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "list_parts", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn list_parts_edge(
|
||||
&self,
|
||||
req: S3Request<ListPartsInput>,
|
||||
) -> S3Result<S3Response<ListPartsOutput>> {
|
||||
record_s3_op(S3Operation::ListParts);
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
usecase.execute_list_parts(req).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The bodies of `impl S3 for FS`, one file per batch of the s3s-to-gateway migration
|
||||
//! (rustfs/backlog#2765), so that parallel batches edit disjoint files. `ecfs.rs` keeps the
|
||||
//! trait impl, which delegates every method here, and the helpers several batches share.
|
||||
//! `scripts/codemods/split_ecfs_legacy_edge.py` produced this split from the unsplit file.
|
||||
|
||||
mod bucket_access;
|
||||
mod bucket_configs;
|
||||
mod bucket_core;
|
||||
mod bucket_replication_lifecycle;
|
||||
mod listing_multipart;
|
||||
mod object_meta;
|
||||
mod object_misc;
|
||||
mod object_read;
|
||||
mod object_write;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,976 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of object ACL, tagging, legal hold, retention and torrent: batch B1 of the
|
||||
//! gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::app::storage_api::object_usecase::bucket::replication::{
|
||||
ReplicateDecision, get_read_proxy_targets, must_replicate_metadata, schedule_metadata_replication,
|
||||
};
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::access::{apply_bucket_generation_guard, has_bypass_governance_header, replication_proxy_allowed};
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::OperationHelper;
|
||||
use crate::storage::options::get_opts;
|
||||
use crate::storage::s3_api::{self, acl};
|
||||
use crate::storage::storage_api::ecfs_consumer::StorageObjectOptions as ObjectOptions;
|
||||
use crate::storage::storage_api::ecfs_consumer::contract::{
|
||||
bucket::{BucketOperations, BucketOptions},
|
||||
object::ObjectLockRetentionOptions,
|
||||
};
|
||||
use crate::storage::storage_api::ecfs_consumer::object_lock::{
|
||||
parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled,
|
||||
};
|
||||
use crate::storage::{
|
||||
BucketVersioningSys, check_retention_for_modification, decode_tags, encode_tags, is_err_object_not_found,
|
||||
is_err_version_not_found, record_replication_proxy,
|
||||
};
|
||||
use crate::table_catalog;
|
||||
use metrics::{counter, histogram};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER;
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_REPLICATION_GENERATION, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, insert_str,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tracing::{debug, error, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
const LOG_SUBSYSTEM_TAGGING: &str = "tagging";
|
||||
|
||||
async fn reject_veeam_sosapi_mutation(fs: &FS, bucket: &str, key: &str) -> S3Result<()> {
|
||||
s3_api::reject_veeam_sosapi_mutation(|| fs.server_ctx.object_store(), bucket, key).await
|
||||
}
|
||||
|
||||
async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> {
|
||||
table_catalog::validate_bucket_object_mutation(bucket, key)
|
||||
.await
|
||||
.map_err(|_| s3_error!(InvalidRequest, "{}", table_catalog::RESERVED_CATALOG_OBJECT_MESSAGE))
|
||||
}
|
||||
|
||||
impl FS {
|
||||
/// Not-found classifier for proxied SDK tagging calls: a raw 404 covers
|
||||
/// NoSuchKey and NoSuchVersion alike; the caller silently tries the next
|
||||
/// replication target.
|
||||
fn proxy_sdk_error_is_not_found<E>(err: &aws_sdk_s3::error::SdkError<E>) -> bool {
|
||||
err.raw_response().is_some_and(|resp| resp.status().as_u16() == 404)
|
||||
}
|
||||
|
||||
/// Selector options for a tagging proxy. Reuses `get_opts` so the
|
||||
/// anti-loop `source-proxy-request` header family and the bucket's
|
||||
/// version-suspension state gate proxying exactly like GET/HEAD.
|
||||
async fn tagging_proxy_opts(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
headers: &http::HeaderMap,
|
||||
) -> Option<ObjectOptions> {
|
||||
get_opts(bucket, object, version_id, None, headers).await.ok()
|
||||
}
|
||||
|
||||
/// Serve a GetObjectTagging for an object missing locally by proxying to
|
||||
/// the bucket's replication targets (MinIO `proxyGetTaggingToRepTarget`,
|
||||
/// backlog#1675 P1-5). None means no target had the object.
|
||||
async fn proxy_get_object_tagging(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
headers: &http::HeaderMap,
|
||||
) -> Option<GetObjectTaggingOutput> {
|
||||
let opts = Self::tagging_proxy_opts(bucket, object, version_id, headers).await?;
|
||||
let targets = get_read_proxy_targets(bucket, object, &opts).await;
|
||||
if targets.is_empty() {
|
||||
return None;
|
||||
}
|
||||
for target in targets {
|
||||
match target
|
||||
.get_object_tagging(&target.bucket, object, opts.version_id.clone())
|
||||
.await
|
||||
{
|
||||
Ok(remote) => {
|
||||
// MinIO-aligned accounting: one total per proxy attempt,
|
||||
// one failed when no target served it.
|
||||
record_replication_proxy(bucket, "GetObjectTagging", false).await;
|
||||
return Some(GetObjectTaggingOutput {
|
||||
tag_set: remote
|
||||
.tag_set
|
||||
.into_iter()
|
||||
.map(|tag| Tag {
|
||||
key: Some(tag.key),
|
||||
value: Some(tag.value),
|
||||
})
|
||||
.collect(),
|
||||
version_id: remote.version_id,
|
||||
});
|
||||
}
|
||||
Err(err) if Self::proxy_sdk_error_is_not_found(&err) => {
|
||||
debug!(bucket, object, arn = %target.arn, "tagging proxy: target does not have the object");
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(bucket, object, arn = %target.arn, error = %err, "tagging proxy: GetObjectTagging against replication target failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
record_replication_proxy(bucket, "GetObjectTagging", true).await;
|
||||
None
|
||||
}
|
||||
|
||||
/// Apply a PutObjectTagging for an object missing locally on a
|
||||
/// replication target (MinIO `proxyTaggingToRepTarget`).
|
||||
async fn proxy_put_object_tagging(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
headers: &http::HeaderMap,
|
||||
tag_set: &TagSet,
|
||||
) -> Option<()> {
|
||||
let opts = Self::tagging_proxy_opts(bucket, object, version_id, headers).await?;
|
||||
let mut tagging = aws_sdk_s3::types::Tagging::builder();
|
||||
for tag in tag_set {
|
||||
let sdk_tag = aws_sdk_s3::types::Tag::builder()
|
||||
.key(tag.key.clone().unwrap_or_default())
|
||||
.value(tag.value.clone().unwrap_or_default())
|
||||
.build()
|
||||
.ok()?;
|
||||
tagging = tagging.tag_set(sdk_tag);
|
||||
}
|
||||
let tagging = tagging.build().ok()?;
|
||||
let targets = get_read_proxy_targets(bucket, object, &opts).await;
|
||||
if targets.is_empty() {
|
||||
return None;
|
||||
}
|
||||
for target in targets {
|
||||
match target
|
||||
.put_object_tagging(&target.bucket, object, opts.version_id.clone(), tagging.clone())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
// MinIO-aligned accounting: one total per proxy attempt,
|
||||
// one failed when no target served it.
|
||||
record_replication_proxy(bucket, "PutObjectTagging", false).await;
|
||||
return Some(());
|
||||
}
|
||||
Err(err) if Self::proxy_sdk_error_is_not_found(&err) => {
|
||||
debug!(bucket, object, arn = %target.arn, "tagging proxy: target does not have the object");
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(bucket, object, arn = %target.arn, error = %err, "tagging proxy: PutObjectTagging against replication target failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
record_replication_proxy(bucket, "PutObjectTagging", true).await;
|
||||
None
|
||||
}
|
||||
|
||||
/// Apply a DeleteObjectTagging for an object missing locally on a
|
||||
/// replication target (MinIO `proxyTaggingToRepTarget`).
|
||||
async fn proxy_delete_object_tagging(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
headers: &http::HeaderMap,
|
||||
) -> Option<()> {
|
||||
let opts = Self::tagging_proxy_opts(bucket, object, version_id, headers).await?;
|
||||
let targets = get_read_proxy_targets(bucket, object, &opts).await;
|
||||
if targets.is_empty() {
|
||||
return None;
|
||||
}
|
||||
for target in targets {
|
||||
match target
|
||||
.delete_object_tagging(&target.bucket, object, opts.version_id.clone())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
// MinIO-aligned accounting: one total per proxy attempt,
|
||||
// one failed when no target served it.
|
||||
record_replication_proxy(bucket, "DeleteObjectTagging", false).await;
|
||||
return Some(());
|
||||
}
|
||||
Err(err) if Self::proxy_sdk_error_is_not_found(&err) => {
|
||||
debug!(bucket, object, arn = %target.arn, "tagging proxy: target does not have the object");
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(bucket, object, arn = %target.arn, error = %err, "tagging proxy: DeleteObjectTagging against replication target failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
record_replication_proxy(bucket, "DeleteObjectTagging", true).await;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "delete_object_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_object_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectTaggingInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectTaggingOutput>> {
|
||||
let start_time = std::time::Instant::now();
|
||||
let mut helper = OperationHelper::new(&req, EventName::ObjectTaggingDelete, S3Operation::DeleteObjectTagging);
|
||||
let DeleteObjectTaggingInput {
|
||||
bucket,
|
||||
key: object,
|
||||
version_id,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &object).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &object).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_store_uninitialized",
|
||||
operation = "delete",
|
||||
"Object tagging operation failed because storage is not initialized"
|
||||
);
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let mut opts = get_opts(&bucket, &object, version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let existing_object_info = match store.get_object_info(&bucket, &object, &opts).await {
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
// Replication lag window: apply the tagging delete on a
|
||||
// replication target that already has the object
|
||||
// (backlog#1675 P1-5). No local object exists, so no bucket
|
||||
// notification event is emitted for the proxied write.
|
||||
if (is_err_object_not_found(&e) || is_err_version_not_found(&e))
|
||||
&& replication_proxy_allowed(&req.extensions)
|
||||
&& Self::proxy_delete_object_tagging(&bucket, &object, version_id.clone(), &req.headers)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
counter!("rustfs_delete_object_tagging_success").increment(1);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "delete")
|
||||
.record(duration.as_secs_f64());
|
||||
return Ok(S3Response::new(DeleteObjectTaggingOutput { version_id }));
|
||||
}
|
||||
return Err(ApiError::from(e).into());
|
||||
}
|
||||
};
|
||||
let dsc = must_replicate_metadata(
|
||||
&bucket,
|
||||
&object,
|
||||
&existing_object_info.user_defined,
|
||||
String::new(),
|
||||
existing_object_info.replication_status.clone(),
|
||||
opts.clone(),
|
||||
)
|
||||
.await;
|
||||
if dsc.replicate_any() {
|
||||
let mut eval_metadata = HashMap::new();
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
|
||||
insert_str(
|
||||
&mut eval_metadata,
|
||||
SUFFIX_TAGGING_TIMESTAMP,
|
||||
OffsetDateTime::now_utc().format(&Rfc3339).unwrap_or_default(),
|
||||
);
|
||||
opts.eval_metadata = Some(eval_metadata);
|
||||
}
|
||||
|
||||
let delete_tags_result = store.delete_object_tags(&bucket, &object, &opts).await;
|
||||
let object_info = delete_tags_result.map_err(|e| {
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_delete_failed",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
error = %e,
|
||||
"Failed to delete object tags"
|
||||
);
|
||||
ApiError::from(e)
|
||||
})?;
|
||||
|
||||
let event_object_info = Some(object_info.clone());
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(object_info, store.clone(), dsc).await;
|
||||
}
|
||||
|
||||
counter!("rustfs_delete_object_tagging_success").increment(1);
|
||||
|
||||
let event_version_id = version_id
|
||||
.as_deref()
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_string)
|
||||
.or_else(|| {
|
||||
event_object_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.version_id.map(|version_id| version_id.to_string()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if let Some(event_object_info) = event_object_info {
|
||||
helper = helper.object(event_object_info);
|
||||
}
|
||||
helper = helper.version_id(event_version_id);
|
||||
|
||||
let result = Ok(S3Response::new(DeleteObjectTaggingOutput { version_id }));
|
||||
let _ = helper.complete(&result);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&object,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata,
|
||||
);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "delete").record(duration.as_secs_f64());
|
||||
result
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_object_acl_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectAclInput>,
|
||||
) -> S3Result<S3Response<GetObjectAclOutput>> {
|
||||
record_s3_op(S3Operation::GetObjectAcl);
|
||||
let GetObjectAclInput {
|
||||
bucket, key, version_id, ..
|
||||
} = req.input;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
store.get_object_info(&bucket, &key, &opts).await.map_err(ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(acl::build_get_object_acl_output()))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_object_legal_hold_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectLegalHoldInput>,
|
||||
) -> S3Result<S3Response<GetObjectLegalHoldOutput>> {
|
||||
let mut helper =
|
||||
OperationHelper::new(&req, EventName::ObjectAccessedGetLegalHold, S3Operation::GetObjectLegalHold).suppress_event();
|
||||
let GetObjectLegalHoldInput {
|
||||
bucket, key, version_id, ..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let _ = store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
validate_bucket_object_lock_enabled(&bucket).await?;
|
||||
|
||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| {
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
event = "object_info_load_failed",
|
||||
bucket = %bucket,
|
||||
object = %key,
|
||||
error = %e,
|
||||
"Failed to load object info"
|
||||
);
|
||||
s3_error!(InternalError, "{}", e.to_string())
|
||||
})?;
|
||||
|
||||
let legal_hold = object_info
|
||||
.user_defined
|
||||
.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER)
|
||||
.map(|v| v.as_str().to_string());
|
||||
|
||||
let status = if let Some(v) = legal_hold {
|
||||
v
|
||||
} else {
|
||||
ObjectLockLegalHoldStatus::OFF.to_string()
|
||||
};
|
||||
|
||||
let output = GetObjectLegalHoldOutput {
|
||||
legal_hold: Some(ObjectLockLegalHold {
|
||||
status: Some(ObjectLockLegalHoldStatus::from(status)),
|
||||
}),
|
||||
};
|
||||
|
||||
let version_id = req.input.version_id.clone().unwrap_or_else(|| Uuid::new_v4().to_string());
|
||||
helper = helper.object(object_info).version_id(version_id);
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_object_retention_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectRetentionInput>,
|
||||
) -> S3Result<S3Response<GetObjectRetentionOutput>> {
|
||||
let mut helper =
|
||||
OperationHelper::new(&req, EventName::ObjectAccessedGetRetention, S3Operation::GetObjectRetention).suppress_event();
|
||||
let GetObjectRetentionInput {
|
||||
bucket, key, version_id, ..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
validate_bucket_object_lock_enabled(&bucket).await?;
|
||||
|
||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let object_info = store.get_object_info(&bucket, &key, &opts).await.map_err(|e| {
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
event = "object_info_load_failed",
|
||||
bucket = %bucket,
|
||||
object = %key,
|
||||
error = %e,
|
||||
"Failed to load object info"
|
||||
);
|
||||
s3_error!(InternalError, "{}", e.to_string())
|
||||
})?;
|
||||
|
||||
let output = GetObjectRetentionOutput {
|
||||
retention: Some(object_lock_retention_from_metadata(&object_info.user_defined)?),
|
||||
};
|
||||
let version_id = req.input.version_id.clone().unwrap_or_default();
|
||||
helper = helper.object(object_info).version_id(version_id);
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(name = "get_object_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn get_object_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectTaggingInput>,
|
||||
) -> S3Result<S3Response<GetObjectTaggingOutput>> {
|
||||
use crate::storage::storage_api::ecstore_bucket::versioning::VersioningApi as _;
|
||||
|
||||
record_s3_op(S3Operation::GetObjectTagging);
|
||||
let start_time = std::time::Instant::now();
|
||||
let bucket = req.input.bucket.as_str();
|
||||
let object = req.input.key.as_str();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, bucket, object).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_store_uninitialized",
|
||||
operation = "get",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
"Object tagging operation failed because storage is not initialized"
|
||||
);
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let version_id = parse_object_version_id(req.input.version_id.clone())?.map(Into::into);
|
||||
let versioning = BucketVersioningSys::get(bucket).await.map_err(ApiError::from)?;
|
||||
let opts = ObjectOptions {
|
||||
version_id,
|
||||
versioned: versioning.prefix_enabled(object),
|
||||
version_suspended: versioning.prefix_suspended(object),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Tags and response identity must come from the same locked metadata snapshot.
|
||||
let info = match store.get_object_info(bucket, object, &opts).await {
|
||||
Ok(info) if info.delete_marker => {
|
||||
return Err(S3Error::new(if opts.version_id.is_some() {
|
||||
S3ErrorCode::MethodNotAllowed
|
||||
} else {
|
||||
S3ErrorCode::NoSuchKey
|
||||
}));
|
||||
}
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
// Replication lag window: the object may exist on a
|
||||
// replication target even though it is missing locally —
|
||||
// proxy the tagging read there (backlog#1675 P1-5).
|
||||
if (is_err_object_not_found(&e) || is_err_version_not_found(&e))
|
||||
&& replication_proxy_allowed(&req.extensions)
|
||||
&& let Some(output) =
|
||||
Self::proxy_get_object_tagging(bucket, object, req.input.version_id.clone(), &req.headers).await
|
||||
{
|
||||
counter!("rustfs_get_object_tagging_success").increment(1);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "get")
|
||||
.record(duration.as_secs_f64());
|
||||
return Ok(S3Response::new(output));
|
||||
}
|
||||
if is_err_object_not_found(&e) {
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_not_found",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
error = %e,
|
||||
"Object tags not found"
|
||||
);
|
||||
return Err(S3Error::new(if opts.version_id.is_some() {
|
||||
S3ErrorCode::NoSuchVersion
|
||||
} else {
|
||||
S3ErrorCode::NoSuchKey
|
||||
}));
|
||||
}
|
||||
error!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_get_failed",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
error = %e,
|
||||
"Failed to load object tags"
|
||||
);
|
||||
return Err(ApiError::from(e).into());
|
||||
}
|
||||
};
|
||||
|
||||
let tag_set = decode_tags(info.user_tags.as_str());
|
||||
debug!(
|
||||
component = LOG_COMPONENT_STORAGE,
|
||||
subsystem = LOG_SUBSYSTEM_TAGGING,
|
||||
event = "object_tagging_decoded",
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
tag_count = tag_set.len(),
|
||||
"Decoded object tags"
|
||||
);
|
||||
|
||||
counter!("rustfs_get_object_tagging_success").increment(1);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "get").record(duration.as_secs_f64());
|
||||
Ok(S3Response::new(GetObjectTaggingOutput {
|
||||
tag_set,
|
||||
version_id: s3_api::s3_response_version_id(info.version_id),
|
||||
}))
|
||||
}
|
||||
|
||||
#[instrument(name = "get_object_torrent", level = "debug", skip(self, _req))]
|
||||
pub(in crate::storage::ecfs) async fn get_object_torrent_edge(
|
||||
&self,
|
||||
_req: S3Request<GetObjectTorrentInput>,
|
||||
) -> S3Result<S3Response<GetObjectTorrentOutput>> {
|
||||
// Torrent functionality is not implemented in RustFS
|
||||
// Per S3 API test expectations, return 404 NoSuchKey (not 501 Not Implemented)
|
||||
// This allows clients to gracefully handle the absence of torrent support
|
||||
record_s3_op(S3Operation::GetObjectTorrent);
|
||||
Err(S3Error::new(S3ErrorCode::NoSuchKey))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_object_acl_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectAclInput>,
|
||||
) -> S3Result<S3Response<PutObjectAclOutput>> {
|
||||
let mut helper = OperationHelper::new(&req, EventName::ObjectAclPut, S3Operation::PutObjectAcl);
|
||||
let bucket = &req.input.bucket;
|
||||
let key = &req.input.key;
|
||||
let version_id = req.input.version_id.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, bucket, key).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let opts: ObjectOptions = get_opts(bucket, key, version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let object_info = store.get_object_info(bucket, key, &opts).await.map_err(ApiError::from)?;
|
||||
|
||||
if req.input.access_control_policy.is_some() {
|
||||
return Err(s3_error!(
|
||||
NotImplemented,
|
||||
"ACL XML grants are not supported; use canned ACL headers or omit ACL"
|
||||
));
|
||||
}
|
||||
|
||||
let event_version_id = version_id
|
||||
.or_else(|| object_info.version_id.map(|version_id| version_id.to_string()))
|
||||
.unwrap_or_default();
|
||||
helper = helper.object(object_info).version_id(event_version_id);
|
||||
|
||||
let result = Ok(S3Response::new(PutObjectAclOutput::default()));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_object_legal_hold_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectLegalHoldInput>,
|
||||
) -> S3Result<S3Response<PutObjectLegalHoldOutput>> {
|
||||
let mut helper =
|
||||
OperationHelper::new(&req, EventName::ObjectCreatedPutLegalHold, S3Operation::PutObjectLegalHold).suppress_event();
|
||||
let PutObjectLegalHoldInput {
|
||||
bucket,
|
||||
key,
|
||||
legal_hold,
|
||||
version_id,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &key).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
if let Err(err) = validate_bucket_object_lock_enabled(&bucket).await {
|
||||
let _ = store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let mut popts = ObjectOptions {
|
||||
mod_time: opts.mod_time,
|
||||
version_id: opts.version_id.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut popts)?;
|
||||
|
||||
// PutObjectLegalHold only rewrites metadata, so replication is not scheduled by the
|
||||
// object PUT path. Schedule it explicitly, otherwise the legal hold never reaches the
|
||||
// replica and the peer copy remains deletable.
|
||||
//
|
||||
// Mirror the PUT path ordering (see object_usecase::put_object): compute the decision
|
||||
// once BEFORE the commit and persist the pending marker with it, so a restart between
|
||||
// the commit and the worker write-back leaves a Pending status on disk for the scanner
|
||||
// to re-drive. Scheduling without that marker would lose the task silently.
|
||||
let dsc = if crate::app::storage_api::bucket::metadata_sys::get_replication_config(&bucket)
|
||||
.await
|
||||
.is_ok()
|
||||
&& let Ok(info) = store.get_object_info(&bucket, &key, &opts).await
|
||||
{
|
||||
must_replicate_metadata(
|
||||
&bucket,
|
||||
&key,
|
||||
&info.user_defined,
|
||||
info.user_tags.as_ref().clone(),
|
||||
popts.delete_marker_replication_status(),
|
||||
popts.clone(),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
ReplicateDecision::new()
|
||||
};
|
||||
|
||||
let mut eval_metadata = parse_object_lock_legal_hold(legal_hold)?;
|
||||
if dsc.replicate_any() {
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
|
||||
}
|
||||
popts.eval_metadata = Some(eval_metadata);
|
||||
|
||||
let info = store.put_object_metadata(&bucket, &key, &popts).await.map_err(|e| {
|
||||
error!("put_object_metadata failed, {}", e.to_string());
|
||||
s3_error!(InternalError, "{}", e.to_string())
|
||||
})?;
|
||||
|
||||
// The current target transport carries the updated metadata in a full-object PUT.
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(info.clone(), store.clone(), dsc).await;
|
||||
}
|
||||
|
||||
let output = PutObjectLegalHoldOutput {
|
||||
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
};
|
||||
let version_id = req.input.version_id.clone().unwrap_or_default();
|
||||
helper = helper.object(info).version_id(version_id);
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata,
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_object_retention_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectRetentionInput>,
|
||||
) -> S3Result<S3Response<PutObjectRetentionOutput>> {
|
||||
let mut helper =
|
||||
OperationHelper::new(&req, EventName::ObjectCreatedPutRetention, S3Operation::PutObjectRetention).suppress_event();
|
||||
let PutObjectRetentionInput {
|
||||
bucket,
|
||||
key,
|
||||
retention,
|
||||
version_id,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &key).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
validate_bucket_object_lock_enabled(&bucket).await?;
|
||||
|
||||
let new_retain_until = retention
|
||||
.as_ref()
|
||||
.and_then(|r| r.retain_until_date.as_ref())
|
||||
.map(|d| OffsetDateTime::from(d.clone()));
|
||||
let new_mode = retention
|
||||
.as_ref()
|
||||
.and_then(|r| r.mode.as_ref())
|
||||
.map(|mode| mode.as_str().to_string());
|
||||
|
||||
let bypass_governance = has_bypass_governance_header(&req.headers);
|
||||
// Keep the early check for existing response behavior; put_object_metadata
|
||||
// repeats the same check after taking the metadata write lock.
|
||||
let check_opts: ObjectOptions = get_opts(&bucket, &key, version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let existing_obj_info = store.get_object_info(&bucket, &key, &check_opts).await.ok();
|
||||
|
||||
if let Some(existing_obj_info) = existing_obj_info.as_ref()
|
||||
&& let Some(block_reason) = check_retention_for_modification(
|
||||
&existing_obj_info.user_defined,
|
||||
new_mode.as_deref(),
|
||||
new_retain_until,
|
||||
bypass_governance,
|
||||
)
|
||||
{
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, block_reason.error_message()));
|
||||
}
|
||||
|
||||
let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
|
||||
opts.object_lock_retention = Some(ObjectLockRetentionOptions {
|
||||
mode: new_mode,
|
||||
retain_until: new_retain_until,
|
||||
bypass_governance,
|
||||
});
|
||||
|
||||
// PutObjectRetention only rewrites metadata, so the object PUT path that normally
|
||||
// computes a replication decision and schedules replication never runs for it. Without
|
||||
// scheduling here the peer keeps the previous, unprotected lock state and a
|
||||
// WORM-protected object stays deletable on the replica.
|
||||
//
|
||||
// Mirror the PUT path ordering (see object_usecase::put_object): compute the decision
|
||||
// once BEFORE the commit and persist the pending marker with it, so a restart between
|
||||
// the commit and the worker write-back leaves a Pending status on disk for the scanner
|
||||
// to re-drive. Scheduling without that marker would lose the task silently.
|
||||
let dsc = match existing_obj_info.as_ref() {
|
||||
Some(info) => {
|
||||
must_replicate_metadata(
|
||||
&bucket,
|
||||
&key,
|
||||
&info.user_defined,
|
||||
info.user_tags.as_ref().clone(),
|
||||
opts.delete_marker_replication_status(),
|
||||
opts.clone(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => ReplicateDecision::new(),
|
||||
};
|
||||
|
||||
let mut eval_metadata = parse_object_lock_retention(retention)?;
|
||||
if dsc.replicate_any() {
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
|
||||
}
|
||||
opts.eval_metadata = Some(eval_metadata);
|
||||
|
||||
let object_info = store.put_object_metadata(&bucket, &key, &opts).await.map_err(|e| {
|
||||
error!("put_object_metadata failed, {}", e.to_string());
|
||||
S3Error::from(ApiError::from(e))
|
||||
})?;
|
||||
|
||||
// The current target transport carries the updated metadata in a full-object PUT.
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(object_info.clone(), store.clone(), dsc).await;
|
||||
}
|
||||
|
||||
let output = PutObjectRetentionOutput {
|
||||
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
|
||||
};
|
||||
|
||||
let version_id = req.input.version_id.clone().unwrap_or_else(|| Uuid::new_v4().to_string());
|
||||
helper = helper.object(object_info).version_id(version_id);
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata,
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(name = "put_object_tagging", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_object_tagging_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectTaggingInput>,
|
||||
) -> S3Result<S3Response<PutObjectTaggingOutput>> {
|
||||
let start_time = std::time::Instant::now();
|
||||
let mut helper = OperationHelper::new(&req, EventName::ObjectTaggingPut, S3Operation::PutObjectTagging);
|
||||
let PutObjectTaggingInput {
|
||||
bucket,
|
||||
key: object,
|
||||
tagging,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
reject_veeam_sosapi_mutation(self, &bucket, &object).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &object).await?;
|
||||
|
||||
crate::storage::s3_api::tagging::validate_object_tag_set(&tagging.tag_set)?;
|
||||
|
||||
let Some(store) = self.server_ctx.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let tags = encode_tags(tagging.tag_set.clone());
|
||||
debug!("Encoded tags: {}", tags);
|
||||
|
||||
let version_id = req.input.version_id.clone();
|
||||
let mut opts = get_opts(&bucket, &object, version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let existing_object_info = match store.get_object_info(&bucket, &object, &opts).await {
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
// Replication lag window: apply the tagging update on a
|
||||
// replication target that already has the object
|
||||
// (backlog#1675 P1-5). No local object exists, so no bucket
|
||||
// notification event is emitted for the proxied write.
|
||||
if (is_err_object_not_found(&e) || is_err_version_not_found(&e))
|
||||
&& replication_proxy_allowed(&req.extensions)
|
||||
&& Self::proxy_put_object_tagging(&bucket, &object, version_id.clone(), &req.headers, &tagging.tag_set)
|
||||
.await
|
||||
.is_some()
|
||||
{
|
||||
counter!("rustfs_put_object_tagging_success").increment(1);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "put")
|
||||
.record(duration.as_secs_f64());
|
||||
return Ok(S3Response::new(PutObjectTaggingOutput {
|
||||
version_id: req.input.version_id.clone(),
|
||||
}));
|
||||
}
|
||||
return Err(ApiError::from(e).into());
|
||||
}
|
||||
};
|
||||
let dsc = must_replicate_metadata(
|
||||
&bucket,
|
||||
&object,
|
||||
&existing_object_info.user_defined,
|
||||
tags.clone(),
|
||||
existing_object_info.replication_status.clone(),
|
||||
opts.clone(),
|
||||
)
|
||||
.await;
|
||||
if dsc.replicate_any() {
|
||||
let mut eval_metadata = HashMap::new();
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
|
||||
insert_str(
|
||||
&mut eval_metadata,
|
||||
SUFFIX_TAGGING_TIMESTAMP,
|
||||
OffsetDateTime::now_utc().format(&Rfc3339).unwrap_or_default(),
|
||||
);
|
||||
opts.eval_metadata = Some(eval_metadata);
|
||||
}
|
||||
|
||||
let put_tags_result = store.put_object_tags(&bucket, &object, &tags, &opts).await;
|
||||
let object_info = put_tags_result.map_err(|e| {
|
||||
error!("Failed to put object tags: {}", e);
|
||||
counter!("rustfs_put_object_tagging_failure").increment(1);
|
||||
ApiError::from(e)
|
||||
})?;
|
||||
|
||||
let event_object_info = Some(object_info.clone());
|
||||
if dsc.replicate_any() {
|
||||
schedule_metadata_replication(object_info, store.clone(), dsc).await;
|
||||
}
|
||||
|
||||
counter!("rustfs_put_object_tagging_success").increment(1);
|
||||
|
||||
let event_version_id = req
|
||||
.input
|
||||
.version_id
|
||||
.as_deref()
|
||||
.filter(|version_id| !version_id.is_empty())
|
||||
.map(str::to_string)
|
||||
.or_else(|| {
|
||||
event_object_info
|
||||
.as_ref()
|
||||
.and_then(|info| info.version_id.map(|version_id| version_id.to_string()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if let Some(event_object_info) = event_object_info {
|
||||
helper = helper.object(event_object_info);
|
||||
}
|
||||
helper = helper.version_id(event_version_id);
|
||||
|
||||
let result = Ok(S3Response::new(PutObjectTaggingOutput {
|
||||
version_id: req.input.version_id.clone(),
|
||||
}));
|
||||
let _ = helper.complete(&result);
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&object,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::ObjectMetadata,
|
||||
);
|
||||
let duration = start_time.elapsed();
|
||||
histogram!("rustfs_object_tagging_operation_duration_seconds", "operation" => "put").record(duration.as_secs_f64());
|
||||
result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of CopyObject, UploadPartCopy, CompleteMultipartUpload, DeleteObject,
|
||||
//! DeleteObjects, RestoreObject and SelectObjectContent: batch E of the gateway migration
|
||||
//! (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "complete_multipart_upload", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn complete_multipart_upload_edge(
|
||||
&self,
|
||||
req: S3Request<CompleteMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CompleteMultipartUploadOutput>> {
|
||||
crate::hp_guard!("S3::complete_multipart_upload");
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
Box::pin(usecase.execute_complete_multipart_upload(req)).await
|
||||
}
|
||||
|
||||
/// Copy an object from one location to another
|
||||
#[instrument(name = "copy_object", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn copy_object_edge(
|
||||
&self,
|
||||
req: S3Request<CopyObjectInput>,
|
||||
) -> S3Result<S3Response<CopyObjectOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_copy_object(req).await
|
||||
}
|
||||
|
||||
/// Delete an object
|
||||
#[instrument(name = "delete_object", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_object_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||
crate::hp_guard!("S3::delete_object");
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<crate::storage::request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.clone());
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
let result = Box::pin(usecase.execute_delete_object(req)).await;
|
||||
if let Err(error) = &result {
|
||||
log_object_internal_error("DeleteObject", request_id.as_deref(), error);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Delete multiple objects
|
||||
#[instrument(name = "delete_objects", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn delete_objects_edge(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectsInput>,
|
||||
) -> S3Result<S3Response<DeleteObjectsOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_delete_objects(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn restore_object_edge(
|
||||
&self,
|
||||
req: S3Request<RestoreObjectInput>,
|
||||
) -> S3Result<S3Response<RestoreObjectOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_restore_object(req).await
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn select_object_content_edge(
|
||||
&self,
|
||||
req: S3Request<SelectObjectContentInput>,
|
||||
) -> S3Result<S3Response<SelectObjectContentOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_select_object_content(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "upload_part_copy", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn upload_part_copy_edge(
|
||||
&self,
|
||||
req: S3Request<UploadPartCopyInput>,
|
||||
) -> S3Result<S3Response<UploadPartCopyOutput>> {
|
||||
record_s3_op(S3Operation::UploadPartCopy);
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::UploadPartCopy, |req| Box::pin(usecase.execute_upload_part_copy(req))).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of GetObject, HeadObject and GetObjectAttributes: batch C of the gateway
|
||||
//! migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::s3_api;
|
||||
use tracing::instrument;
|
||||
|
||||
impl FS {
|
||||
/// Get bucket notification
|
||||
#[instrument(
|
||||
name = "get_object",
|
||||
level = "debug",
|
||||
skip(self, req),
|
||||
fields(start_time=?time::OffsetDateTime::now_utc())
|
||||
)]
|
||||
pub(in crate::storage::ecfs) async fn get_object_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectInput>,
|
||||
) -> S3Result<S3Response<GetObjectOutput>> {
|
||||
crate::hp_guard!("S3::get_object");
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<crate::storage::request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.clone());
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
let result = usecase.execute_get_object(req).await;
|
||||
if let Err(error) = &result {
|
||||
log_object_internal_error("GetObject", request_id.as_deref(), error);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn get_object_attributes_edge(
|
||||
&self,
|
||||
req: S3Request<GetObjectAttributesInput>,
|
||||
) -> S3Result<S3Response<GetObjectAttributesOutput>> {
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_get_object_attributes(req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "head_object", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn head_object_edge(
|
||||
&self,
|
||||
req: S3Request<HeadObjectInput>,
|
||||
) -> S3Result<S3Response<HeadObjectOutput>> {
|
||||
crate::hp_guard!("S3::head_object");
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_head_object(req).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Legacy s3s edge of PutObject and UploadPart (PostObject has no trait method of its own and
|
||||
//! reaches PutObject): batch D of the gateway migration (rustfs/backlog#2765).
|
||||
//!
|
||||
//! `impl S3 for FS` in `ecfs.rs` calls the `*_edge` fn of each operation here. The bodies are the
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
use crate::storage::s3_api;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
|
||||
impl FS {
|
||||
#[instrument(name = "put_object", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn put_object_edge(
|
||||
&self,
|
||||
req: S3Request<PutObjectInput>,
|
||||
) -> S3Result<S3Response<PutObjectOutput>> {
|
||||
crate::hp_guard!("S3::put_object");
|
||||
let usecase = s3_api::object_usecase_for(self);
|
||||
usecase.execute_put_object(self, req).await
|
||||
}
|
||||
|
||||
#[instrument(name = "upload_part", level = "debug", skip(self, req))]
|
||||
pub(in crate::storage::ecfs) async fn upload_part_edge(
|
||||
&self,
|
||||
req: S3Request<UploadPartInput>,
|
||||
) -> S3Result<S3Response<UploadPartOutput>> {
|
||||
crate::hp_guard!("S3::upload_part");
|
||||
record_s3_op(S3Operation::UploadPart);
|
||||
let usecase = s3_api::multipart_usecase_for(self);
|
||||
audit_only_operation(req, S3Operation::UploadPart, |req| usecase.execute_upload_part(req)).await
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tests of the `bucket_access` legacy-edge handlers (batch A2), moved unchanged from `ecfs.rs` and
|
||||
//! `ecfs_test.rs`.
|
||||
|
||||
use super::{build_request, store_uninitialized_premise_holds};
|
||||
use crate::storage::ecfs::*;
|
||||
use http::Method;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_bucket_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetBucketAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_bucket_acl(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_bucket_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutBucketAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_bucket_acl(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tests of the `bucket_replication_lifecycle` legacy-edge handlers (batch A3), moved unchanged
|
||||
//! from `ecfs.rs` and `ecfs_test.rs`.
|
||||
|
||||
use super::super::bucket_replication_lifecycle::{
|
||||
SITE_REPLICATION_GATE_FORCE_DISABLED, SITE_REPLICATION_GATE_FORCE_ENABLED, SITE_REPLICATION_GATE_TEST_OVERRIDE,
|
||||
};
|
||||
use super::{build_request, store_uninitialized_premise_holds};
|
||||
use crate::storage::access::ReqInfo;
|
||||
use crate::storage::ecfs::*;
|
||||
use http::Method;
|
||||
use http::StatusCode;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
fn replication_config_edit_request<T>(input: T, is_owner: bool) -> S3Request<T> {
|
||||
let mut req = S3Request {
|
||||
input,
|
||||
method: Method::PUT,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
req.extensions.insert(ReqInfo {
|
||||
is_owner,
|
||||
..Default::default()
|
||||
});
|
||||
req
|
||||
}
|
||||
|
||||
fn put_bucket_replication_input() -> PutBucketReplicationInput {
|
||||
PutBucketReplicationInput {
|
||||
bucket: "test-bucket".to_string(),
|
||||
checksum_algorithm: None,
|
||||
content_md5: None,
|
||||
expected_bucket_owner: None,
|
||||
replication_configuration: ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: Vec::new(),
|
||||
},
|
||||
token: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn delete_bucket_replication_input() -> DeleteBucketReplicationInput {
|
||||
DeleteBucketReplicationInput {
|
||||
bucket: "test-bucket".to_string(),
|
||||
expected_bucket_owner: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_replication_deny_edit(err: &S3Error) {
|
||||
match err.code() {
|
||||
S3ErrorCode::Custom(code) => assert_eq!(code, "XMinioReplicationDenyEdit"),
|
||||
other => panic!("expected XMinioReplicationDenyEdit, got {other:?}"),
|
||||
}
|
||||
assert_eq!(err.status_code(), Some(StatusCode::BAD_REQUEST));
|
||||
}
|
||||
|
||||
/// Single test on purpose: the branches share the process-wide gate
|
||||
/// override, and parallel tests would race it.
|
||||
#[tokio::test]
|
||||
async fn replication_config_edit_gate_denies_only_non_owner_under_site_replication() {
|
||||
let fs = FS::new();
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_ENABLED, Ordering::SeqCst);
|
||||
|
||||
// Non-owner PUT/DELETE through the real S3 handlers: denied by the
|
||||
// gate before the usecase (and thus the store) is ever touched.
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner PutBucketReplication must be denied while site replication is enabled");
|
||||
assert_replication_deny_edit(&err);
|
||||
let err = fs
|
||||
.delete_bucket_replication(replication_config_edit_request(delete_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner DeleteBucketReplication must be denied while site replication is enabled");
|
||||
assert_replication_deny_edit(&err);
|
||||
|
||||
// Owner passes the gate (the usecase's empty-rules structure error
|
||||
// proves the request reached the usecase instead of the deny path).
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), true))
|
||||
.await
|
||||
.expect_err("owner request should pass the gate and fail later on config validation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
// Without site replication the policy check alone still governs the edit.
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(SITE_REPLICATION_GATE_FORCE_DISABLED, Ordering::SeqCst);
|
||||
let err = fs
|
||||
.put_bucket_replication(replication_config_edit_request(put_bucket_replication_input(), false))
|
||||
.await
|
||||
.expect_err("non-owner request should pass the gate and fail later on config validation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
SITE_REPLICATION_GATE_TEST_OVERRIDE.store(0, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectLockConfigurationInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.object_lock_configuration(Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs
|
||||
.put_object_lock_configuration(build_request(input, Method::PUT))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tests of the `listing_multipart` legacy-edge handlers (batch B2), moved unchanged from `ecfs.rs`
|
||||
//! and `ecfs_test.rs`.
|
||||
|
||||
use super::build_request;
|
||||
use crate::storage::StorageError;
|
||||
use crate::storage::ecfs::*;
|
||||
use http::{Method, StatusCode};
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_apis_report_storage_read_quorum_as_unavailable() {
|
||||
crate::app::gating_test_env::run_large_stack_test("list-quorum-surface", || async {
|
||||
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let bucket = format!("list-quorum-{}", uuid::Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create listing bucket");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let saved_disks = set.disks.read().await.clone();
|
||||
{
|
||||
let mut disks = set.disks.write().await;
|
||||
for disk in disks.iter_mut().skip(1) {
|
||||
*disk = None;
|
||||
}
|
||||
}
|
||||
let fs = FS::new();
|
||||
let errors = [
|
||||
S3::list_objects(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectsInput {
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("V1 cannot list below storage read quorum"),
|
||||
S3::list_objects_v2(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectsV2Input {
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("V2 cannot list below storage read quorum"),
|
||||
S3::list_object_versions(
|
||||
&fs,
|
||||
build_request(
|
||||
ListObjectVersionsInput {
|
||||
bucket,
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect_err("versions cannot list below storage read quorum"),
|
||||
];
|
||||
*set.disks.write().await = saved_disks;
|
||||
for error in errors {
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
|
||||
let source = error.source().expect("retain storage diagnostics");
|
||||
assert!(
|
||||
matches!(
|
||||
source.downcast_ref::<StorageError>(),
|
||||
Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
|
||||
),
|
||||
"the production LIST boundary must expose a typed read-quorum failure: {source}"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tests of the legacy-edge handlers, one module per migration batch, and the fixtures they
|
||||
//! share.
|
||||
|
||||
use crate::storage::ecfs::*;
|
||||
use http::{Extensions, HeaderMap, Method, Uri};
|
||||
|
||||
fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
input,
|
||||
method,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Premise guard for the `*_returns_internal_error_when_store_uninitialized`
|
||||
/// tests (backlog#1830): they assert the error path taken while the global
|
||||
/// object layer is absent. Under nextest — the authoritative runner — every
|
||||
/// test owns its process, so the premise always holds and the assertion
|
||||
/// always runs. Under the documented shared-process `cargo test` fallback a
|
||||
/// sibling test may have initialized the store first; the premise is then
|
||||
/// unattainable, so the test skips instead of asserting against a scenario
|
||||
/// it does not describe.
|
||||
fn store_uninitialized_premise_holds() -> bool {
|
||||
if crate::runtime_sources::current_object_store_handle().is_some() {
|
||||
eprintln!("skipping store-uninitialized assertion: a sibling test already initialized the global object layer");
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
mod bucket_access;
|
||||
mod bucket_replication_lifecycle;
|
||||
mod listing_multipart;
|
||||
mod object_meta;
|
||||
@@ -0,0 +1,208 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Tests of the `object_meta` legacy-edge handlers (batch B1), moved unchanged from `ecfs.rs` and
|
||||
//! `ecfs_test.rs`.
|
||||
|
||||
use super::{build_request, store_uninitialized_premise_holds};
|
||||
use crate::storage::ecfs::*;
|
||||
use http::Method;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_acl(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_legal_hold_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectLegalHoldInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_legal_hold(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_retention_returns_internal_error_when_store_uninitialized() {
|
||||
let input = GetObjectRetentionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_retention(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_legal_hold_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectLegalHoldInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_legal_hold(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_acl_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectAclInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_acl(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_retention_returns_internal_error_when_store_uninitialized() {
|
||||
let input = PutObjectRetentionInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_retention(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = GetObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.get_object_tagging(build_request(input, Method::GET)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_rejects_too_many_tags() {
|
||||
let tag_set = (0..11)
|
||||
.map(|index| Tag {
|
||||
key: Some(format!("k{index}")),
|
||||
value: Some(format!("v{index}")),
|
||||
})
|
||||
.collect();
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging { tag_set })
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Cannot have more than 10 tags per object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_rejects_empty_tag_key_before_store_lookup() {
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some(String::new()),
|
||||
value: Some("v1".to_string()),
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidTag);
|
||||
assert!(err.to_string().contains("Tag key cannot be empty"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = PutObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.tagging(Tagging {
|
||||
tag_set: vec![Tag {
|
||||
key: Some("k".to_string()),
|
||||
value: Some("v".to_string()),
|
||||
}],
|
||||
})
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs.put_object_tagging(build_request(input, Method::PUT)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_delete_object_tagging_returns_internal_error_when_store_uninitialized() {
|
||||
if !store_uninitialized_premise_holds() {
|
||||
return;
|
||||
}
|
||||
let input = DeleteObjectTaggingInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let fs = FS::new();
|
||||
let err = fs
|
||||
.delete_object_tagging(build_request(input, Method::DELETE))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
@@ -14476,34 +14476,165 @@ fn object_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
let source = include_str!("../app/multipart_usecase.rs");
|
||||
const MULTIPART_KEY_GUARD: &str = "validate_table_catalog_object_mutation(&bucket, &key).await?;";
|
||||
const OBJECT_METADATA_OBJECT_GUARD: &str = "validate_table_catalog_object_mutation(&bucket, &object).await?;";
|
||||
const OBJECT_METADATA_KEY_GUARD: &str = MULTIPART_KEY_GUARD;
|
||||
|
||||
fn assert_multipart_reserved_prefix_guard_calls(source: &crate::guard_sources::SourceSet) {
|
||||
assert_eq!(
|
||||
source
|
||||
.matches("validate_table_catalog_object_mutation(&bucket, &key).await?;")
|
||||
.count(),
|
||||
4
|
||||
source.count(MULTIPART_KEY_GUARD),
|
||||
4,
|
||||
"reserved-prefix guard calls across {}",
|
||||
source.describe()
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_object_metadata_reserved_prefix_guard_calls(source: &crate::guard_sources::SourceSet) {
|
||||
assert_eq!(
|
||||
source.count(OBJECT_METADATA_OBJECT_GUARD),
|
||||
2,
|
||||
"reserved-prefix guard calls across {}",
|
||||
source.describe()
|
||||
);
|
||||
assert_eq!(
|
||||
source.count(OBJECT_METADATA_KEY_GUARD),
|
||||
2,
|
||||
"reserved-prefix guard calls across {}",
|
||||
source.describe()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_metadata_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
let source = include_str!("../storage/ecfs.rs");
|
||||
fn multipart_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
assert_multipart_reserved_prefix_guard_calls(&crate::guard_sources::SourceSet::production(
|
||||
"src/app/multipart_usecase.rs",
|
||||
"src/app/multipart_usecase",
|
||||
));
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
source
|
||||
.matches("validate_table_catalog_object_mutation(&bucket, &object).await?;")
|
||||
.count(),
|
||||
2
|
||||
#[test]
|
||||
fn object_metadata_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
assert_object_metadata_reserved_prefix_guard_calls(&crate::guard_sources::SourceSet::production(
|
||||
"src/storage/ecfs.rs",
|
||||
"src/storage/legacy_edge",
|
||||
));
|
||||
}
|
||||
|
||||
/// `count` production calls of `guard`, one per method, for count-guard fixtures.
|
||||
fn guarded_methods(guard: &str, count: usize) -> String {
|
||||
let methods: String = (0..count)
|
||||
.map(|index| format!(" async fn guarded_{index}(&self) {{\n {guard}\n }}\n"))
|
||||
.collect();
|
||||
format!("impl Usecase {{\n{methods}}}\n")
|
||||
}
|
||||
|
||||
fn guard_fixture(files: &[(&str, &str)], file: &str, split_dir: &str) -> crate::guard_sources::SourceSet {
|
||||
let root = crate::guard_sources::fixture_root(files);
|
||||
crate::guard_sources::SourceSet::production_under(root.path(), file, split_dir)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_count_guard_fails_when_a_dropped_production_call_is_replaced_by_test_mentions() {
|
||||
let parent = format!(
|
||||
"{}\n#[cfg(test)]\nmod tests {{\n fn mentions() {{\n {MULTIPART_KEY_GUARD}\n }}\n}}\n",
|
||||
guarded_methods(MULTIPART_KEY_GUARD, 2)
|
||||
);
|
||||
let source = guard_fixture(
|
||||
&[
|
||||
("src/app/multipart_usecase.rs", parent.as_str()),
|
||||
("src/app/multipart_usecase/complete.rs", guarded_methods(MULTIPART_KEY_GUARD, 1).as_str()),
|
||||
],
|
||||
"src/app/multipart_usecase.rs",
|
||||
"src/app/multipart_usecase",
|
||||
);
|
||||
assert_eq!(
|
||||
source
|
||||
.matches("validate_table_catalog_object_mutation(&bucket, &key).await?;")
|
||||
.count(),
|
||||
2
|
||||
source.count(MULTIPART_KEY_GUARD),
|
||||
3,
|
||||
"fixture: three production calls and one test mention"
|
||||
);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_multipart_reserved_prefix_guard_calls(&source));
|
||||
|
||||
assert!(message.contains("reserved-prefix guard calls"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_count_guard_passes_when_calls_move_into_the_split_directory() {
|
||||
let source = guard_fixture(
|
||||
&[
|
||||
("src/app/multipart_usecase.rs", guarded_methods(MULTIPART_KEY_GUARD, 1).as_str()),
|
||||
("src/app/multipart_usecase/complete.rs", guarded_methods(MULTIPART_KEY_GUARD, 3).as_str()),
|
||||
],
|
||||
"src/app/multipart_usecase.rs",
|
||||
"src/app/multipart_usecase",
|
||||
);
|
||||
|
||||
assert_multipart_reserved_prefix_guard_calls(&source);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_metadata_key_count_guard_fails_when_a_moved_call_is_broken() {
|
||||
let moved_and_broken = guarded_methods(MULTIPART_KEY_GUARD, 1).replace("&key)", "key.as_str())");
|
||||
let parent = format!(
|
||||
"{}{}",
|
||||
guarded_methods(OBJECT_METADATA_OBJECT_GUARD, 2),
|
||||
guarded_methods(OBJECT_METADATA_KEY_GUARD, 1)
|
||||
);
|
||||
let source = guard_fixture(
|
||||
&[
|
||||
("src/storage/ecfs.rs", parent.as_str()),
|
||||
("src/storage/legacy_edge/object_write.rs", moved_and_broken.as_str()),
|
||||
],
|
||||
"src/storage/ecfs.rs",
|
||||
"src/storage/legacy_edge",
|
||||
);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_object_metadata_reserved_prefix_guard_calls(&source));
|
||||
|
||||
assert!(message.contains("src/storage/legacy_edge/object_write.rs"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_metadata_key_count_guard_passes_when_a_call_moves_intact() {
|
||||
let parent = format!(
|
||||
"{}{}",
|
||||
guarded_methods(OBJECT_METADATA_OBJECT_GUARD, 2),
|
||||
guarded_methods(OBJECT_METADATA_KEY_GUARD, 1)
|
||||
);
|
||||
let source = guard_fixture(
|
||||
&[
|
||||
("src/storage/ecfs.rs", parent.as_str()),
|
||||
(
|
||||
"src/storage/legacy_edge/object_write.rs",
|
||||
guarded_methods(OBJECT_METADATA_KEY_GUARD, 1).as_str(),
|
||||
),
|
||||
],
|
||||
"src/storage/ecfs.rs",
|
||||
"src/storage/legacy_edge",
|
||||
);
|
||||
|
||||
assert_object_metadata_reserved_prefix_guard_calls(&source);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_metadata_count_guard_fails_when_a_legacy_edge_test_replaces_a_production_call() {
|
||||
let parent = format!(
|
||||
"{}{}",
|
||||
guarded_methods(OBJECT_METADATA_OBJECT_GUARD, 1),
|
||||
guarded_methods(OBJECT_METADATA_KEY_GUARD, 2)
|
||||
);
|
||||
let source = guard_fixture(
|
||||
&[
|
||||
("src/storage/ecfs.rs", parent.as_str()),
|
||||
("src/storage/legacy_edge/tests/object_meta.rs", OBJECT_METADATA_OBJECT_GUARD),
|
||||
],
|
||||
"src/storage/ecfs.rs",
|
||||
"src/storage/legacy_edge",
|
||||
);
|
||||
|
||||
let message = crate::guard_sources::panic_message(move || assert_object_metadata_reserved_prefix_guard_calls(&source));
|
||||
|
||||
assert!(message.contains("reserved-prefix guard calls"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+2
-1
@@ -38,6 +38,7 @@ their issue closes.
|
||||
| `check_no_tokio_io_uring.sh` | ci-gate | Keeps tokio's io-uring backend disabled | ci.yml Quick Checks |
|
||||
| `check_s3s_footprint.sh` | ci-gate | Exact, lower-only baseline on the direct s3s surface ahead of the gateway migration (reads `.config/s3s-footprint-baseline.txt`; `s3_error_lines` leaves out lines whose file imports the macro from `rustfs-s3-types`, rule in the script header); `--mode allowlist` fails on any Rust file referencing s3s outside `.config/s3s-edge-allowlist.txt`, `--dry-run` lists them | ci.yml Quick Checks; `make pre-commit`; `make script-tests` (`--self-test`) |
|
||||
| `check_test_count_ratchet.sh` | ci-gate | Per-crate test-count floor and `#[ignore]` ceiling against `.config/test-count-baseline.txt`; no crate may fall below its tests row or rise above its ignored row (extra tests and new crates pass with a note) | ci.yml Quick Checks; `make pre-commit` |
|
||||
| `guard_source_set.sh` | ci-gate (library) | Sourced helpers for text guards that must survive a mechanical file split (rustfs/backlog#2765): `guard_source_set` resolves a required file plus the split directory (a missing required file fails), `module_relative_rg` rewrites a `super::` rule for each file's module depth | Sourced by `check_architecture_migration_rules.sh` and `check_logging_guardrails.sh`; contract tests `test_guard_source_set.sh`; end-to-end split probe `test_split_guard_rules.sh` (both `make script-tests`) |
|
||||
| `check_unsafe_code_allowances.sh` | ci-gate | Unsafe-code allowance ledger guard | ci.yml Quick Checks |
|
||||
| `layer-dependency-baseline.txt` | ci-gate (data) | Committed baseline consumed by `check_layer_dependencies.sh` | [Architecture guard troubleshooting](../docs/operations/architecture-guard-troubleshooting.md) |
|
||||
| `static.sh` | ci-gate | Static-build helper executed inside image builds | `Dockerfile.source`, `Dockerfile.decommission-local` |
|
||||
@@ -169,7 +170,7 @@ their issue closes.
|
||||
|---|---|---|---|
|
||||
| `codemods/` | dev-tool | Scripted, idempotent source migrations for the gateway migration (rustfs/backlog#2734); `s3_error_to_s3_types.py` moves s3s error-type paths onto `rustfs-s3-types` per crate closure and refuses the s3s edge allowlist | `make script-tests` (`--self-test`); usage in each script's docstring |
|
||||
| `fuzz/` | ci-gate | Unified cargo-fuzz runner and helpers for the `fuzz/` sub-workspace | fuzz.yml; `fuzz/README.md` |
|
||||
| `release/` | ci-gate | Release creation and DEB/RPM version-normalization helpers | build.yml; package.yml; manual-main-packages.yml |
|
||||
| `release/` | ci-gate | Release creation and DEB/RPM version-normalization helpers | build.yml; package.yml |
|
||||
| `s3-tests/` | ci-gate | ceph/s3-tests compatibility harness (allow-lists, patches, report tooling) | ci.yml; e2e-s3tests.yml; `scripts/s3-tests/README.md` |
|
||||
| `security/` | ci-gate | Workflow-pin enforcement and release supply-chain asset generation | audit.yml; build.yml |
|
||||
| `table-catalog/` | dev-tool | S3-Tables / pyiceberg validation suite | `docs/architecture/s3-tables-support-matrix.md` |
|
||||
|
||||
@@ -118,6 +118,45 @@ require_source_contains "docs/architecture/ecstore-api-facade-inventory.md" "Do
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
# Rules about one of the large files below read that file plus the sibling
|
||||
# directory a mechanical split moves its code into (rustfs/backlog#2765), so a
|
||||
# split neither breaks nor silently blinds them. The *_SPLIT_SOURCES arrays hold
|
||||
# only the directory part, for rules that select the file itself by glob.
|
||||
# shellcheck source=scripts/guard_source_set.sh
|
||||
source "${ROOT_DIR}/scripts/guard_source_set.sh"
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/storage/ecfs.rs rustfs/src/storage/legacy_edge ||
|
||||
report_failure "ECFS guard source set is unreadable"
|
||||
ECFS_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/app/bucket_usecase.rs rustfs/src/app/bucket_usecase ||
|
||||
report_failure "bucket usecase guard source set is unreadable"
|
||||
BUCKET_USECASE_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
BUCKET_USECASE_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/app/multipart_usecase.rs rustfs/src/app/multipart_usecase ||
|
||||
report_failure "multipart usecase guard source set is unreadable"
|
||||
MULTIPART_USECASE_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
MULTIPART_USECASE_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/lifecycle_transition_api_test ||
|
||||
report_failure "lifecycle transition API test guard source set is unreadable"
|
||||
LIFECYCLE_TRANSITION_TEST_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/storage/ecfs_test.rs rustfs/src/storage/legacy_edge/tests ||
|
||||
report_failure "ECFS test guard source set is unreadable"
|
||||
ECFS_TEST_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
# Every .rs file under rustfs/src/app, for depth-aware rules over the whole app tree.
|
||||
APP_RUNTIME_RULE_SOURCES=()
|
||||
while IFS= read -r app_source; do
|
||||
APP_RUNTIME_RULE_SOURCES+=("$app_source")
|
||||
done < <(cd "$ROOT_DIR" && rg --files rustfs/src/app --glob '*.rs' --glob '!rustfs/src/app/storage_api.rs' | LC_ALL=C sort)
|
||||
APP_OBJECT_SOURCES=()
|
||||
while IFS= read -r app_source; do
|
||||
APP_OBJECT_SOURCES+=("$app_source")
|
||||
done < <(cd "$ROOT_DIR" && rg --files rustfs/src/app/object --glob '*.rs' | LC_ALL=C sort)
|
||||
APP_USECASE_RULE_SOURCES=("${BUCKET_USECASE_GUARD_SOURCES[@]}" rustfs/src/app/object_usecase.rs "${APP_OBJECT_SOURCES[@]}"
|
||||
"${MULTIPART_USECASE_GUARD_SOURCES[@]}" "${LIFECYCLE_TRANSITION_TEST_GUARD_SOURCES[@]}")
|
||||
if [[ ${#APP_RUNTIME_RULE_SOURCES[@]} -eq 0 || ${#APP_OBJECT_SOURCES[@]} -eq 0 ]]; then
|
||||
report_failure "app source listing for depth-aware import rules is empty"
|
||||
fi
|
||||
|
||||
PR_TYPES_FILE="${TMP_DIR}/pr_types.txt"
|
||||
PR_TYPE_HITS_FILE="${TMP_DIR}/pr_type_hits.txt"
|
||||
SOURCE_MARKERS_FILE="${TMP_DIR}/source_markers.txt"
|
||||
@@ -2157,7 +2196,7 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::app::context::(?:\{[^}]*\b(?:resolve_object_store_handle|resolve_buffer_config|resolve_internode_metrics|resolve_local_node_name|resolve_action_credentials|resolve_notify_interface|resolve_performance_metrics|resolve_encryption_service|resolve_region|resolve_ready_iam_handle)\b|(?:resolve_object_store_handle|resolve_buffer_config|resolve_internode_metrics|resolve_local_node_name|resolve_action_credentials|resolve_notify_interface|resolve_performance_metrics|resolve_encryption_service|resolve_region|resolve_ready_iam_handle)\b)' \
|
||||
rustfs/src/storage/access.rs \
|
||||
rustfs/src/storage/ecfs.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
rustfs/src/storage/ecfs_extend.rs \
|
||||
rustfs/src/storage/helper.rs \
|
||||
rustfs/src/storage/concurrency/manager.rs \
|
||||
@@ -2175,7 +2214,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::app::(?:bucket_usecase|multipart_usecase|object_usecase|object\b)|Default(?:Bucket|Multipart|Object)Usecase::from_global\(\)' \
|
||||
rustfs/src/storage/ecfs.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
--glob '*.rs' || true
|
||||
) >"$RUSTFS_STORAGE_ECFS_USECASE_BYPASS_HITS_FILE"
|
||||
|
||||
@@ -2228,7 +2267,10 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'use crate::storage::\*;' rustfs/src/app --glob '*_usecase.rs' --glob 'object/*.rs' || true
|
||||
# A glob containing a slash is anchored at the search root, so object files
|
||||
# need the leading **/ to match under rustfs/src/app.
|
||||
rg -n --with-filename 'use crate::storage::\*;' rustfs/src/app --glob '*_usecase.rs' --glob '**/object/*.rs' \
|
||||
${BUCKET_USECASE_SPLIT_SOURCES[@]+"${BUCKET_USECASE_SPLIT_SOURCES[@]}"} ${MULTIPART_USECASE_SPLIT_SOURCES[@]+"${MULTIPART_USECASE_SPLIT_SOURCES[@]}"} || true
|
||||
) >"$RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE" ]]; then
|
||||
@@ -2246,7 +2288,8 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::storage::s3_api::|use crate::storage::s3_api|super::s3_api::|use super::s3_api' rustfs/src/app --glob '*_usecase.rs' --glob 'object/*.rs' || true
|
||||
rg -n --with-filename 'crate::storage::s3_api::|use crate::storage::s3_api|super::s3_api::|use super::s3_api' rustfs/src/app --glob '*_usecase.rs' --glob '**/object/*.rs' \
|
||||
${BUCKET_USECASE_SPLIT_SOURCES[@]+"${BUCKET_USECASE_SPLIT_SOURCES[@]}"} ${MULTIPART_USECASE_SPLIT_SOURCES[@]+"${MULTIPART_USECASE_SPLIT_SOURCES[@]}"} || true
|
||||
) >"$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE" ]]; then
|
||||
@@ -2258,13 +2301,19 @@ fi
|
||||
{
|
||||
rg -n --with-filename \
|
||||
'(use crate::storage::(access|helper|options|request_context|sse|timeout_wrapper|head_prefix|concurrency|ecfs)|crate::storage::sse::EncryptionKeyKind|use crate::storage::\{|use crate::storage::[A-Z])' \
|
||||
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs rustfs/src/app/object || true
|
||||
rg -n --with-filename \
|
||||
'use super::(?:\{[^}]*\b(?:DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|StorageError|DiskError|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b|(?:object_api_utils::to_s3s_etag|storageclass|StorageError|DiskError|DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b)' \
|
||||
rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/lifecycle_transition_api_test || true
|
||||
rg -n --with-filename \
|
||||
'use super::(?:\{[^}]*\b(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b|(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b)|super::(?:lifecycle|metadata_sys|object_lock|quota|replication|tagging|target|utils|versioning_sys|transition_api)::|super::super::(?:metadata_sys|lifecycle|target)::' \
|
||||
rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/lifecycle_transition_api_test rustfs/src/app/capacity_dirty_scope_test.rs rustfs/src/app/context.rs rustfs/src/app/context/handles.rs rustfs/src/app/context/interfaces.rs rustfs/src/app/context/runtime_sources.rs || true
|
||||
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs rustfs/src/app/object \
|
||||
${BUCKET_USECASE_SPLIT_SOURCES[@]+"${BUCKET_USECASE_SPLIT_SOURCES[@]}"} ${MULTIPART_USECASE_SPLIT_SOURCES[@]+"${MULTIPART_USECASE_SPLIT_SOURCES[@]}"} || true
|
||||
# The super:: chain that names crate::app depends on each file's depth, so
|
||||
# these rules are rewritten per depth (scripts/guard_source_set.sh).
|
||||
module_relative_rg rustfs/src/app \
|
||||
'(?:use @UP@|use crate::app::)(?:\{[^}]*\b(?:DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|StorageError|DiskError|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b|(?:object_api_utils::to_s3s_etag|storageclass|StorageError|DiskError|DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b)' \
|
||||
-n --with-filename -- "${APP_USECASE_RULE_SOURCES[@]}" ||
|
||||
echo "storage helper import rule could not search its sources"
|
||||
module_relative_rg rustfs/src/app \
|
||||
'(?:use @UP@|use crate::app::)(?:\{[^}]*\b(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b|(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b)|(?:@UP@|crate::app::)(?:lifecycle|metadata_sys|object_lock|quota|replication|tagging|target|utils|versioning_sys|transition_api)::|@UP@super::(?:metadata_sys|lifecycle|target)::' \
|
||||
-n --with-filename -- "${APP_USECASE_RULE_SOURCES[@]}" rustfs/src/app/capacity_dirty_scope_test.rs \
|
||||
rustfs/src/app/context.rs rustfs/src/app/context/handles.rs rustfs/src/app/context/interfaces.rs rustfs/src/app/context/runtime_sources.rs ||
|
||||
echo "storage owner import rule could not search its sources"
|
||||
}
|
||||
) >"$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE"
|
||||
|
||||
@@ -2277,6 +2326,7 @@ fi
|
||||
rg -n --with-filename \
|
||||
'(use crate::storage::(access|request_context|ecfs)|use crate::storage::\{[^}]*Storage(ObjectInfo|ObjectOptions|PutObjReader)|crate::storage::StorageObjectOptions)' \
|
||||
rustfs/src/app/*_test.rs \
|
||||
${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]+"${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]}"} \
|
||||
rustfs/src/admin/router.rs \
|
||||
rustfs/src/admin/console.rs \
|
||||
rustfs/src/admin/handlers/heal.rs \
|
||||
@@ -2291,16 +2341,14 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
{
|
||||
rg -n --with-filename \
|
||||
'use super::(?:\{[^}]*\b(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|AppWarmBackend|WarmBackendGetOpts|TierConfig|TierType|PoolStatus|PoolDecommissionInfo|RebalStatus)\b|(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|AppWarmBackend|WarmBackendGetOpts|TierConfig|TierType|PoolStatus|PoolDecommissionInfo|RebalStatus)\b)' \
|
||||
rustfs/src/app \
|
||||
--glob '*.rs' \
|
||||
--glob '!rustfs/src/app/storage_api.rs' || true
|
||||
rg -n --with-filename \
|
||||
'super::(?:get_server_info|get_total_usable_capacity|get_total_usable_capacity_free|apply_bucket_usage_memory_overlay|load_data_usage_from_backend|record_bucket_object_delete_memory|record_bucket_object_write_memory|remove_bucket_usage_from_backend|get_global_|global_rustfs_port|new_object_layer_fn|set_object_store_resolver|set_global_storage_class|collect_scanner_metrics_report|init_local_disks)|super::super::(?:get_server_info|get_total_usable_capacity|get_total_usable_capacity_free|apply_bucket_usage_memory_overlay|load_data_usage_from_backend|record_bucket_object_delete_memory|record_bucket_object_write_memory|remove_bucket_usage_from_backend|get_global_|global_rustfs_port|new_object_layer_fn|set_object_store_resolver|set_global_storage_class|collect_scanner_metrics_report|init_local_disks)' \
|
||||
rustfs/src/app \
|
||||
--glob '*.rs' \
|
||||
--glob '!rustfs/src/app/storage_api.rs' || true
|
||||
module_relative_rg rustfs/src/app \
|
||||
'use @UP@(?:\{[^}]*\b(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|AppWarmBackend|WarmBackendGetOpts|TierConfig|TierType|PoolStatus|PoolDecommissionInfo|RebalStatus)\b|(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|AppWarmBackend|WarmBackendGetOpts|TierConfig|TierType|PoolStatus|PoolDecommissionInfo|RebalStatus)\b)' \
|
||||
-n --with-filename -- "${APP_RUNTIME_RULE_SOURCES[@]}" ||
|
||||
echo "runtime facade import rule could not search its sources"
|
||||
module_relative_rg rustfs/src/app \
|
||||
'@UP@(?:get_server_info|get_total_usable_capacity|get_total_usable_capacity_free|apply_bucket_usage_memory_overlay|load_data_usage_from_backend|record_bucket_object_delete_memory|record_bucket_object_write_memory|remove_bucket_usage_from_backend|get_global_|global_rustfs_port|new_object_layer_fn|set_object_store_resolver|set_global_storage_class|collect_scanner_metrics_report|init_local_disks)|@UP@super::(?:get_server_info|get_total_usable_capacity|get_total_usable_capacity_free|apply_bucket_usage_memory_overlay|load_data_usage_from_backend|record_bucket_object_delete_memory|record_bucket_object_write_memory|remove_bucket_usage_from_backend|get_global_|global_rustfs_port|new_object_layer_fn|set_object_store_resolver|set_global_storage_class|collect_scanner_metrics_report|init_local_disks)' \
|
||||
-n --with-filename -- "${APP_RUNTIME_RULE_SOURCES[@]}" ||
|
||||
echo "runtime facade path rule could not search its sources"
|
||||
rg -n --with-filename \
|
||||
'crate::app::(?:ECStore|EndpointServerPools|Endpoint|Endpoints|PoolEndpoints|StorageClassConfig|TierConfigMgr|DailyAllTierStats|ScannerMetricsReport|ReplicationStats|DynReplicationPool|NotificationSys|BucketBandwidthMonitor|ExpiryState|get_global_|set_object_store_resolver)' \
|
||||
rustfs/src --glob '*.rs' || true
|
||||
@@ -2524,9 +2572,22 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n -U --with-filename \
|
||||
'use\s+(?:super::super|crate::storage)::\{[^}]*?(?:BucketMetadata|DEFAULT_READ_BUFFER_SIZE|bucket_metadata_sys_initialized|get_global_bucket_metadata_sys|set_bucket_metadata|apply_cors_headers|apply_default_lock_retention_metadata|check_preconditions|decode_tags_to_map|get_adaptive_buffer_size_with_profile|get_buffer_size_opt_in|is_etag_equal|matches_origin_pattern|parse_etag|parse_object_lock_legal_hold|parse_object_lock_retention|process_lambda_configurations|process_queue_configurations|process_topic_configurations|remove_object_lock_metadata_for_copy|remove_object_lock_retention_metadata|validate_bucket_object_lock_enabled|validate_list_object_unordered_with_delimiter)' \
|
||||
rustfs/src/storage/ecfs_test.rs || true
|
||||
# ecfs_test.rs holds its tests in an inline `mod tests`, so the storage root
|
||||
# is one super:: past the depth of the file itself. legacy_edge/tests/ is
|
||||
# where a split of ecfs.rs moves them, with or without that inline module,
|
||||
# so both depths are forbidden there.
|
||||
{
|
||||
module_relative_rg rustfs/src/storage \
|
||||
'use\s+(?:@UP@super::|crate::storage::)\{[^}]*?(?:BucketMetadata|DEFAULT_READ_BUFFER_SIZE|bucket_metadata_sys_initialized|get_global_bucket_metadata_sys|set_bucket_metadata|apply_cors_headers|apply_default_lock_retention_metadata|check_preconditions|decode_tags_to_map|get_adaptive_buffer_size_with_profile|get_buffer_size_opt_in|is_etag_equal|matches_origin_pattern|parse_etag|parse_object_lock_legal_hold|parse_object_lock_retention|process_lambda_configurations|process_queue_configurations|process_topic_configurations|remove_object_lock_metadata_for_copy|remove_object_lock_retention_metadata|validate_bucket_object_lock_enabled|validate_list_object_unordered_with_delimiter)' \
|
||||
-n -U --with-filename -- rustfs/src/storage/ecfs_test.rs ||
|
||||
echo "storage-owner test import rule could not search rustfs/src/storage/ecfs_test.rs"
|
||||
if [[ ${#ECFS_TEST_GUARD_SOURCES[@]} -gt 1 ]]; then
|
||||
module_relative_rg rustfs/src/storage \
|
||||
'use\s+(?:@UP@(?:super::)?|crate::storage::)\{[^}]*?(?:BucketMetadata|DEFAULT_READ_BUFFER_SIZE|bucket_metadata_sys_initialized|get_global_bucket_metadata_sys|set_bucket_metadata|apply_cors_headers|apply_default_lock_retention_metadata|check_preconditions|decode_tags_to_map|get_adaptive_buffer_size_with_profile|get_buffer_size_opt_in|is_etag_equal|matches_origin_pattern|parse_etag|parse_object_lock_legal_hold|parse_object_lock_retention|process_lambda_configurations|process_queue_configurations|process_topic_configurations|remove_object_lock_metadata_for_copy|remove_object_lock_retention_metadata|validate_bucket_object_lock_enabled|validate_list_object_unordered_with_delimiter)' \
|
||||
-n -U --with-filename -- "${ECFS_TEST_GUARD_SOURCES[@]:1}" ||
|
||||
echo "storage-owner test import rule could not search rustfs/src/storage/legacy_edge/tests"
|
||||
fi
|
||||
}
|
||||
) >"$RUSTFS_STORAGE_OWNER_TEST_ROOT_CONSUMER_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_STORAGE_OWNER_TEST_ROOT_CONSUMER_HITS_FILE" ]]; then
|
||||
@@ -4561,7 +4622,7 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::storage::core_storage_compat' \
|
||||
rustfs/src/storage/mod.rs \
|
||||
rustfs/src/storage/ecfs.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
rustfs/src/storage/ecfs_extend.rs \
|
||||
rustfs/src/storage/access.rs \
|
||||
rustfs/src/storage/head_prefix.rs \
|
||||
@@ -4577,11 +4638,12 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
{
|
||||
rg -n --with-filename 'crate::app::usecase_storage_compat' \
|
||||
rustfs/src/app/*.rs || true
|
||||
rustfs/src/app/*.rs ${BUCKET_USECASE_SPLIT_SOURCES[@]+"${BUCKET_USECASE_SPLIT_SOURCES[@]}"} ${MULTIPART_USECASE_SPLIT_SOURCES[@]+"${MULTIPART_USECASE_SPLIT_SOURCES[@]}"} \
|
||||
${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]+"${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]}"} || true
|
||||
rg -n --with-filename 'crate::admin::router_storage_compat' \
|
||||
rustfs/src/admin/router.rs || true
|
||||
rg -n --with-filename 'crate::storage::core_storage_compat' \
|
||||
rustfs/src/storage/ecfs_test.rs || true
|
||||
"${ECFS_TEST_GUARD_SOURCES[@]}" || true
|
||||
}
|
||||
) >"$RUSTFS_LOCAL_COMPAT_RELATIVE_CONSUMER_HITS_FILE"
|
||||
|
||||
|
||||
@@ -22,7 +22,8 @@ classify_source_layer() {
|
||||
printf 'composition'
|
||||
elif [[ "$file" == rustfs/src/app/* ]]; then
|
||||
printf 'app'
|
||||
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
|
||||
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] ||
|
||||
[[ "$file" == rustfs/src/storage/legacy_edge/* ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
|
||||
[[ "$file" == rustfs/src/storage/s3_compat/* ]]; then
|
||||
printf 'interface'
|
||||
elif [[ "$file" == rustfs/src/* ]]; then
|
||||
@@ -55,6 +56,7 @@ classify_target_layer() {
|
||||
storage)
|
||||
storage_path="${path#storage::}"
|
||||
if [[ "$storage_path" == "ecfs" ]] || [[ "$storage_path" == ecfs::* ]] ||
|
||||
[[ "$storage_path" == "legacy_edge" ]] || [[ "$storage_path" == legacy_edge::* ]] ||
|
||||
[[ "$storage_path" == "s3_api" ]] || [[ "$storage_path" == s3_api::* ]] ||
|
||||
[[ "$storage_path" == "s3_compat" ]] || [[ "$storage_path" == s3_compat::* ]]; then
|
||||
printf 'interface'
|
||||
@@ -137,6 +139,60 @@ run_layer_model_self_tests() {
|
||||
assert_dependency_direction 'allowed' "$s3_compat_source" "$app_target"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$s3_compat_target"
|
||||
assert_dependency_direction 'reverse' "$storage_source" "$s3_compat_target"
|
||||
|
||||
# A mechanical split moves ecfs.rs code into storage/legacy_edge/ and the
|
||||
# usecase files' code into same-named directories; each keeps its layer.
|
||||
assert_source_layer 'interface' rustfs/src/storage/ecfs.rs
|
||||
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/bucket_core.rs
|
||||
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/tests/object_meta.rs
|
||||
assert_source_layer 'infra' rustfs/src/storage/legacy_edge.rs
|
||||
assert_source_layer 'infra' rustfs/src/storage/legacy_edge_helper.rs
|
||||
assert_source_layer 'app' rustfs/src/app/bucket_usecase/cors.rs
|
||||
assert_source_layer 'app' rustfs/src/app/multipart_usecase/complete.rs
|
||||
assert_target_layer 'interface' storage::legacy_edge
|
||||
assert_target_layer 'interface' storage::legacy_edge::object_meta
|
||||
assert_target_layer 'infra' storage::legacy_edge_helper
|
||||
assert_dependency_direction 'allowed' "$(classify_source_layer rustfs/src/storage/legacy_edge/bucket_core.rs)" "$app_target"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$(classify_target_layer storage::legacy_edge::bucket_core)"
|
||||
assert_dependency_direction 'reverse' "$storage_source" "$(classify_target_layer storage::legacy_edge)"
|
||||
assert_dedicated_test_file 'yes' rustfs/src/app/lifecycle_transition_api_test.rs
|
||||
assert_dedicated_test_file 'yes' rustfs/src/app/lifecycle_transition_api_test/put.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/app/lifecycle_transition_api_test_support.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/app/lifecycle_transition_api.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/app/bucket_usecase/cors.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/storage/legacy_edge/bucket_core.rs
|
||||
}
|
||||
|
||||
assert_source_layer() {
|
||||
local expected="$1" file="$2" actual
|
||||
|
||||
actual="$(classify_source_layer "$file")"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
printf 'Layer dependency guard self-test failed: %s classified as %s (expected %s)\n' "$file" "$actual" "$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_target_layer() {
|
||||
local expected="$1" path="$2" actual
|
||||
|
||||
actual="$(classify_target_layer "$path")"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
printf 'Layer dependency guard self-test failed: crate::%s classified as %s (expected %s)\n' "$path" "$actual" "$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_dedicated_test_file() {
|
||||
local expected="$1" file="$2" actual='no'
|
||||
|
||||
if is_dedicated_test_file "$file"; then
|
||||
actual='yes'
|
||||
fi
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
printf 'Layer dependency guard self-test failed: %s dedicated test file = %s (expected %s)\n' "$file" "$actual" "$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
normalize_import_group_item() {
|
||||
@@ -251,11 +307,20 @@ normalize_import_path() {
|
||||
printf '%s\n' "$path"
|
||||
}
|
||||
|
||||
# The guard is file-scoped: dedicated test modules are excluded, while inline
|
||||
# #[cfg(test)] imports remain subject to the source file's layer. The
|
||||
# lifecycle_transition_api_test/ directory is where a mechanical split of
|
||||
# lifecycle_transition_api_test.rs moves its tests (rustfs/backlog#2765).
|
||||
is_dedicated_test_file() {
|
||||
local file="$1"
|
||||
|
||||
[[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]] ||
|
||||
[[ "$file" == rustfs/src/app/lifecycle_transition_api_test/* ]]
|
||||
}
|
||||
|
||||
emit_crate_use_statements() {
|
||||
(cd "$ROOT_DIR" && rg --files -g '*.rs' rustfs/src | while IFS= read -r file; do
|
||||
# The guard is file-scoped: dedicated test modules are excluded, while
|
||||
# inline #[cfg(test)] imports remain subject to the source file's layer.
|
||||
if [[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]]; then
|
||||
if is_dedicated_test_file "$file"; then
|
||||
continue
|
||||
fi
|
||||
perl -0777 -ne '
|
||||
|
||||
@@ -1094,11 +1094,29 @@ trace_hot_spans=(
|
||||
"rustfs/src/app/object:execute_get_object"
|
||||
)
|
||||
|
||||
# A hot span in a file that is due for a mechanical split (rustfs/backlog#2765)
|
||||
# is also looked up in the directory the split moves its functions into; the
|
||||
# named file itself stays required.
|
||||
# shellcheck source=scripts/guard_source_set.sh
|
||||
source "${repo_root}/scripts/guard_source_set.sh"
|
||||
|
||||
for hot_span in "${trace_hot_spans[@]}"; do
|
||||
file="${hot_span%%:*}"
|
||||
function="${hot_span##*:}"
|
||||
hot_span_sources=("$file")
|
||||
split_dir=""
|
||||
case "$file" in
|
||||
rustfs/src/app/bucket_usecase.rs) split_dir="rustfs/src/app/bucket_usecase" ;;
|
||||
esac
|
||||
if [[ -n "$split_dir" ]]; then
|
||||
if ! guard_source_set "$repo_root" "$file" "$split_dir"; then
|
||||
echo "❌ logging guardrail violation: $file::$function guard source is missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
hot_span_sources=("${GUARD_SOURCES[@]}")
|
||||
fi
|
||||
trace_span_pattern="#\\[(tracing::)?instrument\\([^]]*level = \\\"trace\\\"[^]]*\\)\\]([[:space:]]+#\\[[^]]+\\])*[[:space:]]+(pub(\\([^)]*\\))?[[:space:]]+)?(super[[:space:]]+)?async fn ${function}\\b"
|
||||
if ! rg -U "$trace_span_pattern" "$file" >/dev/null; then
|
||||
if ! rg -U "$trace_span_pattern" "${hot_span_sources[@]}" >/dev/null; then
|
||||
echo "❌ logging guardrail violation: $file::$function must remain TRACE-only" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -1,442 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Guard: tests outside an operation's owning family call its use-case only through the test seam.
|
||||
|
||||
Preparatory refactor P4 of the s3s-to-gateway migration. Each migration batch
|
||||
changes the use-case signatures of its own operations. Tests owned by another
|
||||
family that call those entry points as an oracle would break and pull a second
|
||||
batch into the same file, so they call the `#[cfg(test)]` seams in
|
||||
rustfs/src/app/test_seams/ instead; the owning batch then edits one seam body.
|
||||
|
||||
For every entry point in SEAMS this guard fails when
|
||||
|
||||
* a direct method call `.execute_<op>(` appears in test code (a module or
|
||||
item under `#[cfg(test)]`, a file whose module declaration or an ancestor's
|
||||
is under `#[cfg(test)]`, or any file under rustfs/tests/ or rustfs/benches/)
|
||||
in a file that is neither the seam nor one of the operation's owner files;
|
||||
* the seam file is missing, or no longer calls the entry point exactly once;
|
||||
* an owner file listed below does not exist (a stale table must not widen
|
||||
what passes);
|
||||
* a file holding a direct call cannot be placed in the module tree;
|
||||
* no Rust file is found at all.
|
||||
|
||||
Production calls (the legacy S3 trait, admin and metadata routes) are not test
|
||||
code and are not judged here. Comments and string literals are ignored.
|
||||
|
||||
Usage:
|
||||
scripts/check_test_seams.py
|
||||
scripts/check_test_seams.py --self-test
|
||||
|
||||
Exit status: 0 clean, 1 violations or missing inputs, 2 usage error.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
SCAN_ROOT = "rustfs"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Seam:
|
||||
batch: str
|
||||
seam_file: str
|
||||
owners: tuple[str, ...] # files that own the operation; each must exist
|
||||
owner_dirs: tuple[str, ...] = () # directories a split of an owner file may create
|
||||
|
||||
|
||||
SEAMS = {
|
||||
"execute_get_object": Seam(
|
||||
batch="C",
|
||||
seam_file="rustfs/src/app/test_seams/get_object.rs",
|
||||
owners=(
|
||||
"rustfs/src/app/object/get.rs",
|
||||
"rustfs/src/app/object/head.rs",
|
||||
"rustfs/src/app/object/conditional_read_bench.rs",
|
||||
"rustfs/src/app/object/on_demand_migration_put.rs",
|
||||
),
|
||||
),
|
||||
"execute_head_object": Seam(
|
||||
batch="C",
|
||||
seam_file="rustfs/src/app/test_seams/head_object.rs",
|
||||
owners=(
|
||||
"rustfs/src/app/object/get.rs",
|
||||
"rustfs/src/app/object/head.rs",
|
||||
),
|
||||
),
|
||||
"execute_list_objects_v2": Seam(
|
||||
batch="B2",
|
||||
seam_file="rustfs/src/app/test_seams/list_objects_v2.rs",
|
||||
owners=("rustfs/src/app/bucket_usecase.rs",),
|
||||
# P2 splits bucket_usecase.rs into bucket_usecase/*.rs; narrow this to the listing file once it exists.
|
||||
owner_dirs=("rustfs/src/app/bucket_usecase/",),
|
||||
),
|
||||
"execute_put_object": Seam(
|
||||
batch="D",
|
||||
seam_file="rustfs/src/app/test_seams/put_object.rs",
|
||||
owners=(
|
||||
"rustfs/src/app/object/put.rs",
|
||||
"rustfs/src/app/lifecycle_transition_api_test/put.rs",
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
CFG_TEST = re.compile(r"#!?\[\s*cfg\s*\(\s*(?:all\s*\(\s*)?test\b")
|
||||
TEST_ATTR = re.compile(r"#\[\s*(?:tokio::)?test\b")
|
||||
CALL = re.compile(r"\.\s*(execute_get_object|execute_head_object|execute_list_objects_v2|execute_put_object)\s*\(")
|
||||
CHAR_LITERAL = re.compile(r"'(?:\\(?:[nrt\\0'\"]|x[0-9a-fA-F]{2}|u\{[0-9a-fA-F_]{1,6}\})|[^\\'\n])'")
|
||||
RAW_STRING = re.compile(r'b?r(#*)"')
|
||||
|
||||
|
||||
class InputError(Exception):
|
||||
"""A required input is missing or cannot be interpreted."""
|
||||
|
||||
|
||||
def code_mask(text: str) -> bytearray:
|
||||
"""1 for every character that is Rust code, 0 inside comments and literals."""
|
||||
mask = bytearray(len(text))
|
||||
i, n = 0, len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if text.startswith("//", i):
|
||||
j = text.find("\n", i)
|
||||
i = n if j < 0 else j
|
||||
continue
|
||||
if text.startswith("/*", i):
|
||||
depth, j = 1, i + 2
|
||||
while j < n and depth:
|
||||
if text.startswith("/*", j):
|
||||
depth, j = depth + 1, j + 2
|
||||
elif text.startswith("*/", j):
|
||||
depth, j = depth - 1, j + 2
|
||||
else:
|
||||
j += 1
|
||||
i = j
|
||||
continue
|
||||
raw = RAW_STRING.match(text, i)
|
||||
if raw and (i == 0 or not (text[i - 1].isalnum() or text[i - 1] == "_")):
|
||||
end = text.find('"' + raw.group(1), raw.end())
|
||||
i = n if end < 0 else end + 1 + len(raw.group(1))
|
||||
continue
|
||||
if c == '"' or (text.startswith('b"', i) and (i == 0 or not text[i - 1].isalnum())):
|
||||
j = i + (2 if c == "b" else 1)
|
||||
while j < n and text[j] != '"':
|
||||
j += 2 if text[j] == "\\" else 1
|
||||
i = j + 1
|
||||
continue
|
||||
if c == "'":
|
||||
char = CHAR_LITERAL.match(text, i)
|
||||
if char:
|
||||
i = char.end()
|
||||
continue
|
||||
mask[i] = 1
|
||||
i += 1
|
||||
return mask
|
||||
|
||||
|
||||
def test_regions(text: str, mask: bytearray) -> list[tuple[int, int]]:
|
||||
"""Spans of items under `#[cfg(test)]` or `#[test]`/`#[tokio::test]` (the whole file for `#![cfg(test)]`)."""
|
||||
regions = []
|
||||
for match in [*CFG_TEST.finditer(text), *TEST_ATTR.finditer(text)]:
|
||||
if not mask[match.start()]:
|
||||
continue
|
||||
if text[match.start() + 1] == "!":
|
||||
return [(0, len(text))]
|
||||
regions.append((match.start(), _item_end(text, mask, match.start())))
|
||||
return regions
|
||||
|
||||
|
||||
def _item_end(text: str, mask: bytearray, attr_start: int) -> int:
|
||||
"""End offset of the item an attribute at attr_start applies to."""
|
||||
depth, i, n = 0, attr_start, len(text)
|
||||
in_attr = True
|
||||
while i < n:
|
||||
if mask[i]:
|
||||
ch = text[i]
|
||||
if ch in "([{":
|
||||
if ch == "{" and depth == 0 and not in_attr:
|
||||
return _matching(text, mask, i) + 1
|
||||
depth += 1
|
||||
elif ch in ")]}":
|
||||
depth -= 1
|
||||
if depth == 0 and ch == "]":
|
||||
in_attr = False
|
||||
elif ch == ";" and depth == 0 and not in_attr:
|
||||
return i + 1
|
||||
elif ch == "#" and depth == 0:
|
||||
in_attr = True
|
||||
i += 1
|
||||
raise InputError(f"unterminated item after attribute at offset {attr_start}")
|
||||
|
||||
|
||||
def _matching(text: str, mask: bytearray, open_pos: int) -> int:
|
||||
depth = 0
|
||||
for i in range(open_pos, len(text)):
|
||||
if mask[i]:
|
||||
depth += text[i] in "([{"
|
||||
depth -= text[i] in ")]}"
|
||||
if depth == 0:
|
||||
return i
|
||||
raise InputError(f"unbalanced brace at offset {open_pos}")
|
||||
|
||||
|
||||
def module_declaration(root: Path, path: str) -> tuple[list[str], str] | None:
|
||||
"""(candidate parent files, module name) for a file below src/, or None for a crate root."""
|
||||
parts = Path(path)
|
||||
if parts.name in ("lib.rs", "main.rs") and parts.parent.name == "src":
|
||||
return None
|
||||
if parts.name == "mod.rs":
|
||||
name, directory = parts.parent.name, parts.parent.parent
|
||||
else:
|
||||
name, directory = parts.stem, parts.parent
|
||||
if directory.name == "bin" and parts.name != "mod.rs":
|
||||
return None
|
||||
candidates = [directory.with_suffix(".rs"), directory / "mod.rs"]
|
||||
if directory.name == "src":
|
||||
candidates = [directory / "lib.rs", directory / "main.rs"]
|
||||
return [str(parent) for parent in candidates if (root / parent).is_file()], name
|
||||
|
||||
|
||||
def is_test_only_file(root: Path, path: str, cache: dict[str, bool]) -> bool:
|
||||
if path in cache:
|
||||
return cache[path]
|
||||
parts = Path(path).parts
|
||||
if len(parts) > 1 and parts[1] in ("tests", "benches"):
|
||||
cache[path] = True
|
||||
return True
|
||||
text = (root / path).read_text()
|
||||
mask = code_mask(text)
|
||||
if any(text[m.start() + 1] == "!" for m in CFG_TEST.finditer(text) if mask[m.start()]):
|
||||
cache[path] = True
|
||||
return True
|
||||
declaration = module_declaration(root, path)
|
||||
if declaration is None:
|
||||
if Path(path).name in ("lib.rs", "main.rs") or Path(path).parent.name == "bin":
|
||||
cache[path] = False
|
||||
return False
|
||||
raise InputError(f"{path}: cannot find the file that declares this module")
|
||||
parents, name = declaration
|
||||
declarations = []
|
||||
for parent in parents:
|
||||
parent_text = (root / parent).read_text()
|
||||
parent_mask = code_mask(parent_text)
|
||||
pattern = rf"(?m)^[ \t]*(?:pub(?:\([^)\n]*\))?[ \t]+)?mod[ \t]+{re.escape(name)}[ \t]*;"
|
||||
for m in re.finditer(pattern, parent_text):
|
||||
if parent_mask[m.start() + len(m.group()) - len(m.group().lstrip())]:
|
||||
declarations.append((parent, parent_text, parent_mask, m.start()))
|
||||
if len(declarations) != 1:
|
||||
raise InputError(f"{path}: expected one `mod {name};` in {' or '.join(parents) or 'a parent file'}, found {len(declarations)}")
|
||||
parent, parent_text, parent_mask, offset = declarations[0]
|
||||
cfg_test = any(start <= offset < end for start, end in test_regions(parent_text, parent_mask))
|
||||
result = cfg_test or is_test_only_file(root, parent, cache)
|
||||
cache[path] = result
|
||||
return result
|
||||
|
||||
|
||||
def rust_files(root: Path) -> list[str]:
|
||||
listed = subprocess.run(
|
||||
["git", "ls-files", "--cached", "--others", "--exclude-standard", "--", f"{SCAN_ROOT}/*.rs"],
|
||||
cwd=root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if listed.returncode != 0:
|
||||
raise InputError(f"git ls-files failed: {listed.stderr.strip()}")
|
||||
files = sorted(line for line in listed.stdout.splitlines() if line and (root / line).is_file())
|
||||
if not files:
|
||||
raise InputError(f"no Rust files found under {SCAN_ROOT}/")
|
||||
return files
|
||||
|
||||
|
||||
def check(root: Path, files: list[str] | None = None) -> list[str]:
|
||||
errors: list[str] = []
|
||||
for method, seam in SEAMS.items():
|
||||
seam_path = root / seam.seam_file
|
||||
if not seam_path.is_file():
|
||||
errors.append(f"{seam.seam_file}: seam for {method} is missing")
|
||||
else:
|
||||
text = seam_path.read_text()
|
||||
mask = code_mask(text)
|
||||
calls = [m for m in CALL.finditer(text) if mask[m.start()] and m.group(1) == method]
|
||||
if len(calls) != 1:
|
||||
errors.append(f"{seam.seam_file}: must call {method} exactly once, found {len(calls)}")
|
||||
for owner in seam.owners:
|
||||
if not (root / owner).is_file():
|
||||
errors.append(f"{owner}: owner file of {method} is missing; update SEAMS in {Path(__file__).name}")
|
||||
files = rust_files(root) if files is None else files
|
||||
cache: dict[str, bool] = {}
|
||||
for path in files:
|
||||
text = (root / path).read_text()
|
||||
mask = code_mask(text)
|
||||
calls = [m for m in CALL.finditer(text) if mask[m.start()]]
|
||||
if not calls:
|
||||
continue
|
||||
whole_file = is_test_only_file(root, path, cache)
|
||||
regions = test_regions(text, mask)
|
||||
for match in calls:
|
||||
method = match.group(1)
|
||||
seam = SEAMS[method]
|
||||
if path == seam.seam_file or path in seam.owners or path.startswith(seam.owner_dirs or ("\0",)):
|
||||
continue
|
||||
if whole_file or any(start <= match.start() < end for start, end in regions):
|
||||
line = text.count("\n", 0, match.start()) + 1
|
||||
errors.append(
|
||||
f"{path}:{line}: test calls {method} directly; batch {seam.batch} owns it, "
|
||||
f"call crate::app::test_seams::{method.removeprefix('execute_')} instead"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if argv == ["--self-test"]:
|
||||
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTest)
|
||||
return 0 if unittest.TextTestRunner(verbosity=1).run(suite).wasSuccessful() else 1
|
||||
if argv:
|
||||
print(__doc__.split("Usage:", 1)[1].split("\n\n", 1)[0], file=sys.stderr)
|
||||
return 2
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
try:
|
||||
errors = check(root)
|
||||
except InputError as error:
|
||||
print(f"test seam guard: {error}", file=sys.stderr)
|
||||
return 1
|
||||
for error in errors:
|
||||
print(error, file=sys.stderr)
|
||||
if errors:
|
||||
print(f"test seam guard failed: {len(errors)} finding(s)", file=sys.stderr)
|
||||
return 1
|
||||
print("test seam guard passed")
|
||||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Self-test
|
||||
|
||||
|
||||
SEAM_BODY = {
|
||||
"execute_get_object": "pub(crate) fn get_object(u: &U, r: R) -> F { u.execute_get_object(r) }\n",
|
||||
"execute_head_object": "pub(crate) fn head_object(u: &U, r: R) -> F { u.execute_head_object(r) }\n",
|
||||
"execute_list_objects_v2": "pub(crate) fn list_objects_v2(u: &U, r: R) -> F { u.execute_list_objects_v2(r) }\n",
|
||||
"execute_put_object": "pub(crate) fn put_object(u: &U, f: &Fs, r: R) -> F { u.execute_put_object(f, r) }\n",
|
||||
}
|
||||
|
||||
|
||||
class SelfTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.tmp.name)
|
||||
self.files: dict[str, str] = {}
|
||||
self.write("rustfs/src/main.rs", "mod app;\n")
|
||||
self.write(
|
||||
"rustfs/src/app/mod.rs",
|
||||
"mod object;\nmod bucket_usecase;\nmod lifecycle_transition_api_test;\nmod storage;\n"
|
||||
"#[cfg(test)]\nmod test_seams;\n#[cfg(test)]\nmod gating_test;\n",
|
||||
)
|
||||
self.write("rustfs/src/app/object/mod.rs", "mod get;\nmod head;\nmod put;\n#[cfg(test)]\nmod conditional_read_bench;\nmod on_demand_migration_put;\n")
|
||||
self.write("rustfs/src/app/test_seams/mod.rs", "mod get_object;\nmod head_object;\nmod list_objects_v2;\nmod put_object;\n")
|
||||
for method, seam in SEAMS.items():
|
||||
self.write(seam.seam_file, SEAM_BODY[method])
|
||||
for owner in seam.owners:
|
||||
if owner not in self.files:
|
||||
self.write(owner, "fn owner() {}\n")
|
||||
self.write("rustfs/src/app/lifecycle_transition_api_test/mod.rs", "mod put;\nmod delete;\n")
|
||||
self.write("rustfs/src/app/storage.rs", "fn prod(u: &U) { u.execute_get_object(r); }\n")
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def write(self, path: str, text: str):
|
||||
target = self.root / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(text)
|
||||
self.files[path] = text
|
||||
|
||||
def run_check(self) -> list[str]:
|
||||
return check(self.root, sorted(p for p in self.files if p.endswith(".rs")))
|
||||
|
||||
# -- negative cases: each must fail --
|
||||
|
||||
def test_direct_call_in_cfg_test_module_of_non_owner_fails(self):
|
||||
self.write("rustfs/src/app/storage.rs", "fn prod() {}\n#[cfg(test)]\nmod tests {\n fn t() { u.execute_get_object(r); }\n}\n")
|
||||
self.assertEqual(len(self.run_check()), 1)
|
||||
|
||||
def test_direct_call_split_across_lines_fails(self):
|
||||
self.write("rustfs/src/app/storage.rs", "#[cfg(test)]\nmod tests {\n fn t() { usecase\n .execute_head_object(r); }\n}\n")
|
||||
self.assertRegex(self.run_check()[0], r"storage\.rs:4: test calls execute_head_object directly")
|
||||
|
||||
def test_direct_call_in_cfg_test_helper_fn_fails(self):
|
||||
self.write("rustfs/src/app/storage.rs", "#[cfg(test)]\n#[allow(dead_code)]\nfn helper() { u.execute_list_objects_v2(r); }\n")
|
||||
self.assertEqual(len(self.run_check()), 1)
|
||||
|
||||
def test_file_declared_under_cfg_test_fails(self):
|
||||
self.write("rustfs/src/app/gating_test.rs", "fn t() { u.execute_put_object(&fs, r); }\n")
|
||||
self.assertEqual(len(self.run_check()), 1)
|
||||
|
||||
def test_file_whose_ancestor_is_cfg_test_fails(self):
|
||||
self.write("rustfs/src/app/test_seams/extra.rs", "fn t() { u.execute_get_object(r); }\n")
|
||||
self.write("rustfs/src/app/test_seams/mod.rs", self.files["rustfs/src/app/test_seams/mod.rs"] + "mod extra;\n")
|
||||
self.assertEqual(len(self.run_check()), 1)
|
||||
|
||||
def test_put_from_non_owner_lifecycle_file_fails(self):
|
||||
self.write("rustfs/src/app/lifecycle_transition_api_test/delete.rs", "#[test]\nfn t() { u.execute_put_object(&fs, r); }\n")
|
||||
self.write("rustfs/src/app/mod.rs", self.files["rustfs/src/app/mod.rs"].replace("mod lifecycle", "#[cfg(test)]\nmod lifecycle"))
|
||||
self.assertEqual(len(self.run_check()), 1)
|
||||
|
||||
def test_direct_call_in_test_fn_without_cfg_test_fails(self):
|
||||
self.write("rustfs/src/app/storage.rs", "#[tokio::test]\nasync fn t() { u.execute_get_object(r).await; }\n")
|
||||
self.assertEqual(len(self.run_check()), 1)
|
||||
|
||||
def test_integration_test_file_fails(self):
|
||||
self.write("rustfs/tests/oracle.rs", "fn t() { u.execute_get_object(r); }\n")
|
||||
self.assertEqual(len(self.run_check()), 1)
|
||||
|
||||
def test_missing_seam_fails(self):
|
||||
(self.root / SEAMS["execute_get_object"].seam_file).unlink()
|
||||
del self.files[SEAMS["execute_get_object"].seam_file]
|
||||
self.assertRegex(" ".join(self.run_check()), "seam for execute_get_object is missing")
|
||||
|
||||
def test_seam_that_stopped_wrapping_fails(self):
|
||||
self.write(SEAMS["execute_put_object"].seam_file, "pub(crate) fn put_object() { todo!() }\n")
|
||||
self.assertRegex(" ".join(self.run_check()), "must call execute_put_object exactly once, found 0")
|
||||
|
||||
def test_missing_owner_file_fails(self):
|
||||
owner = SEAMS["execute_put_object"].owners[0]
|
||||
(self.root / owner).unlink()
|
||||
del self.files[owner]
|
||||
self.assertRegex(" ".join(self.run_check()), "owner file of execute_put_object is missing")
|
||||
|
||||
def test_undeclared_file_with_direct_call_fails_loudly(self):
|
||||
self.write("rustfs/src/app/orphan.rs", "fn t() { u.execute_get_object(r); }\n")
|
||||
with self.assertRaisesRegex(InputError, "expected one `mod orphan;`"):
|
||||
self.run_check()
|
||||
|
||||
def test_no_rust_files_fails(self):
|
||||
empty = Path(self.tmp.name) / "empty"
|
||||
empty.mkdir()
|
||||
subprocess.run(["git", "init", "-q"], cwd=empty, check=True)
|
||||
with self.assertRaisesRegex(InputError, "no Rust files found"):
|
||||
rust_files(empty)
|
||||
|
||||
# -- positive cases: each must pass --
|
||||
|
||||
def test_seam_call_in_non_owner_test_passes(self):
|
||||
self.write("rustfs/src/app/storage.rs", "#[cfg(test)]\nmod tests {\n fn t() { crate::app::test_seams::get_object(&u, r); }\n}\n")
|
||||
self.assertEqual(self.run_check(), [])
|
||||
|
||||
def test_owner_test_and_production_calls_pass(self):
|
||||
self.write("rustfs/src/app/object/put.rs", "fn prod() {}\n#[cfg(test)]\nmod tests {\n fn t() { u.execute_put_object(&fs, r); }\n}\n")
|
||||
self.assertEqual(self.run_check(), [])
|
||||
|
||||
def test_calls_in_comments_and_strings_are_ignored(self):
|
||||
self.write("rustfs/src/app/storage.rs", '#[cfg(test)]\nmod tests {\n // u.execute_get_object(r)\n const S: &str = "u.execute_get_object(r)";\n}\n')
|
||||
self.assertEqual(self.run_check(), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,624 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Split rustfs/src/app/lifecycle_transition_api_test.rs into one file per operation owner.
|
||||
|
||||
Preparatory refactor P4 for the s3s-to-gateway migration. Nine batches each
|
||||
migrate one family of S3 operations and change that family's use-case
|
||||
signatures. While every lifecycle integration test lives in one 3,700-line
|
||||
file, two batches edit the same file. This codemod moves every top-level item
|
||||
of that file, verbatim with its attributes and comments, into one of the
|
||||
module files below, chosen by the explicit ASSIGNMENT table:
|
||||
|
||||
lifecycle_transition_api_test.rs module root: license header, the
|
||||
original `use` block, module list
|
||||
lifecycle_transition_api_test/
|
||||
common.rs shared fixtures (no owner; frozen during batches)
|
||||
transition.rs ILM transition-engine tests that call no S3 use-case
|
||||
lifecycle_config.rs PutBucketLifecycleConfiguration (batch A3)
|
||||
object_lock.rs PutObjectRetention / PutObjectLegalHold (batch B1)
|
||||
put.rs PutObject (batch D)
|
||||
copy.rs CopyObject, including tests that mix put and copy (batch E)
|
||||
delete.rs DeleteObject / DeleteObjects (batch E)
|
||||
restore.rs RestoreObject (batch E)
|
||||
multipart.rs CompleteMultipartUpload (batch E)
|
||||
|
||||
Items are moved verbatim. The only rewrites, applied to moved items and
|
||||
nowhere else, are the ones a move out of the parent module requires:
|
||||
|
||||
* `super::` becomes `super::super::` (the items now live one module deeper);
|
||||
* `s3s::S3ErrorCode` becomes `S3ErrorCode`, imported once in the module root
|
||||
from `crate::app::storage_api::s3`, and a function-local `use s3s::S3;`
|
||||
becomes `use crate::app::storage_api::s3::S3;`, so no new file names s3s
|
||||
(scripts/check_s3s_footprint.sh counts files);
|
||||
* items placed in common.rs that a sibling file uses gain `pub(super)`;
|
||||
* a call to a test seam's use-case entry point (`execute_get_object`,
|
||||
`execute_head_object`, `execute_list_objects_v2`, `execute_put_object`) in
|
||||
a file that does not own that operation becomes a call to the seam in
|
||||
`crate::app::test_seams` with the same arguments (scripts/check_test_seams.py).
|
||||
|
||||
Every non-blank line of the input lands in exactly one output file. The run
|
||||
fails (exit 1) and writes nothing when an item is missing from ASSIGNMENT, an
|
||||
ASSIGNMENT entry names no item, an item would be moved twice, a moved file
|
||||
still names s3s, or a cross-owner seam call cannot be rewritten.
|
||||
|
||||
Usage:
|
||||
scripts/codemods/split_lifecycle_transition_api_test.py [--no-fmt]
|
||||
scripts/codemods/split_lifecycle_transition_api_test.py --self-test
|
||||
|
||||
The input must be the unsplit file. A second run over the split tree finds the
|
||||
module root instead and exits 1 without writing; to regenerate, restore the
|
||||
unsplit file (for example `git show <rev>:rustfs/src/app/lifecycle_transition_api_test.rs`),
|
||||
re-run, and the output is byte-identical to the committed split. Unless
|
||||
--no-fmt is given, `rustfmt --edition 2024` formats the written files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
SOURCE = Path("rustfs/src/app/lifecycle_transition_api_test.rs")
|
||||
TARGET_DIR = Path("rustfs/src/app/lifecycle_transition_api_test")
|
||||
|
||||
# Module files in the order the root declares them, with a one-line owner note.
|
||||
MODULES = {
|
||||
"common": "Shared fixtures for the lifecycle integration tests; owned by no operation batch.",
|
||||
"copy": "CopyObject tests, including tests that also PUT (batch E owns CopyObject).",
|
||||
"delete": "DeleteObject and DeleteObjects tests (batch E owns both).",
|
||||
"lifecycle_config": "PutBucketLifecycleConfiguration tests (batch A3 owns the lifecycle operations).",
|
||||
"multipart": "CompleteMultipartUpload tests (batch E owns CompleteMultipartUpload).",
|
||||
"object_lock": "PutObjectRetention and PutObjectLegalHold tests (batch B1 owns both).",
|
||||
"put": "PutObject tests (batch D owns PutObject).",
|
||||
"restore": "RestoreObject tests (batch E owns RestoreObject).",
|
||||
"transition": "ILM transition-engine tests that call no S3 use-case; owned by no operation batch.",
|
||||
}
|
||||
|
||||
# Every top-level item that is not a `use` declaration, by name.
|
||||
ASSIGNMENT = {
|
||||
# Fixtures. A fixture used by one owner's tests moves with them; common.rs keeps the ones several owners share.
|
||||
"GLOBAL_ENV": "common",
|
||||
"TRANSITION_WAIT_TIMEOUT": "common",
|
||||
"RESTORE_SUSPENDED_WAIT_TIMEOUT": "restore",
|
||||
"ENV_GET_CODEC_STREAMING_ENABLE": "transition",
|
||||
"ENV_GET_CODEC_STREAMING_ROLLOUT": "transition",
|
||||
"ENV_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED": "transition",
|
||||
"ENV_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED": "transition",
|
||||
"ENV_GET_CODEC_STREAMING_MIN_SIZE": "transition",
|
||||
"RUSTFS_EXPECTED_CURRENT_VERSION_ID": "common",
|
||||
"init_tracing": "common",
|
||||
"setup_test_env": "common",
|
||||
"create_test_bucket": "common",
|
||||
"suspend_test_bucket": "restore",
|
||||
"upload_test_object": "common",
|
||||
"transition_uploaded_object_directly": "restore",
|
||||
"set_bucket_lifecycle_transition_with_tier": "common",
|
||||
"set_bucket_object_lock_enabled": "common",
|
||||
"expiration_lifecycle_configuration": "lifecycle_config",
|
||||
"expired_object_delete_marker_lifecycle_configuration": "lifecycle_config",
|
||||
"del_marker_expiration_lifecycle_configuration": "lifecycle_config",
|
||||
"register_mock_tier": "common",
|
||||
"wait_for_transition": "common",
|
||||
"wait_for_restore_completion": "restore",
|
||||
"with_forced_immediate_enqueue_timeout": "common",
|
||||
"with_get_codec_streaming_remote_probe_env": "transition",
|
||||
"wait_for_object_absence": "delete",
|
||||
"wait_for_delete_marker": "common",
|
||||
"build_request": "common",
|
||||
"streaming_blob_from_bytes": "common",
|
||||
"read_object_bytes": "common",
|
||||
"live_object_version_count": "common",
|
||||
"install_malformed_versioning_config": "delete",
|
||||
"install_malformed_replication_config": "delete",
|
||||
"install_delete_replication_config": "common",
|
||||
# ILM transition engine; no S3 use-case entry point.
|
||||
"transition_wait_timeout_reports_metadata_state_and_read_failure": "transition",
|
||||
"get_transitioned_object_uses_remote_codec_fallback_path": "transition",
|
||||
"duplicate_transition_task_skips_already_transitioned_version": "transition",
|
||||
"transition_rejects_stale_etag_before_remote_upload": "transition",
|
||||
"transition_commit_rejects_replaced_source": "transition",
|
||||
"cancelled_transition_waiting_for_prepared_reader_cleans_remote": "transition",
|
||||
"lifecycle_transition_marks_dirty_disks_for_capacity_manager": "transition",
|
||||
"immediate_transition_timeout_eventually_completes_via_compensation": "transition",
|
||||
# PutBucketLifecycleConfiguration (A3).
|
||||
"compensation_driven_delete_marker_still_honors_lifecycle_cleanup": "lifecycle_config",
|
||||
"put_bucket_lifecycle_configuration_expires_existing_objects": "lifecycle_config",
|
||||
"put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_object_lock_bucket": "lifecycle_config",
|
||||
"put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_object_lock_bucket": "lifecycle_config",
|
||||
"put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expiration_on_object_lock_bucket": "lifecycle_config",
|
||||
"put_bucket_lifecycle_configuration_rejects_zero_day_expiration": "lifecycle_config",
|
||||
# PutObjectRetention / PutObjectLegalHold (B1).
|
||||
"object_lock_handlers_schedule_replication": "object_lock",
|
||||
# PutObject (D).
|
||||
"put_object_if_none_match_existing_object_returns_precondition_failed": "put",
|
||||
"put_object_computes_replication_decision_exactly_once": "put",
|
||||
"put_object_extract_computes_replication_decision_per_entry": "put",
|
||||
# CopyObject (E); mixed put+copy tests go here.
|
||||
"copy_object_if_none_match_existing_destination_returns_precondition_failed": "copy",
|
||||
"undo_copy_requires_the_observed_current_version": "copy",
|
||||
"undo_copy_serializes_a_concurrent_put_until_after_commit": "copy",
|
||||
"copy_object_allows_new_version_for_locked_destination_but_blocks_explicit_overwrite": "copy",
|
||||
"concurrent_reverse_copy_object_does_not_deadlock_with_reader_locks": "copy",
|
||||
"put_and_copy_object_transition_immediately_via_usecases": "copy",
|
||||
"compensation_driven_copy_still_completes_transition": "copy",
|
||||
"copy_object_computes_replication_decision_and_strips_stale_status": "copy",
|
||||
# CompleteMultipartUpload (E).
|
||||
"complete_multipart_upload_transitions_immediately_via_usecase": "multipart",
|
||||
"compensation_driven_complete_multipart_upload_still_transitions": "multipart",
|
||||
"compensation_driven_complete_multipart_upload_still_transitions_inner": "multipart",
|
||||
# DeleteObject / DeleteObjects (E).
|
||||
"undo_delete_only_removes_the_current_matching_delete_marker": "delete",
|
||||
"delete_transitioned_object_removes_remote_tier_copy_via_usecase": "delete",
|
||||
"compensation_driven_transition_still_cleans_remote_tier_on_delete": "delete",
|
||||
"compensation_driven_versioned_delete_still_creates_delete_marker": "delete",
|
||||
"delete_objects_resolves_bucket_versioning_once_per_request": "delete",
|
||||
"delete_objects_treats_empty_directory_version_as_absent": "delete",
|
||||
"delete_objects_preserves_explicit_null_directory_version": "delete",
|
||||
"delete_object_versioning_config_failure_leaves_latest_object_intact": "delete",
|
||||
"delete_object_replication_config_failure_leaves_latest_object_intact": "delete",
|
||||
"delete_object_uses_one_replication_config_generation": "delete",
|
||||
"replica_delete_uses_authorized_source_version_and_its_tags": "delete",
|
||||
"force_delete_replication_config_failure_leaves_prefix_intact": "delete",
|
||||
"replica_delete_replication_config_failure_leaves_object_intact": "delete",
|
||||
"delete_objects_replication_config_failure_does_not_call_storage_delete": "delete",
|
||||
"nonexistent_bucket_fails_before_versioning_lookup_on_get_head_delete": "delete",
|
||||
# RestoreObject (E).
|
||||
"restore_object_usecase_reports_ongoing_conflict": "restore",
|
||||
"restore_object_usecase_recovers_cancelled_post_commit_generation": "restore",
|
||||
"restore_object_usecase_recovers_cancelled_post_commit_generation_inner": "restore",
|
||||
"restore_object_usecase_completes_suspended_null_version_in_place": "restore",
|
||||
"restore_object_usecase_completes_suspended_null_version_in_place_inner": "restore",
|
||||
"restore_object_usecase_accepts_exactly_one_of_two_concurrent_restores": "restore",
|
||||
}
|
||||
|
||||
# Seam entry points: method name -> (seam function, module that owns the operation here).
|
||||
# A call in any other module is rewritten to `test_seams::<seam>(&receiver, args...)`.
|
||||
SEAMS = {
|
||||
"execute_get_object": ("get_object", None),
|
||||
"execute_head_object": ("head_object", None),
|
||||
"execute_list_objects_v2": ("list_objects_v2", None),
|
||||
"execute_put_object": ("put_object", "put"),
|
||||
}
|
||||
|
||||
ROOT_EXTRA_USE = "use crate::app::storage_api::s3::S3ErrorCode;\n"
|
||||
S3S_PATH = re.compile(r'(^|[^"A-Za-z0-9_])s3s::')
|
||||
SUPER_PATH = re.compile(r"(?<![A-Za-z0-9_:])super::")
|
||||
ITEM_NAME = re.compile(
|
||||
r"^(?:pub(?:\([^)]*\))?\s+)?(?:(?:async|unsafe|const|extern)\s+)*"
|
||||
r"(?P<kind>fn|static|const|struct|enum|type|trait|mod|impl|macro_rules!|use)\s*(?P<name>[A-Za-z_][A-Za-z0-9_]*)?"
|
||||
)
|
||||
VISIBILITY_TARGET = re.compile(r"^(?P<indent>[ \t]*)(?P<head>(?:async\s+|unsafe\s+)*(?:fn|static|const)\s)", re.M)
|
||||
|
||||
|
||||
class SplitError(Exception):
|
||||
"""The input cannot be split exactly as the tables describe: exit 1."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lexing: positions of code characters, skipping comments and literals.
|
||||
|
||||
|
||||
def code_mask(text: str) -> bytearray:
|
||||
"""1 for every character that is Rust code, 0 inside comments and literals."""
|
||||
mask = bytearray(len(text))
|
||||
i, n = 0, len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if text.startswith("//", i):
|
||||
j = text.find("\n", i)
|
||||
i = n if j < 0 else j
|
||||
continue
|
||||
if text.startswith("/*", i):
|
||||
depth, j = 1, i + 2
|
||||
while j < n and depth:
|
||||
if text.startswith("/*", j):
|
||||
depth, j = depth + 1, j + 2
|
||||
elif text.startswith("*/", j):
|
||||
depth, j = depth - 1, j + 2
|
||||
else:
|
||||
j += 1
|
||||
i = j
|
||||
continue
|
||||
raw = re.compile(r'b?r(#*)"').match(text, i)
|
||||
if raw and (i == 0 or not (text[i - 1].isalnum() or text[i - 1] == "_")):
|
||||
end = text.find('"' + raw.group(1), raw.end())
|
||||
if end < 0:
|
||||
raise SplitError("unterminated raw string")
|
||||
i = end + 1 + len(raw.group(1))
|
||||
continue
|
||||
if c == '"' or (c == "b" and text.startswith('b"', i) and (i == 0 or not text[i - 1].isalnum())):
|
||||
j = i + (2 if c == "b" else 1)
|
||||
while j < n and text[j] != '"':
|
||||
j += 2 if text[j] == "\\" else 1
|
||||
i = j + 1
|
||||
continue
|
||||
if c == "'":
|
||||
char = re.compile(r"'(?:\\(?:[nrt\\0'\"]|x[0-9a-fA-F]{2}|u\{[0-9a-fA-F_]{1,6}\})|[^\\'\n])'").match(text, i)
|
||||
if char:
|
||||
i = char.end()
|
||||
continue
|
||||
mask[i] = 1
|
||||
i += 1
|
||||
return mask
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
name: str
|
||||
kind: str
|
||||
text: str # verbatim, from its first comment/attribute line through its last line, with newline
|
||||
|
||||
|
||||
def split_items(text: str) -> tuple[str, list[Item]]:
|
||||
"""Return (license header, top-level items) and prove every non-blank line is covered once."""
|
||||
mask = code_mask(text)
|
||||
ends: list[int] = []
|
||||
depth = 0
|
||||
for i, ch in enumerate(text):
|
||||
if not mask[i]:
|
||||
continue
|
||||
if ch in "([{":
|
||||
depth += 1
|
||||
elif ch in ")]}":
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
raise SplitError(f"unbalanced closing '{ch}' at offset {i}")
|
||||
if depth == 0 and ch == "}":
|
||||
nxt = next((k for k in range(i + 1, len(text)) if mask[k] and not text[k].isspace()), None)
|
||||
if nxt is None or text[nxt] != ";":
|
||||
ends.append(i)
|
||||
elif ch == ";" and depth == 0:
|
||||
ends.append(i)
|
||||
if depth:
|
||||
raise SplitError("unbalanced braces at end of file")
|
||||
first_code = next(i for i in range(len(text)) if mask[i] and not text[i].isspace())
|
||||
header_end = text.rfind("\n", 0, first_code) + 1
|
||||
header = text[:header_end]
|
||||
# The header is the leading comment block; anything else before the first item belongs to that item.
|
||||
header_lines = header.splitlines(keepends=True)
|
||||
while header_lines and header_lines[-1].strip():
|
||||
header_lines.pop()
|
||||
header = "".join(header_lines)
|
||||
if not header.strip():
|
||||
raise SplitError("the file does not start with a license header")
|
||||
items: list[Item] = []
|
||||
start = len(header)
|
||||
for end in ends:
|
||||
line_end = text.find("\n", end)
|
||||
line_end = len(text) if line_end < 0 else line_end + 1
|
||||
chunk = text[start:line_end]
|
||||
stripped = chunk.lstrip("\n")
|
||||
if not stripped.strip():
|
||||
raise SplitError(f"empty item before offset {end}")
|
||||
items.append(classify(stripped))
|
||||
start = line_end
|
||||
if text[start:].strip():
|
||||
raise SplitError("text after the last item")
|
||||
covered = [line for item in items for line in item.text.splitlines() if line.strip()]
|
||||
covered = [line for line in header.splitlines() if line.strip()] + covered
|
||||
expected = [line for line in text.splitlines() if line.strip()]
|
||||
if covered != expected:
|
||||
raise SplitError("item split does not cover every non-blank input line exactly once")
|
||||
return header, items
|
||||
|
||||
|
||||
def classify(chunk: str) -> Item:
|
||||
mask = code_mask(chunk)
|
||||
depth = 0
|
||||
for line_start, line in _lines_with_offsets(chunk):
|
||||
body = "".join(ch if mask[line_start + k] else " " for k, ch in enumerate(line)).strip()
|
||||
if not body:
|
||||
continue
|
||||
if depth == 0 and not body.startswith("#"):
|
||||
match = ITEM_NAME.match(body)
|
||||
if not match:
|
||||
raise SplitError(f"unrecognised item head: {body[:80]!r}")
|
||||
kind = match.group("kind")
|
||||
return Item(name=match.group("name") or "", kind=kind, text=chunk)
|
||||
depth += body.count("[") - body.count("]")
|
||||
raise SplitError(f"item without a head: {chunk[:80]!r}")
|
||||
|
||||
|
||||
def _lines_with_offsets(text: str):
|
||||
offset = 0
|
||||
for line in text.splitlines(keepends=True):
|
||||
yield offset, line.rstrip("\n")
|
||||
offset += len(line)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rewrites applied to moved items.
|
||||
|
||||
|
||||
def rewrite_paths(text: str) -> str:
|
||||
mask = code_mask(text)
|
||||
out = []
|
||||
last = 0
|
||||
for match in SUPER_PATH.finditer(text):
|
||||
if mask[match.start()]:
|
||||
out.append(text[last : match.start()])
|
||||
out.append("super::super::")
|
||||
last = match.end()
|
||||
out.append(text[last:])
|
||||
text = "".join(out)
|
||||
text = text.replace("use s3s::S3;", "use crate::app::storage_api::s3::S3;")
|
||||
text = re.sub(r"(?<![A-Za-z0-9_:])s3s::S3ErrorCode\b", "S3ErrorCode", text)
|
||||
return text
|
||||
|
||||
|
||||
def make_shared(text: str) -> str:
|
||||
"""Give the item head `pub(super)`; attributes and comments before it are untouched."""
|
||||
mask = code_mask(text)
|
||||
for match in VISIBILITY_TARGET.finditer(text):
|
||||
if mask[match.start("head")] and _depth_at(text, mask, match.start("head")) == 0:
|
||||
pos = match.start("head")
|
||||
return text[:pos] + "pub(super) " + text[pos:]
|
||||
raise SplitError(f"cannot find the head of shared item {text[:80]!r}")
|
||||
|
||||
|
||||
def _depth_at(text: str, mask: bytearray, pos: int) -> int:
|
||||
depth = 0
|
||||
for i in range(pos):
|
||||
if mask[i]:
|
||||
depth += text[i] in "([{"
|
||||
depth -= text[i] in ")]}"
|
||||
return depth
|
||||
|
||||
|
||||
def rewrite_seam_calls(text: str, module: str) -> tuple[str, int]:
|
||||
"""Rewrite `<receiver>.execute_<op>(args)` to `test_seams::<seam>(&<receiver>, args)` outside the owner."""
|
||||
count = 0
|
||||
for method, (seam, owner) in SEAMS.items():
|
||||
if owner == module:
|
||||
continue
|
||||
while True:
|
||||
mask = code_mask(text)
|
||||
match = next((m for m in re.finditer(r"\.\s*" + method + r"\(", text) if mask[m.start()]), None)
|
||||
if match is None:
|
||||
break
|
||||
recv_start = _receiver_start(text, match.start())
|
||||
receiver = text[recv_start : match.start()].rstrip()
|
||||
if not receiver:
|
||||
raise SplitError(f"{module}: call to {method} without a receiver")
|
||||
args_start = match.end()
|
||||
args_end = _matching(text, mask, match.end() - 1)
|
||||
args = text[args_start:args_end]
|
||||
text = text[:recv_start] + f"test_seams::{seam}(&{receiver}, {args})" + text[args_end + 1 :]
|
||||
count += 1
|
||||
return text, count
|
||||
|
||||
|
||||
def _receiver_start(text: str, dot: int) -> int:
|
||||
i = dot
|
||||
while i > 0 and text[i - 1].isspace():
|
||||
i -= 1
|
||||
end = i
|
||||
if end and text[end - 1] == ")":
|
||||
depth = 0
|
||||
while i > 0:
|
||||
i -= 1
|
||||
depth += text[i] == ")"
|
||||
depth -= text[i] == "("
|
||||
if depth == 0:
|
||||
break
|
||||
while i > 0 and (text[i - 1].isalnum() or text[i - 1] in "_:"):
|
||||
i -= 1
|
||||
if i == end or (i > 0 and text[i - 1] == "."):
|
||||
raise SplitError(f"unsupported receiver before offset {dot}: only a name or a path call is rewritten")
|
||||
return i
|
||||
|
||||
|
||||
def _matching(text: str, mask: bytearray, open_pos: int) -> int:
|
||||
depth = 0
|
||||
for i in range(open_pos, len(text)):
|
||||
if not mask[i]:
|
||||
continue
|
||||
depth += text[i] in "([{"
|
||||
depth -= text[i] in ")]}"
|
||||
if depth == 0:
|
||||
return i
|
||||
raise SplitError(f"unbalanced call at offset {open_pos}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assembly.
|
||||
|
||||
|
||||
def build(text: str) -> dict[Path, str]:
|
||||
"""Return the full content of every output file, keyed by repository-relative path."""
|
||||
header, items = split_items(text)
|
||||
license_block = header.rstrip("\n") + "\n"
|
||||
uses = [item for item in items if item.kind == "use"]
|
||||
moved = [item for item in items if item.kind != "use"]
|
||||
if items[: len(uses)] != uses:
|
||||
raise SplitError("a `use` declaration follows another item; the root keeps only the leading use block")
|
||||
names = [item.name for item in moved]
|
||||
duplicates = sorted({name for name in names if names.count(name) > 1})
|
||||
if duplicates:
|
||||
raise SplitError(f"items defined twice: {', '.join(duplicates)}")
|
||||
unassigned = [name for name in names if name not in ASSIGNMENT]
|
||||
stale = sorted(set(ASSIGNMENT) - set(names))
|
||||
if unassigned or stale:
|
||||
raise SplitError(
|
||||
"ASSIGNMENT does not match the input:"
|
||||
+ "".join(f"\n unassigned item: {name}" for name in unassigned)
|
||||
+ "".join(f"\n no such item: {name}" for name in stale)
|
||||
)
|
||||
unknown = sorted({module for module in ASSIGNMENT.values() if module not in MODULES})
|
||||
if unknown:
|
||||
raise SplitError(f"ASSIGNMENT names undeclared modules: {', '.join(unknown)}")
|
||||
|
||||
bodies: dict[str, list[str]] = {module: [] for module in MODULES}
|
||||
for item in moved:
|
||||
module = ASSIGNMENT[item.name]
|
||||
body, _ = rewrite_seam_calls(rewrite_paths(item.text), module)
|
||||
bodies[module].append(body)
|
||||
|
||||
# A common item gains `pub(super)` only when a sibling uses it; the rest stay private to common.rs.
|
||||
sibling_code = _code_only("\n".join(body for module, items in bodies.items() if module != "common" for body in items))
|
||||
common_items = [item for item in moved if ASSIGNMENT[item.name] == "common"]
|
||||
common_names = {item.name for item in common_items if re.search(rf"\b{re.escape(item.name)}\b", sibling_code)}
|
||||
bodies["common"] = [
|
||||
make_shared(body) if item.name in common_names else body for item, body in zip(common_items, bodies["common"])
|
||||
]
|
||||
outputs: dict[Path, str] = {}
|
||||
root_uses = "".join(item.text if item.text.endswith("\n") else item.text + "\n" for item in uses)
|
||||
root = (
|
||||
license_block
|
||||
+ "\n"
|
||||
+ "//! Lifecycle, transition and object-version integration tests, split by the batch that owns each operation.\n"
|
||||
+ "//!\n"
|
||||
+ "//! Generated by scripts/codemods/split_lifecycle_transition_api_test.py; each module file names its owner.\n"
|
||||
+ "\n"
|
||||
+ root_uses
|
||||
+ ROOT_EXTRA_USE
|
||||
+ "\n"
|
||||
+ "".join(f"mod {module};\n" for module in MODULES)
|
||||
)
|
||||
outputs[SOURCE] = root
|
||||
for module, doc in MODULES.items():
|
||||
content = bodies[module]
|
||||
if not content:
|
||||
raise SplitError(f"module {module} receives no item")
|
||||
joined = "\n".join(content)
|
||||
preamble = ["use super::*;\n"]
|
||||
if module != "common" and any(re.search(rf"\b{re.escape(name)}\b", joined) for name in common_names):
|
||||
preamble.append("use super::common::*;\n")
|
||||
if "test_seams::" in joined:
|
||||
preamble.append("use crate::app::test_seams;\n")
|
||||
text_out = license_block + "\n" + f"//! {doc}\n" + "\n" + "".join(preamble) + "\n" + joined
|
||||
leftover = [line for line in text_out.splitlines() if S3S_PATH.search(line)]
|
||||
if leftover:
|
||||
raise SplitError(f"{module}.rs still names s3s: {leftover[0].strip()}")
|
||||
for method, (_, owner) in SEAMS.items():
|
||||
if owner != module and re.search(r"\." + method + r"\(", _code_only(text_out)):
|
||||
raise SplitError(f"{module}.rs still calls {method} directly")
|
||||
outputs[TARGET_DIR / f"{module}.rs"] = text_out
|
||||
return outputs
|
||||
|
||||
|
||||
def _code_only(text: str) -> str:
|
||||
mask = code_mask(text)
|
||||
return "".join(ch if mask[i] or ch == "\n" else " " for i, ch in enumerate(text))
|
||||
|
||||
|
||||
def run(root: Path, fmt: bool) -> int:
|
||||
source = root / SOURCE
|
||||
try:
|
||||
text = source.read_text()
|
||||
except OSError as error:
|
||||
print(f"error: cannot read {SOURCE}: {error}", file=sys.stderr)
|
||||
return 1
|
||||
if re.search(r"(?m)^mod common;$", text):
|
||||
print(f"error: {SOURCE} is already split; restore the unsplit file first", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
outputs = build(text)
|
||||
except SplitError as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 1
|
||||
stray = sorted(p.name for p in (root / TARGET_DIR).glob("*.rs") if TARGET_DIR / p.name not in outputs) if (
|
||||
root / TARGET_DIR
|
||||
).is_dir() else []
|
||||
if stray:
|
||||
print(f"error: {TARGET_DIR} holds files this codemod does not write: {', '.join(stray)}", file=sys.stderr)
|
||||
return 1
|
||||
(root / TARGET_DIR).mkdir(parents=True, exist_ok=True)
|
||||
for path, content in outputs.items():
|
||||
(root / path).write_text(content)
|
||||
if fmt:
|
||||
paths = [str(root / path) for path in outputs]
|
||||
result = subprocess.run(["rustfmt", "--edition", "2024", *paths], cwd=root, check=False)
|
||||
if result.returncode != 0:
|
||||
print("error: rustfmt failed on the written files", file=sys.stderr)
|
||||
return 1
|
||||
for path in outputs:
|
||||
print(f"wrote {path} ({len((root / path).read_text().splitlines())} lines)")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if argv == ["--self-test"]:
|
||||
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTest)
|
||||
return 0 if unittest.TextTestRunner(verbosity=1).run(suite).wasSuccessful() else 1
|
||||
if argv not in ([], ["--no-fmt"]):
|
||||
print(__doc__.split("Usage:", 1)[1].split("\n\n", 1)[0], file=sys.stderr)
|
||||
return 2
|
||||
top = subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True, check=False)
|
||||
if top.returncode != 0:
|
||||
print("error: not inside a git checkout", file=sys.stderr)
|
||||
return 2
|
||||
return run(Path(top.stdout.strip()), fmt=argv != ["--no-fmt"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Self-test
|
||||
|
||||
|
||||
class SelfTest(unittest.TestCase):
|
||||
HEADER = "// Copyright\n// License\n\n"
|
||||
|
||||
def items(self, body: str) -> list[Item]:
|
||||
return split_items(self.HEADER + body)[1]
|
||||
|
||||
def test_items_keep_attributes_comments_and_braces_in_literals(self):
|
||||
body = 'use a::{b, c};\n\n// note\n#[test]\nfn one() {\n let s = "}";\n let c = \'}\';\n}\n\nstatic X: u8 = 1;\n'
|
||||
items = self.items(body)
|
||||
self.assertEqual([(i.kind, i.name) for i in items], [("use", "a"), ("fn", "one"), ("static", "X")])
|
||||
self.assertTrue(items[1].text.startswith("// note\n#[test]\n"))
|
||||
|
||||
def test_raw_strings_and_lifetimes_do_not_confuse_depth(self):
|
||||
items = self.items("fn a<'x>(v: &'x str) {\n let r = r#\"{\"#;\n}\nfn b() {}\n")
|
||||
self.assertEqual([i.name for i in items], ["a", "b"])
|
||||
|
||||
def test_unbalanced_input_is_rejected(self):
|
||||
with self.assertRaises(SplitError):
|
||||
split_items(self.HEADER + "fn a() {\n")
|
||||
|
||||
def test_unassigned_item_is_rejected(self):
|
||||
with self.assertRaisesRegex(SplitError, "unassigned item: not_in_table"):
|
||||
build(self.HEADER + "use x::y;\nfn not_in_table() {}\n")
|
||||
|
||||
def test_super_paths_move_one_level_down_but_not_in_strings(self):
|
||||
source = 'fn a() {\n use super::x::Y;\n let s = "super::z";\n}\n'
|
||||
self.assertEqual(rewrite_paths(source), 'fn a() {\n use super::super::x::Y;\n let s = "super::z";\n}\n')
|
||||
|
||||
def test_s3s_error_code_and_trait_paths_are_rewritten(self):
|
||||
source = "fn a() {\n use s3s::S3;\n f(&s3s::S3ErrorCode::NoSuchKey);\n}\n"
|
||||
self.assertEqual(
|
||||
rewrite_paths(source),
|
||||
"fn a() {\n use crate::app::storage_api::s3::S3;\n f(&S3ErrorCode::NoSuchKey);\n}\n",
|
||||
)
|
||||
|
||||
def test_shared_items_gain_pub_super_after_attributes(self):
|
||||
self.assertEqual(make_shared("#[allow(x)]\nasync fn a() {}\n"), "#[allow(x)]\npub(super) async fn a() {}\n")
|
||||
self.assertEqual(make_shared("static A: u8 = 1;\n"), "pub(super) static A: u8 = 1;\n")
|
||||
|
||||
def test_cross_owner_calls_use_the_seam(self):
|
||||
source = "fn t() {\n Box::pin(usecase\n .execute_put_object(&fs, req(x)))\n}\n"
|
||||
rewritten, count = rewrite_seam_calls(source, "copy")
|
||||
self.assertEqual(count, 1)
|
||||
self.assertEqual(rewritten, "fn t() {\n Box::pin(test_seams::put_object(&usecase, &fs, req(x)))\n}\n")
|
||||
|
||||
def test_owner_calls_stay_direct(self):
|
||||
source = "fn t() {\n usecase.execute_put_object(&fs, req)\n}\n"
|
||||
self.assertEqual(rewrite_seam_calls(source, "put"), (source, 0))
|
||||
|
||||
def test_call_expression_receivers_are_kept_whole(self):
|
||||
source = "fn t() {\n Usecase::with(Some(a)).execute_get_object(r)\n}\n"
|
||||
rewritten, _ = rewrite_seam_calls(source, "delete")
|
||||
self.assertEqual(rewritten, "fn t() {\n test_seams::get_object(&Usecase::with(Some(a)), r)\n}\n")
|
||||
|
||||
def test_already_split_input_is_refused(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / SOURCE
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_text(self.HEADER + "mod common;\n")
|
||||
self.assertEqual(run(Path(tmp), fmt=False), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -13,7 +13,7 @@ OUTPUT="$4"
|
||||
|
||||
case "$ARCH" in
|
||||
amd64|arm64) ;;
|
||||
*) echo "Unsupported rc architecture: '$ARCH' (expected amd64 or arm64; an empty value means the caller's TARGETARCH build argument was not set)" >&2; exit 1 ;;
|
||||
*) echo "Unsupported rc architecture: $ARCH" >&2; exit 1 ;;
|
||||
esac
|
||||
case "$LIBC" in
|
||||
musl) LIBC_SUFFIX="" ;;
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Source-set helper for text guards that must survive a mechanical file split.
|
||||
# Sourced by check_architecture_migration_rules.sh and check_logging_guardrails.sh;
|
||||
# contract tests: scripts/test_guard_source_set.sh.
|
||||
#
|
||||
# A guard that greps one named file stops seeing the code it guards once that
|
||||
# file is split into a sibling directory. guard_source_set resolves the set of
|
||||
# files such a guard is about instead:
|
||||
#
|
||||
# guard_source_set <root> <required-file> [<split-dir>...]
|
||||
#
|
||||
# On success it sets the global array GUARD_SOURCES to <required-file> followed
|
||||
# by every `.rs` file under each <split-dir> that exists, in byte order of path
|
||||
# (LC_ALL=C), all relative to <root>. An absent <split-dir> contributes nothing.
|
||||
# It returns 1 with a message on stderr when <required-file> is not a regular
|
||||
# file (a guard whose input is missing must fail, not pass) or when a
|
||||
# <split-dir> exists but is not a directory. On failure GUARD_SOURCES still
|
||||
# holds <required-file>, so a caller that records the failure and continues
|
||||
# hands the missing path to its search tool rather than nothing.
|
||||
|
||||
guard_source_set() {
|
||||
local root="$1" required="$2" split_dir path listing
|
||||
shift 2
|
||||
GUARD_SOURCES=("$required")
|
||||
|
||||
if [[ ! -f "${root}/${required}" ]]; then
|
||||
printf 'guard source %s is missing\n' "$required" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
for split_dir in "$@"; do
|
||||
if [[ ! -e "${root}/${split_dir}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ ! -d "${root}/${split_dir}" ]]; then
|
||||
printf 'guard split path %s is not a directory\n' "$split_dir" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! listing="$(set -o pipefail; cd "$root" && find "$split_dir" -type f -name '*.rs' | LC_ALL=C sort)"; then
|
||||
printf 'guard split directory %s cannot be listed\n' "$split_dir" >&2
|
||||
return 1
|
||||
fi
|
||||
while IFS= read -r path; do
|
||||
if [[ -n "$path" ]]; then
|
||||
GUARD_SOURCES+=("$path")
|
||||
fi
|
||||
done <<<"$listing"
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# A rule that forbids reaching an owner module through `super::` is only right
|
||||
# at the depth it was written for: in rustfs/src/app/bucket_usecase.rs `super`
|
||||
# is `crate::app`, but in rustfs/src/app/bucket_usecase/cors.rs `super` is the
|
||||
# bucket usecase itself and `crate::app` is `super::super`. module_relative_rg
|
||||
# rewrites a pattern for each file's depth below the owner module directory:
|
||||
#
|
||||
# module_relative_rg <module-dir> <template> [<rg-option>...] -- <file>...
|
||||
#
|
||||
# Every `@UP@` in <template> becomes the `super::` chain that names the owner
|
||||
# module from that file's own scope: one `super::` for <module-dir>/x.rs and
|
||||
# <module-dir>/x/mod.rs, two for <module-dir>/x/y.rs, and so on. A file at
|
||||
# depth 0 (<module-dir>/mod.rs) keeps one `super::`, the form the rule had
|
||||
# before it was depth-aware. Files are searched in groups of equal depth, so
|
||||
# rg's output stays `path:line:text`; rg's own errors (exit status 2) fail the
|
||||
# call, and "no match" (status 1) does not.
|
||||
module_relative_depth() {
|
||||
local module_dir="$1" file="$2" relative slashes depth
|
||||
relative="${file#"$module_dir"/}"
|
||||
slashes="${relative//[!\/]/}"
|
||||
depth=$((${#slashes} + 1))
|
||||
if [[ "${relative##*/}" == mod.rs ]]; then
|
||||
depth=$((depth - 1))
|
||||
fi
|
||||
printf '%s\n' "$depth"
|
||||
}
|
||||
|
||||
module_relative_rg() {
|
||||
local module_dir="$1" template="$2" file depth max_depth=0 up pattern status index count
|
||||
shift 2
|
||||
local -a options=() files=() depths=() group=()
|
||||
while [[ $# -gt 0 && "$1" != "--" ]]; do
|
||||
options+=("$1")
|
||||
shift
|
||||
done
|
||||
if [[ $# -eq 0 ]]; then
|
||||
printf 'module_relative_rg: missing -- before the file list\n' >&2
|
||||
return 2
|
||||
fi
|
||||
shift
|
||||
for file in "$@"; do
|
||||
if [[ "$file" != "$module_dir"/* ]]; then
|
||||
printf 'module_relative_rg: %s is not under %s\n' "$file" "$module_dir" >&2
|
||||
return 2
|
||||
fi
|
||||
depth="$(module_relative_depth "$module_dir" "$file")"
|
||||
files+=("$file")
|
||||
depths+=("$depth")
|
||||
if ((depth > max_depth)); then
|
||||
max_depth=$depth
|
||||
fi
|
||||
done
|
||||
count=${#files[@]}
|
||||
for ((depth = 0; depth <= max_depth; depth++)); do
|
||||
group=()
|
||||
for ((index = 0; index < count; index++)); do
|
||||
if [[ "${depths[index]}" == "$depth" ]]; then
|
||||
group+=("${files[index]}")
|
||||
fi
|
||||
done
|
||||
if [[ ${#group[@]} -eq 0 ]]; then
|
||||
continue
|
||||
fi
|
||||
up="super::"
|
||||
for ((index = 1; index < depth; index++)); do
|
||||
up+="super::"
|
||||
done
|
||||
pattern="${template//@UP@/$up}"
|
||||
status=0
|
||||
rg ${options[@]+"${options[@]}"} "$pattern" "${group[@]}" || status=$?
|
||||
if [[ $status -gt 1 ]]; then
|
||||
return "$status"
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
@@ -1,406 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Manual zip/deb/rpm packaging for one Linux target.
|
||||
# Version rules and deb/rpm metadata come from package_versions.sh.
|
||||
# deb/rpm payloads are built by build_deb.sh and build_rpm.sh.
|
||||
# Dependency strings match scripts/release/deliver.py package().
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
helper="${repo_root}/scripts/release/package_versions.sh"
|
||||
|
||||
die() {
|
||||
printf 'package_manual_artifacts: %s\n' "$1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat <<'EOF' >&2
|
||||
usage: package_manual_artifacts.sh matrix <x86|arm|all>
|
||||
package_manual_artifacts.sh validate --version VER --arch ARCH --libc LIBC
|
||||
package_manual_artifacts.sh package --version VER --arch ARCH --libc LIBC --binary PATH --output DIR [--zip-extra FILE]...
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
require_value() {
|
||||
local flag=$1
|
||||
# An empty value is a version/path error, not a missing argument. Flag-shaped
|
||||
# values mean the next option was consumed by mistake.
|
||||
if [[ $# -lt 2 || ${2:-} == --* ]]; then
|
||||
die "missing value for ${flag}"
|
||||
fi
|
||||
}
|
||||
|
||||
resolve_version() {
|
||||
local version=$1 arch=$2 libc=$3
|
||||
local release_err prerelease_err release_out prerelease_out
|
||||
release_err=$(mktemp)
|
||||
prerelease_err=$(mktemp)
|
||||
if release_out=$(bash "$helper" release "$version" "" "$arch" "$libc" 2>"$release_err"); then
|
||||
printf 'release\n%s\n' "$release_out"
|
||||
rm -f "$release_err" "$prerelease_err"
|
||||
return 0
|
||||
fi
|
||||
if prerelease_out=$(bash "$helper" prerelease "$version" "" "$arch" "$libc" 2>"$prerelease_err"); then
|
||||
printf 'prerelease\n%s\n' "$prerelease_out"
|
||||
rm -f "$release_err" "$prerelease_err"
|
||||
return 0
|
||||
fi
|
||||
printf 'invalid package version: %s\n' "$version" >&2
|
||||
printf 'not a release: %s\n' "$(<"$release_err")" >&2
|
||||
printf 'not a prerelease: %s\n' "$(<"$prerelease_err")" >&2
|
||||
rm -f "$release_err" "$prerelease_err"
|
||||
return 1
|
||||
}
|
||||
|
||||
load_versions() {
|
||||
local resolved line key value
|
||||
resolved=$(resolve_version "$version" "$arch" "$libc") || exit 1
|
||||
build_type=${resolved%%$'\n'*}
|
||||
local rest=${resolved#*$'\n'}
|
||||
deb_version=''
|
||||
rpm_version=''
|
||||
rpm_release=''
|
||||
deb_file=''
|
||||
rpm_file=''
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
[[ -n "$line" ]] || continue
|
||||
key=${line%%=*}
|
||||
value=${line#*=}
|
||||
case "$key" in
|
||||
deb_version) deb_version=$value ;;
|
||||
rpm_version) rpm_version=$value ;;
|
||||
rpm_release) rpm_release=$value ;;
|
||||
deb_file) deb_file=$value ;;
|
||||
rpm_file) rpm_file=$value ;;
|
||||
*) die "unexpected package_versions field: ${key}" ;;
|
||||
esac
|
||||
done <<< "$rest"
|
||||
[[ -n "$deb_version" && -n "$rpm_version" && -n "$rpm_release" && -n "$deb_file" && -n "$rpm_file" ]] ||
|
||||
die "incomplete package version output"
|
||||
local stem=${deb_file%.deb}
|
||||
[[ "$stem" == "${rpm_file%.rpm}" ]] || die "deb and rpm file names use different stems"
|
||||
[[ "$stem" == "rustfs-linux-${arch}-${libc}-v${version}" ]] ||
|
||||
die "package stem does not match rustfs-linux-${arch}-${libc}-v${version}"
|
||||
zip_file="${stem}.zip"
|
||||
case "$build_type" in
|
||||
release | prerelease) ;;
|
||||
*) die "unsupported build type: ${build_type}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
emit_resolved() {
|
||||
printf 'build_type=%s\n' "$build_type"
|
||||
printf 'deb_version=%s\n' "$deb_version"
|
||||
printf 'rpm_version=%s\n' "$rpm_version"
|
||||
printf 'rpm_release=%s\n' "$rpm_release"
|
||||
printf 'deb_file=%s\n' "$deb_file"
|
||||
printf 'rpm_file=%s\n' "$rpm_file"
|
||||
printf 'zip_file=%s\n' "$zip_file"
|
||||
}
|
||||
|
||||
emit_matrix() {
|
||||
local choice=$1
|
||||
command -v python3 >/dev/null 2>&1 || die "python3 is required"
|
||||
python3 - "$choice" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
choice = sys.argv[1]
|
||||
# Linux rows match the release matrix in .github/workflows/build.yml.
|
||||
# arch, libc, and build_os are the packaging names for those same rows.
|
||||
rows = [
|
||||
{"target_id": "linux-x86_64-musl", "os": "sm-standard-4", "target": "x86_64-unknown-linux-musl", "cross": False, "platform": "linux", "rustflags": "", "features": "", "glibc": "", "arch": "x86_64", "libc": "musl", "build_os": "linux-x86_64"},
|
||||
{"target_id": "linux-aarch64-musl", "os": "ubicloud-standard-16-arm", "target": "aarch64-unknown-linux-musl", "cross": False, "platform": "linux", "rustflags": "", "features": "", "glibc": "", "arch": "aarch64", "libc": "musl", "build_os": "linux-aarch64"},
|
||||
{"target_id": "linux-x86_64-gnu", "os": "sm-standard-4", "target": "x86_64-unknown-linux-gnu", "cross": True, "platform": "linux", "rustflags": "", "features": "pyroscope", "glibc": "2.31", "arch": "x86_64", "libc": "gnu", "build_os": "linux-x86_64"},
|
||||
{"target_id": "linux-aarch64-gnu", "os": "ubicloud-standard-16-arm", "target": "aarch64-unknown-linux-gnu", "cross": True, "platform": "linux", "rustflags": "", "features": "pyroscope", "glibc": "2.31", "arch": "aarch64", "libc": "gnu", "build_os": "linux-aarch64"},
|
||||
]
|
||||
if choice == "all":
|
||||
selected = rows
|
||||
elif choice == "x86":
|
||||
selected = [row for row in rows if row["arch"] == "x86_64"]
|
||||
elif choice == "arm":
|
||||
selected = [row for row in rows if row["arch"] == "aarch64"]
|
||||
else:
|
||||
raise SystemExit(
|
||||
"package_manual_artifacts: unsupported architecture "
|
||||
f"(expected x86, arm, or all): {choice}"
|
||||
)
|
||||
if len(selected) == 0:
|
||||
raise SystemExit("package_manual_artifacts: no targets selected")
|
||||
json.dump({"include": selected}, sys.stdout, separators=(",", ":"))
|
||||
sys.stdout.write("\n")
|
||||
PY
|
||||
}
|
||||
|
||||
parse_identity() {
|
||||
version=''
|
||||
arch=''
|
||||
libc=''
|
||||
local saw_version=0 saw_arch=0 saw_libc=0
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--version)
|
||||
require_value "$@"
|
||||
saw_version=1
|
||||
version=$2
|
||||
shift 2
|
||||
;;
|
||||
--arch)
|
||||
require_value "$@"
|
||||
saw_arch=1
|
||||
arch=$2
|
||||
shift 2
|
||||
;;
|
||||
--libc)
|
||||
require_value "$@"
|
||||
saw_libc=1
|
||||
libc=$2
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
die "unknown argument: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
[[ "$saw_version" == 1 && "$saw_arch" == 1 && "$saw_libc" == 1 ]] ||
|
||||
die "validate requires --version, --arch, and --libc"
|
||||
}
|
||||
|
||||
abspath() {
|
||||
local path=$1
|
||||
local dir base
|
||||
dir=$(cd "$(dirname "$path")" && pwd)
|
||||
base=$(basename "$path")
|
||||
printf '%s/%s' "$dir" "$base"
|
||||
}
|
||||
|
||||
sha256_file() {
|
||||
sha256sum "$1" | awk 'NR==1 { print $1 }'
|
||||
}
|
||||
|
||||
package_binary() {
|
||||
local binary='' output=''
|
||||
local -a zip_extras=()
|
||||
local saw_version=0 saw_arch=0 saw_libc=0
|
||||
version=''
|
||||
arch=''
|
||||
libc=''
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--version)
|
||||
require_value "$@"
|
||||
saw_version=1
|
||||
version=$2
|
||||
shift 2
|
||||
;;
|
||||
--arch)
|
||||
require_value "$@"
|
||||
saw_arch=1
|
||||
arch=$2
|
||||
shift 2
|
||||
;;
|
||||
--libc)
|
||||
require_value "$@"
|
||||
saw_libc=1
|
||||
libc=$2
|
||||
shift 2
|
||||
;;
|
||||
--binary)
|
||||
require_value "$@"
|
||||
binary=$2
|
||||
shift 2
|
||||
;;
|
||||
--output)
|
||||
require_value "$@"
|
||||
output=$2
|
||||
shift 2
|
||||
;;
|
||||
--zip-extra)
|
||||
require_value "$@"
|
||||
zip_extras+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
die "unknown argument: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
[[ "$saw_version" == 1 && "$saw_arch" == 1 && "$saw_libc" == 1 && -n "$binary" && -n "$output" ]] ||
|
||||
die "package requires --version, --arch, --libc, --binary, and --output"
|
||||
[[ -f "$binary" && ! -L "$binary" ]] || die "binary must be a regular file"
|
||||
load_versions
|
||||
|
||||
local -a extra_names=()
|
||||
local extra base seen=' '
|
||||
local has_json=0 has_sha=0
|
||||
for extra in "${zip_extras[@]}"; do
|
||||
base=$(basename "$extra")
|
||||
case "$base" in
|
||||
rustfs.cpu-symbol-catalog.json) has_json=1 ;;
|
||||
rustfs.cpu-symbol-catalog.sha256) has_sha=1 ;;
|
||||
*) die "unsupported zip member: ${base}" ;;
|
||||
esac
|
||||
[[ "$seen" != *" ${base} "* ]] || die "duplicate zip member: ${base}"
|
||||
seen+=" ${base} "
|
||||
[[ -f "$extra" && ! -L "$extra" ]] || die "zip extra must be a regular file: ${base}"
|
||||
extra_names+=("$base")
|
||||
done
|
||||
[[ "$has_json" == "$has_sha" ]] ||
|
||||
die "cpu symbol catalog json and sha256 must be packaged together"
|
||||
|
||||
local tool
|
||||
for tool in zip unzip sha256sum dpkg-deb fakeroot rpm rpm2cpio cpio fpm; do
|
||||
command -v "$tool" >/dev/null 2>&1 || die "required tool not found: ${tool}"
|
||||
done
|
||||
|
||||
binary=$(abspath "$binary")
|
||||
local -a absolute_extras=()
|
||||
for extra in "${zip_extras[@]}"; do
|
||||
absolute_extras+=("$(abspath "$extra")")
|
||||
done
|
||||
mkdir -p "$output"
|
||||
output=$(cd "$output" && pwd)
|
||||
local produced
|
||||
for produced in "$zip_file" "$deb_file" "$rpm_file"; do
|
||||
[[ ! -e "$output/$produced" ]] || die "refusing to overwrite ${output}/${produced}"
|
||||
done
|
||||
|
||||
# Globals so the EXIT trap can still see them after this function returns.
|
||||
work=$(mktemp -d)
|
||||
stage=$(mktemp -d)
|
||||
cleanup() {
|
||||
rm -rf "${work:-}" "${stage:-}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
local zip_dir="$work/zip"
|
||||
mkdir -p "$zip_dir"
|
||||
cp "$binary" "$zip_dir/rustfs"
|
||||
chmod 755 "$zip_dir/rustfs"
|
||||
local index=0
|
||||
for extra in "${absolute_extras[@]}"; do
|
||||
cp "$extra" "$zip_dir/${extra_names[$index]}"
|
||||
index=$((index + 1))
|
||||
done
|
||||
(
|
||||
cd "$zip_dir"
|
||||
zip "$stage/$zip_file" rustfs "${extra_names[@]}"
|
||||
)
|
||||
|
||||
local members expected
|
||||
members=$(unzip -Z1 "$stage/$zip_file" | sort)
|
||||
expected=$(printf '%s\n' rustfs "${extra_names[@]}" | sort)
|
||||
[[ "$members" == "$expected" ]] || die "zip members do not match the release archive"
|
||||
[[ "$(unzip -p "$stage/$zip_file" rustfs | sha256sum | awk '{print $1}')" == "$(sha256_file "$binary")" ]] ||
|
||||
die "zip binary differs from the packaged executable"
|
||||
|
||||
ln -s "$repo_root/scripts" "$work/scripts"
|
||||
ln -s "$repo_root/deploy" "$work/deploy"
|
||||
cp "$repo_root/LICENSE" "$repo_root/README.md" "$work/"
|
||||
mkdir -p "$work/bin"
|
||||
cp "$binary" "$work/bin/rustfs"
|
||||
chmod 755 "$work/bin/rustfs"
|
||||
|
||||
local deb_arch deb_depends rpm_depends
|
||||
case "$arch" in
|
||||
x86_64) deb_arch=amd64 ;;
|
||||
aarch64) deb_arch=arm64 ;;
|
||||
*) die "unsupported architecture: ${arch}" ;;
|
||||
esac
|
||||
case "$libc" in
|
||||
gnu)
|
||||
deb_depends='libc6 (>= 2.31)'
|
||||
rpm_depends='glibc >= 2.31'
|
||||
;;
|
||||
musl)
|
||||
deb_depends=''
|
||||
rpm_depends=''
|
||||
;;
|
||||
*) die "unsupported libc: ${libc}" ;;
|
||||
esac
|
||||
|
||||
(
|
||||
cd "$work"
|
||||
GITHUB_OUTPUT="$work/outputs" \
|
||||
DEB_VERSION="$deb_version" \
|
||||
RPM_VERSION="$rpm_version" \
|
||||
RPM_RELEASE="$rpm_release" \
|
||||
DEB_FILE="$deb_file" \
|
||||
RPM_FILE="$rpm_file" \
|
||||
DEB_ARCH="$deb_arch" \
|
||||
RPM_ARCH="$arch" \
|
||||
DEB_DEPENDS="$deb_depends" \
|
||||
RPM_DEPENDS="$rpm_depends" \
|
||||
LIBC="$libc" \
|
||||
bash "$repo_root/scripts/release/build_deb.sh"
|
||||
GITHUB_OUTPUT="$work/outputs" \
|
||||
DEB_VERSION="$deb_version" \
|
||||
RPM_VERSION="$rpm_version" \
|
||||
RPM_RELEASE="$rpm_release" \
|
||||
DEB_FILE="$deb_file" \
|
||||
RPM_FILE="$rpm_file" \
|
||||
DEB_ARCH="$deb_arch" \
|
||||
RPM_ARCH="$arch" \
|
||||
DEB_DEPENDS="$deb_depends" \
|
||||
RPM_DEPENDS="$rpm_depends" \
|
||||
LIBC="$libc" \
|
||||
bash "$repo_root/scripts/release/build_rpm.sh"
|
||||
)
|
||||
cp "$work/$deb_file" "$stage/$deb_file"
|
||||
cp "$work/$rpm_file" "$stage/$rpm_file"
|
||||
|
||||
local unpacked expected_hash actual_hash
|
||||
expected_hash=$(sha256_file "$binary")
|
||||
unpacked="$work/unpacked-deb"
|
||||
mkdir -p "$unpacked"
|
||||
dpkg-deb --extract "$stage/$deb_file" "$unpacked"
|
||||
actual_hash=$(sha256_file "$unpacked/usr/bin/rustfs")
|
||||
[[ "$actual_hash" == "$expected_hash" ]] || die "deb binary differs from the packaged executable"
|
||||
[[ "$(dpkg-deb -f "$stage/$deb_file" Package)" == rustfs ]] || die "deb package name mismatch"
|
||||
[[ "$(dpkg-deb -f "$stage/$deb_file" Version)" == "$deb_version" ]] || die "deb version mismatch"
|
||||
[[ "$(dpkg-deb -f "$stage/$deb_file" Architecture)" == "$deb_arch" ]] || die "deb architecture mismatch"
|
||||
|
||||
unpacked="$work/unpacked-rpm"
|
||||
mkdir -p "$unpacked"
|
||||
rpm2cpio "$stage/$rpm_file" | (cd "$unpacked" && cpio --extract --make-directories --quiet ./usr/bin/rustfs)
|
||||
actual_hash=$(sha256_file "$unpacked/usr/bin/rustfs")
|
||||
[[ "$actual_hash" == "$expected_hash" ]] || die "rpm binary differs from the packaged executable"
|
||||
local rpm_metadata expected_metadata
|
||||
rpm_metadata=$(rpm -qp --qf '%{NAME}\n%{VERSION}\n%{RELEASE}\n%{ARCH}\n' "$stage/$rpm_file")
|
||||
expected_metadata=$(printf 'rustfs\n%s\n%s\n%s' "$rpm_version" "$rpm_release" "$arch")
|
||||
[[ "$rpm_metadata" == "$expected_metadata" ]] || die "rpm metadata mismatch"
|
||||
|
||||
mv "$stage/$zip_file" "$output/$zip_file"
|
||||
mv "$stage/$deb_file" "$output/$deb_file"
|
||||
mv "$stage/$rpm_file" "$output/$rpm_file"
|
||||
emit_resolved
|
||||
}
|
||||
|
||||
main() {
|
||||
local command=${1:-}
|
||||
case "$command" in
|
||||
matrix)
|
||||
[[ $# -eq 2 ]] || usage
|
||||
emit_matrix "$2"
|
||||
;;
|
||||
validate)
|
||||
shift
|
||||
parse_identity "$@"
|
||||
load_versions
|
||||
emit_resolved
|
||||
;;
|
||||
package)
|
||||
shift
|
||||
package_binary "$@"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,226 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Contract tests for manual origin/main packaging."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "release" / "package_manual_artifacts.sh"
|
||||
WORKFLOW = ROOT / ".github" / "workflows" / "manual-main-packages.yml"
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from ci_gate import workflow_document # noqa: E402
|
||||
import validation_registry as registry # noqa: E402
|
||||
|
||||
|
||||
def run_script(*args, check=False):
|
||||
return subprocess.run(["bash", str(SCRIPT), *args], cwd=ROOT, text=True,
|
||||
capture_output=True, timeout=60, check=check)
|
||||
|
||||
|
||||
def trigger(text):
|
||||
match = re.search(r"^on:\n(.*?)(?=^[^ #\n]|\Z)", text, re.M | re.S)
|
||||
if match is None:
|
||||
raise AssertionError("workflow has no event declaration")
|
||||
return re.findall(r"^ (\w+):", match.group(1), re.M)
|
||||
|
||||
|
||||
class VersionAndMatrixTests(unittest.TestCase):
|
||||
def test_release_and_prerelease_names_come_from_package_versions(self):
|
||||
cases = {
|
||||
("1.2.3", "x86_64", "gnu"): """\
|
||||
build_type=release
|
||||
deb_version=1.2.3
|
||||
rpm_version=1.2.3
|
||||
rpm_release=1
|
||||
deb_file=rustfs-linux-x86_64-gnu-v1.2.3.deb
|
||||
rpm_file=rustfs-linux-x86_64-gnu-v1.2.3.rpm
|
||||
zip_file=rustfs-linux-x86_64-gnu-v1.2.3.zip
|
||||
""",
|
||||
("1.2.3-alpha.1", "aarch64", "gnu"): """\
|
||||
build_type=prerelease
|
||||
deb_version=1.2.3~alpha.1
|
||||
rpm_version=1.2.3~alpha.1
|
||||
rpm_release=1
|
||||
deb_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.deb
|
||||
rpm_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.rpm
|
||||
zip_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.zip
|
||||
""",
|
||||
("1.2.3-beta.2", "x86_64", "musl"): """\
|
||||
build_type=prerelease
|
||||
deb_version=1.2.3~beta.2
|
||||
rpm_version=1.2.3~beta.2
|
||||
rpm_release=1
|
||||
deb_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.deb
|
||||
rpm_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.rpm
|
||||
zip_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.zip
|
||||
""",
|
||||
("1.2.3-rc.4", "aarch64", "musl"): """\
|
||||
build_type=prerelease
|
||||
deb_version=1.2.3~rc.4
|
||||
rpm_version=1.2.3~rc.4
|
||||
rpm_release=1
|
||||
deb_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.deb
|
||||
rpm_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.rpm
|
||||
zip_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.zip
|
||||
""",
|
||||
}
|
||||
for (version, arch, libc), expected in cases.items():
|
||||
with self.subTest(version=version, arch=arch, libc=libc):
|
||||
result = run_script("validate", "--version", version, "--arch", arch, "--libc", libc)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout, expected)
|
||||
|
||||
def test_invalid_versions_fail_without_stdout(self):
|
||||
for version in ("", "v1.2.3", "1.2", "01.2.3", "1.2.3-preview.1", "1.2.3-alpha",
|
||||
"1.2.3-rc.01", "1.2.3-rc.1-extra", "1.2.3 ", "1.2.3$(id)", "dev-abcdef"):
|
||||
with self.subTest(version=version):
|
||||
result = run_script("validate", "--version", version, "--arch", "x86_64", "--libc", "gnu")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, "")
|
||||
self.assertIn("invalid package version", result.stderr)
|
||||
self.assertIn("not a release:", result.stderr)
|
||||
self.assertIn("not a prerelease:", result.stderr)
|
||||
|
||||
def test_invalid_version_does_not_create_packages(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
binary = root / "rustfs"
|
||||
binary.write_bytes(b"not packaged")
|
||||
output = root / "dist"
|
||||
result = run_script("package", "--version", "v1.2.3", "--arch", "x86_64", "--libc", "gnu",
|
||||
"--binary", str(binary), "--output", str(output))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, "")
|
||||
self.assertFalse(output.exists())
|
||||
|
||||
def test_catalog_members_must_be_the_official_pair(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
binary = root / "rustfs"
|
||||
binary.write_bytes(b"not packaged")
|
||||
catalog = root / "rustfs.cpu-symbol-catalog.json"
|
||||
catalog.write_text("{}\n", encoding="utf-8")
|
||||
result = run_script("package", "--version", "1.2.3", "--arch", "x86_64", "--libc", "gnu",
|
||||
"--binary", str(binary), "--output", str(root / "dist"),
|
||||
"--zip-extra", str(catalog))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("packaged together", result.stderr)
|
||||
self.assertFalse((root / "dist").exists())
|
||||
other = root / "notes.txt"
|
||||
other.write_text("nope\n", encoding="utf-8")
|
||||
rejected = run_script("package", "--version", "1.2.3", "--arch", "x86_64", "--libc", "musl",
|
||||
"--binary", str(binary), "--output", str(root / "dist"),
|
||||
"--zip-extra", str(other))
|
||||
self.assertNotEqual(rejected.returncode, 0)
|
||||
self.assertIn("unsupported zip member", rejected.stderr)
|
||||
|
||||
def test_architecture_matrix_matches_official_linux_rows(self):
|
||||
source = (ROOT / ".github" / "workflows" / "build.yml").read_text(encoding="utf-8")
|
||||
official = json.loads(re.search(r"all='(\{.*?\})'", source, re.S).group(1))["include"]
|
||||
linux = [row for row in official if row["platform"] == "linux"]
|
||||
|
||||
def derived(row):
|
||||
arch = "aarch64" if row["target"].startswith("aarch64-") else "x86_64"
|
||||
libc = row["target_id"].rsplit("-", 1)[1]
|
||||
return {**row, "arch": arch, "libc": libc, "build_os": f"linux-{arch}"}
|
||||
|
||||
expected = {
|
||||
"all": [derived(row) for row in linux],
|
||||
"x86": [derived(row) for row in linux if row["target"].startswith("x86_64-")],
|
||||
"arm": [derived(row) for row in linux if row["target"].startswith("aarch64-")],
|
||||
}
|
||||
self.assertEqual([row["target_id"] for row in expected["x86"]],
|
||||
["linux-x86_64-musl", "linux-x86_64-gnu"])
|
||||
self.assertEqual([row["target_id"] for row in expected["arm"]],
|
||||
["linux-aarch64-musl", "linux-aarch64-gnu"])
|
||||
for choice, rows in expected.items():
|
||||
with self.subTest(choice=choice):
|
||||
result = run_script("matrix", choice, check=True)
|
||||
self.assertEqual(json.loads(result.stdout), {"include": rows})
|
||||
for choice in ("", "ALL", "linux", "x86_64", "x86,arm"):
|
||||
with self.subTest(choice=choice):
|
||||
result = run_script("matrix", choice)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, "")
|
||||
self.assertIn("unsupported architecture", result.stderr)
|
||||
|
||||
|
||||
class WorkflowContractTests(unittest.TestCase):
|
||||
def test_dispatch_builds_origin_main_and_only_uploads_artifacts(self):
|
||||
text = WORKFLOW.read_text(encoding="utf-8")
|
||||
self.assertEqual(trigger(text), ["workflow_dispatch"])
|
||||
document = workflow_document(text)
|
||||
events = document.get("on", document.get(True))
|
||||
self.assertEqual(set(events), {"workflow_dispatch"})
|
||||
inputs = events["workflow_dispatch"]["inputs"]
|
||||
self.assertEqual(inputs["architecture"]["type"], "choice")
|
||||
self.assertEqual(inputs["architecture"]["options"], ["x86", "arm", "all"])
|
||||
self.assertTrue(inputs["architecture"]["required"])
|
||||
self.assertEqual(inputs["version"]["type"], "string")
|
||||
self.assertTrue(inputs["version"]["required"])
|
||||
self.assertEqual(document["permissions"], {"contents": "read"})
|
||||
self.assertEqual(re.findall(r"^\s+ref:\s*(.+)$", text, re.M),
|
||||
["main", "${{ needs.prepare.outputs.source_sha }}"])
|
||||
self.assertIn("refs/heads/main", text)
|
||||
self.assertNotIn("github.sha", text)
|
||||
self.assertIn("package_manual_artifacts.sh validate", text)
|
||||
self.assertIn("package_manual_artifacts.sh matrix", text)
|
||||
self.assertIn("package_manual_artifacts.sh package", text)
|
||||
self.assertIn("RUSTFS_BUILD_VERSION: ${{ needs.prepare.outputs.version }}", text)
|
||||
self.assertIn("cargo zigbuild --release --locked", text)
|
||||
self.assertIn("cargo build --release --locked", text)
|
||||
self.assertIn("scripts/release/check_glibc.py", text)
|
||||
self.assertIn('cache-save-if: \'false\'', text)
|
||||
self.assertIn("actions/upload-artifact@", text)
|
||||
self.assertIn("dist/*", text)
|
||||
for forbidden in ("contents: write", "aws s3", "gh release", "softprops/action-gh-release",
|
||||
"docker push", "R2_ACCESS_KEY_ID", "workflow_call:"):
|
||||
self.assertNotIn(forbidden, text)
|
||||
self.assertEqual(text.count("persist-credentials: false"), text.count("actions/checkout@"))
|
||||
item = registry.workflow_entry(".github/workflows/manual-main-packages.yml")
|
||||
self.assertEqual(item["name"], "Manual main packages")
|
||||
self.assertEqual(item["events"], ["workflow_dispatch"])
|
||||
self.assertEqual(item["schedule"], [])
|
||||
self.assertEqual(item["monitor"], "none")
|
||||
self.assertNotEqual(item["origin"], "baseline")
|
||||
|
||||
def test_registry_accepts_the_added_workflow(self):
|
||||
self.assertTrue(registry.check_wiring(registry.load_registry()))
|
||||
|
||||
|
||||
class PackageToolTests(unittest.TestCase):
|
||||
def test_zip_deb_and_rpm_wrap_the_same_binary_when_tools_exist(self):
|
||||
required = ("dpkg-deb", "fakeroot", "rpm", "rpm2cpio", "cpio", "fpm", "zip", "unzip")
|
||||
missing = [tool for tool in required if shutil.which(tool) is None]
|
||||
if missing or os.name != "posix":
|
||||
self.skipTest("packaging tools not installed: " + ", ".join(missing))
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
payload = b"manual main package fixture\n"
|
||||
binary = root / "rustfs"
|
||||
binary.write_bytes(payload)
|
||||
result = run_script("package", "--version", "9.9.9-rc.8", "--arch", "x86_64", "--libc", "gnu",
|
||||
"--binary", str(binary), "--output", str(root / "dist"))
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
dist = root / "dist"
|
||||
self.assertEqual(sorted(path.name for path in dist.iterdir()), [
|
||||
"rustfs-linux-x86_64-gnu-v9.9.9-rc.8.deb",
|
||||
"rustfs-linux-x86_64-gnu-v9.9.9-rc.8.rpm",
|
||||
"rustfs-linux-x86_64-gnu-v9.9.9-rc.8.zip",
|
||||
])
|
||||
self.assertIn("deb_version=9.9.9~rc.8", result.stdout)
|
||||
self.assertIn("zip_file=rustfs-linux-x86_64-gnu-v9.9.9-rc.8.zip", result.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -509,7 +509,9 @@ pass/fail table in the job summary.
|
||||
incomplete parameterized cases.
|
||||
- `api_coverage.py` — quantifies S3 API surface coverage by comparing the
|
||||
s3s `S3` trait (at the revision pinned in Cargo.toml) against the methods
|
||||
RustFS overrides in `impl S3 for FS`:
|
||||
RustFS overrides in `impl S3 for FS`, found exactly once across
|
||||
`rustfs/src/storage/ecfs.rs` and `rustfs/src/storage/legacy_edge/`
|
||||
(contract tests: `test_api_coverage.py`, run by `make script-tests`):
|
||||
|
||||
```bash
|
||||
python3 scripts/s3-tests/api_coverage.py # summary + missing ops
|
||||
|
||||
@@ -17,7 +17,8 @@
|
||||
|
||||
Compares the operations defined by the s3s `S3` trait (at the revision pinned
|
||||
in the workspace Cargo.toml) against the methods RustFS overrides in
|
||||
`impl S3 for FS` (rustfs/src/storage/ecfs.rs). Trait methods that are not
|
||||
`impl S3 for FS` (rustfs/src/storage/ecfs.rs, or the rustfs/src/storage/legacy_edge/
|
||||
directory a mechanical split of that file moves code into). Trait methods that are not
|
||||
overridden fall back to the s3s default implementation, which returns
|
||||
NotImplemented — so "overridden" is a faithful proxy for "implemented".
|
||||
|
||||
@@ -36,6 +37,8 @@ import sys
|
||||
import tempfile
|
||||
|
||||
ECFS_PATH = "rustfs/src/storage/ecfs.rs"
|
||||
ECFS_SPLIT_DIR = "rustfs/src/storage/legacy_edge"
|
||||
IMPL_MARKER = "impl S3 for FS {"
|
||||
TRAIT_RELPATH = "crates/s3s/src/s3_trait.rs"
|
||||
|
||||
|
||||
@@ -97,14 +100,28 @@ def trait_operations(trait_source: pathlib.Path) -> dict[str, str]:
|
||||
return ops
|
||||
|
||||
|
||||
def implemented_operations(ecfs_source: pathlib.Path) -> set[str]:
|
||||
src = ecfs_source.read_text(encoding="utf-8")
|
||||
marker = "impl S3 for FS {"
|
||||
start = src.find(marker)
|
||||
if start < 0:
|
||||
raise RuntimeError(f"`{marker}` not found in {ecfs_source}")
|
||||
block = extract_block(src, start)
|
||||
return set(re.findall(r"async fn (\w+)", block))
|
||||
def ecfs_sources(root: pathlib.Path) -> list[pathlib.Path]:
|
||||
"""Return ecfs.rs (required) followed by every .rs file under the split directory, in path order."""
|
||||
required = root / ECFS_PATH
|
||||
if not required.is_file():
|
||||
raise RuntimeError(f"{required} is missing")
|
||||
split_dir = root / ECFS_SPLIT_DIR
|
||||
if not split_dir.exists():
|
||||
return [required]
|
||||
if not split_dir.is_dir():
|
||||
raise RuntimeError(f"{split_dir} is not a directory")
|
||||
return [required, *sorted(split_dir.rglob("*.rs"), key=lambda path: path.relative_to(root).as_posix())]
|
||||
|
||||
|
||||
def implemented_operations(root: pathlib.Path) -> set[str]:
|
||||
"""Return the methods of the one `impl S3 for FS` block across the ECFS source set."""
|
||||
blocks = []
|
||||
for source in ecfs_sources(root):
|
||||
src = source.read_text(encoding="utf-8")
|
||||
blocks += [extract_block(src, match.start()) for match in re.finditer(re.escape(IMPL_MARKER), src)]
|
||||
if len(blocks) != 1:
|
||||
raise RuntimeError(f"`{IMPL_MARKER}` must occur exactly once under {ECFS_PATH} and {ECFS_SPLIT_DIR}, found {len(blocks)}")
|
||||
return set(re.findall(r"async fn (\w+)", blocks[0]))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -122,7 +139,7 @@ def main() -> int:
|
||||
trait_source = args.s3s_trait or locate_trait_source(git_url, rev)
|
||||
|
||||
all_ops = trait_operations(trait_source)
|
||||
impl_ops = implemented_operations(root / ECFS_PATH)
|
||||
impl_ops = implemented_operations(root)
|
||||
|
||||
stale = sorted(impl_ops - all_ops.keys())
|
||||
implemented = sorted(op for op in all_ops if op in impl_ops)
|
||||
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Contract tests for how api_coverage.py finds `impl S3 for FS` across the ECFS source set."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import api_coverage # noqa: E402
|
||||
|
||||
IMPL = "impl S3 for FS {\n async fn get_object(&self) {}\n async fn put_object(&self) {}\n}\n"
|
||||
|
||||
|
||||
class ImplementedOperationsTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def write(self, relative: str, text: str) -> None:
|
||||
path = self.root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
def test_reads_the_impl_from_ecfs_when_the_split_directory_is_absent(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n" + IMPL)
|
||||
|
||||
self.assertEqual(api_coverage.implemented_operations(self.root), {"get_object", "put_object"})
|
||||
|
||||
def test_finds_the_impl_after_it_moves_into_the_split_directory(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/bucket_core.rs", "impl FS {\n async fn helper(&self) {}\n}\n")
|
||||
|
||||
self.assertEqual(api_coverage.implemented_operations(self.root), {"get_object", "put_object"})
|
||||
|
||||
def test_missing_ecfs_fails_even_when_the_split_directory_holds_the_impl(self) -> None:
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "ecfs.rs is missing"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_an_impl_in_both_ecfs_and_the_split_directory_fails(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, IMPL)
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "exactly once.*found 2"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_no_impl_anywhere_fails(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/bucket_core.rs", "impl FS {}\n")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "found 0"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_an_impl_in_a_non_rust_file_of_the_split_directory_is_not_source(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/notes.md", IMPL)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "found 0"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_a_split_path_that_is_a_file_fails(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, IMPL)
|
||||
self.write(api_coverage.ECFS_SPLIT_DIR, "stray\n")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "not a directory"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Executable
+237
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env bash
|
||||
# Contract tests for scripts/guard_source_set.sh, the source-set helper that
|
||||
# lets text guards read a file plus the directory a mechanical split moves its
|
||||
# code into (rustfs/backlog#2765).
|
||||
#
|
||||
# Each case builds a throwaway tree; nothing in the real workspace is read.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck source=scripts/guard_source_set.sh
|
||||
source "${GUARD_SOURCE_SET_SCRIPT:-$SCRIPT_DIR/guard_source_set.sh}"
|
||||
TMP_ROOT="$(mktemp -d)"
|
||||
trap 'chmod -R u+rwx "$TMP_ROOT" 2>/dev/null || true; rm -rf "$TMP_ROOT"' EXIT
|
||||
|
||||
passed=0
|
||||
skipped=0
|
||||
case_no=0
|
||||
FX=""
|
||||
STATUS=0
|
||||
ERR=""
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
new_case() {
|
||||
case_no=$((case_no + 1))
|
||||
FX="$TMP_ROOT/case$case_no"
|
||||
mkdir -p "$FX"
|
||||
}
|
||||
|
||||
put() {
|
||||
mkdir -p "$(dirname "$FX/$1")"
|
||||
printf 'fn guarded() {}\n' >"$FX/$1"
|
||||
}
|
||||
|
||||
# run_set <args...>: call guard_source_set on the case tree, capturing status and stderr.
|
||||
run_set() {
|
||||
local err_file="$FX/.stderr"
|
||||
STATUS=0
|
||||
guard_source_set "$FX" "$@" 2>"$err_file" || STATUS=$?
|
||||
ERR="$(cat "$err_file")"
|
||||
}
|
||||
|
||||
sources_are() {
|
||||
local expected actual
|
||||
expected="$(printf '%s\n' "$@")"
|
||||
actual="$(printf '%s\n' "${GUARD_SOURCES[@]}")"
|
||||
[[ "$actual" == "$expected" ]] || fail "case $case_no: sources were [$actual], expected [$expected]"
|
||||
}
|
||||
|
||||
ok() {
|
||||
passed=$((passed + 1))
|
||||
}
|
||||
|
||||
# --- positive cases ---------------------------------------------------------
|
||||
|
||||
# An absent split directory contributes nothing.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 0 ]] || fail "absent split dir: status $STATUS ($ERR)"
|
||||
sources_are src/usecase.rs
|
||||
ok
|
||||
|
||||
# A present split directory adds its .rs files, recursively, in byte order,
|
||||
# after the required file; non-.rs files are not source.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
put src/usecase/zeta.rs
|
||||
put src/usecase/B.rs
|
||||
put src/usecase/a.rs
|
||||
put src/usecase/tests/nested.rs
|
||||
printf 'not rust\n' >"$FX/src/usecase/notes.md"
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 0 ]] || fail "present split dir: status $STATUS ($ERR)"
|
||||
sources_are src/usecase.rs src/usecase/B.rs src/usecase/a.rs src/usecase/tests/nested.rs src/usecase/zeta.rs
|
||||
ok
|
||||
|
||||
# Several split directories are read in argument order; an empty one adds nothing.
|
||||
new_case
|
||||
put src/ecfs.rs
|
||||
put src/legacy_edge/b.rs
|
||||
mkdir -p "$FX/src/empty"
|
||||
put src/other/a.rs
|
||||
run_set src/ecfs.rs src/legacy_edge src/empty src/missing src/other
|
||||
[[ "$STATUS" == 0 ]] || fail "several split dirs: status $STATUS ($ERR)"
|
||||
sources_are src/ecfs.rs src/legacy_edge/b.rs src/other/a.rs
|
||||
ok
|
||||
|
||||
# --- negative cases ---------------------------------------------------------
|
||||
|
||||
# A missing required file fails and names the file.
|
||||
new_case
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "missing required file: status $STATUS"
|
||||
[[ "$ERR" == *"src/usecase.rs"* ]] || fail "missing required file: stderr does not name it: $ERR"
|
||||
sources_are src/usecase.rs
|
||||
ok
|
||||
|
||||
# A missing required file fails even when the split directory holds the code:
|
||||
# the split must keep the file, and a guard must not pass on half its input.
|
||||
new_case
|
||||
put src/usecase/moved.rs
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "missing required file with split dir: status $STATUS"
|
||||
[[ "$ERR" == *"is missing"* ]] || fail "missing required file with split dir: $ERR"
|
||||
ok
|
||||
|
||||
# A required path that is a directory is not a source file.
|
||||
new_case
|
||||
mkdir -p "$FX/src/usecase.rs"
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "required directory: status $STATUS"
|
||||
ok
|
||||
|
||||
# A split path that exists as a regular file fails rather than being skipped.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
printf 'stray\n' >"$FX/src/usecase"
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "split path is a file: status $STATUS"
|
||||
[[ "$ERR" == *"not a directory"* ]] || fail "split path is a file: $ERR"
|
||||
ok
|
||||
|
||||
# A later split path that is a file still fails after an earlier valid one.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
put src/usecase/a.rs
|
||||
printf 'stray\n' >"$FX/src/second"
|
||||
run_set src/usecase.rs src/usecase src/second
|
||||
[[ "$STATUS" == 1 ]] || fail "second split path is a file: status $STATUS"
|
||||
ok
|
||||
|
||||
# An unlistable split directory fails rather than contributing nothing.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
put src/usecase/hidden/a.rs
|
||||
chmod 000 "$FX/src/usecase/hidden"
|
||||
if [[ "$(id -u)" == 0 ]]; then
|
||||
echo "SKIP: unlistable split directory case needs a non-root user (root ignores directory permissions)"
|
||||
skipped=$((skipped + 1))
|
||||
else
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "unlistable split dir: status $STATUS"
|
||||
[[ "$ERR" == *"cannot be listed"* ]] || fail "unlistable split dir: $ERR"
|
||||
ok
|
||||
fi
|
||||
chmod 755 "$FX/src/usecase/hidden"
|
||||
|
||||
# --- module_relative_rg ------------------------------------------------------
|
||||
#
|
||||
# The templates are read from check_architecture_migration_rules.sh itself, so
|
||||
# these cases exercise the rules as shipped, not copies of them.
|
||||
ARCH_RULES="${ARCH_RULES_SCRIPT:-$SCRIPT_DIR/check_architecture_migration_rules.sh}"
|
||||
TEMPLATES=()
|
||||
while IFS= read -r template; do
|
||||
TEMPLATES+=("$template")
|
||||
done < <(awk '
|
||||
pending { sub(/^[[:space:]]*'"'"'/, ""); sub(/'"'"'[[:space:]]*\\$/, ""); print; pending = 0; next }
|
||||
/module_relative_rg rustfs\/src\/(app|storage) \\$/ { pending = 1 }
|
||||
' "$ARCH_RULES")
|
||||
[[ ${#TEMPLATES[@]} == 6 ]] || fail "expected 6 depth-aware rules in $ARCH_RULES, found ${#TEMPLATES[@]}"
|
||||
STORAGE_HELPER_RULE="${TEMPLATES[0]}"
|
||||
STORAGE_OWNER_RULE="${TEMPLATES[1]}"
|
||||
RUNTIME_IMPORT_RULE="${TEMPLATES[2]}"
|
||||
RUNTIME_PATH_RULE="${TEMPLATES[3]}"
|
||||
ECFS_TEST_RULE="${TEMPLATES[4]}"
|
||||
ECFS_TEST_SPLIT_RULE="${TEMPLATES[5]}"
|
||||
|
||||
# relative_hits <module-dir> <template> <relative-file> <line>: rg output for one file.
|
||||
relative_hits() {
|
||||
local module_dir="$1" template="$2" file="$3" line="$4"
|
||||
mkdir -p "$(dirname "$FX/$file")"
|
||||
printf '%s\n' "$line" >"$FX/$file"
|
||||
(cd "$FX" && module_relative_rg "$module_dir" "$template" -n -U --with-filename -- "$file")
|
||||
}
|
||||
|
||||
flagged() {
|
||||
[[ -n "$(relative_hits "$@")" ]] || fail "case $case_no: not flagged: $3: $4"
|
||||
ok
|
||||
}
|
||||
|
||||
not_flagged() {
|
||||
[[ -z "$(relative_hits "$@")" ]] || fail "case $case_no: flagged: $3: $4"
|
||||
ok
|
||||
}
|
||||
|
||||
new_case
|
||||
[[ "$(module_relative_depth rustfs/src/app rustfs/src/app/bucket_usecase.rs)" == 1 ]] || fail "depth of a module file"
|
||||
[[ "$(module_relative_depth rustfs/src/app rustfs/src/app/object/mod.rs)" == 1 ]] || fail "depth of a mod.rs"
|
||||
[[ "$(module_relative_depth rustfs/src/app rustfs/src/app/bucket_usecase/cors.rs)" == 2 ]] || fail "depth of a split file"
|
||||
[[ "$(module_relative_depth rustfs/src/app rustfs/src/app/multipart_usecase/tests/a.rs)" == 3 ]] || fail "depth of a nested split file"
|
||||
ok
|
||||
|
||||
# Forbidden forms: each spelling that names crate::app (or the storage root) is flagged.
|
||||
new_case
|
||||
flagged rustfs/src/app "$STORAGE_HELPER_RULE" rustfs/src/app/bucket_usecase.rs 'use super::StorageError;'
|
||||
flagged rustfs/src/app "$STORAGE_HELPER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::super::StorageError;'
|
||||
flagged rustfs/src/app "$STORAGE_HELPER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use crate::app::StorageError;'
|
||||
flagged rustfs/src/app "$STORAGE_HELPER_RULE" rustfs/src/app/object/mod.rs 'use super::{DiskError, StorageError};'
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/multipart_usecase/complete.rs 'use super::super::{quota, versioning_sys};'
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase.rs 'super::tagging::helper();'
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/tagging.rs 'super::super::tagging::helper();'
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase.rs 'super::super::metadata_sys::get();'
|
||||
flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::super::super::metadata_sys::get();'
|
||||
flagged rustfs/src/app "$RUNTIME_IMPORT_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::super::ECStore;'
|
||||
flagged rustfs/src/app "$RUNTIME_PATH_RULE" rustfs/src/app/lifecycle_transition_api_test/put.rs 'super::super::get_server_info();'
|
||||
flagged rustfs/src/storage "$ECFS_TEST_RULE" rustfs/src/storage/ecfs_test.rs ' use super::super::{BucketMetadata};'
|
||||
flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs 'use super::super::super::{BucketMetadata};'
|
||||
flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs ' use super::super::super::super::{check_preconditions};'
|
||||
|
||||
# Ordinary forms: a split file importing from its own parent module is not flagged.
|
||||
new_case
|
||||
not_flagged rustfs/src/app "$STORAGE_HELPER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::StorageError;'
|
||||
not_flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::{DefaultBucketUsecase, ObjectInfo, ObjectOptions};'
|
||||
not_flagged rustfs/src/app "$STORAGE_OWNER_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::tagging::helper();'
|
||||
not_flagged rustfs/src/app "$RUNTIME_IMPORT_RULE" rustfs/src/app/bucket_usecase/cors.rs 'use super::{ECStore, DefaultBucketUsecase};'
|
||||
not_flagged rustfs/src/app "$RUNTIME_PATH_RULE" rustfs/src/app/bucket_usecase/cors.rs 'super::get_server_info();'
|
||||
not_flagged rustfs/src/storage "$ECFS_TEST_SPLIT_RULE" rustfs/src/storage/legacy_edge/tests/object_meta.rs ' use super::{BucketMetadata};'
|
||||
|
||||
# A file outside the module directory, or a missing -- separator, is an error, not a silent pass.
|
||||
new_case
|
||||
status=0
|
||||
module_relative_rg rustfs/src/app 'x' -n -- rustfs/src/storage/ecfs.rs 2>/dev/null || status=$?
|
||||
[[ "$status" == 2 ]] || fail "file outside the module dir: status $status"
|
||||
status=0
|
||||
module_relative_rg rustfs/src/app 'x' -n rustfs/src/app/a.rs 2>/dev/null || status=$?
|
||||
[[ "$status" == 2 ]] || fail "missing --: status $status"
|
||||
status=0
|
||||
(cd "$FX" && module_relative_rg rustfs/src/app 'x' -n -- rustfs/src/app/missing.rs) 2>/dev/null || status=$?
|
||||
[[ "$status" == 2 ]] || fail "unreadable file: status $status"
|
||||
ok
|
||||
|
||||
echo "guard_source_set contract tests: $passed passed, $skipped skipped"
|
||||
@@ -11,7 +11,6 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / "release"))
|
||||
@@ -64,57 +63,6 @@ class PackageIntegrationTests(unittest.TestCase):
|
||||
deliver.package(fixtures.inputs(), "linux-x86_64-musl", root / "output")
|
||||
self.assertFalse(list((root / "output").glob("packages-*.json")))
|
||||
|
||||
def test_manual_packages_match_release_names_and_bytes(self):
|
||||
script = Path(__file__).resolve().parent / "release" / "package_manual_artifacts.sh"
|
||||
cases = (
|
||||
("9.9.9-rc.8", "x86_64", "gnu", True),
|
||||
("9.9.9", "aarch64", "musl", False),
|
||||
)
|
||||
for version, arch, libc, with_catalog in cases:
|
||||
with self.subTest(version=version, arch=arch, libc=libc), tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
payload = f"manual-package:{version}:{arch}:{libc}\n".encode()
|
||||
binary = root / "rustfs"
|
||||
binary.write_bytes(payload)
|
||||
command = ["bash", str(script), "package", "--version", version, "--arch", arch, "--libc", libc,
|
||||
"--binary", str(binary), "--output", str(root / "dist")]
|
||||
if with_catalog:
|
||||
for name in ("rustfs.cpu-symbol-catalog.json", "rustfs.cpu-symbol-catalog.sha256"):
|
||||
(root / name).write_text(name + "\n", encoding="utf-8")
|
||||
command.extend(["--zip-extra", str(root / name)])
|
||||
subprocess.run(command, check=True, text=True, timeout=180)
|
||||
stem = f"rustfs-linux-{arch}-{libc}-v{version}"
|
||||
dist = root / "dist"
|
||||
self.assertEqual(sorted(path.name for path in dist.iterdir()),
|
||||
[f"{stem}.deb", f"{stem}.rpm", f"{stem}.zip"])
|
||||
expected_hash = hashlib.sha256(payload).hexdigest()
|
||||
with zipfile.ZipFile(dist / f"{stem}.zip") as archive:
|
||||
names = sorted(archive.namelist())
|
||||
self.assertEqual(hashlib.sha256(archive.read("rustfs")).hexdigest(), expected_hash)
|
||||
if with_catalog:
|
||||
self.assertEqual(names, ["rustfs", "rustfs.cpu-symbol-catalog.json", "rustfs.cpu-symbol-catalog.sha256"])
|
||||
else:
|
||||
self.assertEqual(names, ["rustfs"])
|
||||
deb = dist / f"{stem}.deb"
|
||||
self.assertEqual(subprocess.check_output(["dpkg-deb", "-f", deb, "Version"], text=True).strip(),
|
||||
version.replace("-", "~"))
|
||||
self.assertEqual(subprocess.check_output(["dpkg-deb", "-f", deb, "Architecture"], text=True).strip(),
|
||||
"amd64" if arch == "x86_64" else "arm64")
|
||||
depends = subprocess.check_output(["dpkg-deb", "-f", deb, "Depends"], text=True).strip()
|
||||
self.assertEqual(depends, "libc6 (>= 2.31)" if libc == "gnu" else "")
|
||||
unpacked = root / "deb"
|
||||
subprocess.run(["dpkg-deb", "--extract", str(deb), str(unpacked)], check=True)
|
||||
self.assertEqual(hashlib.sha256((unpacked / "usr/bin/rustfs").read_bytes()).hexdigest(), expected_hash)
|
||||
rpm = dist / f"{stem}.rpm"
|
||||
identity = subprocess.check_output(
|
||||
["rpm", "-qp", "--qf", "%{NAME} %{VERSION} %{RELEASE} %{ARCH}", str(rpm)], text=True)
|
||||
self.assertEqual(identity, f"rustfs {version.replace('-', '~')} 1 {arch}")
|
||||
requires = subprocess.check_output(["rpm", "-qp", "--requires", str(rpm)], text=True)
|
||||
if libc == "gnu":
|
||||
self.assertIn("glibc >= 2.31", requires)
|
||||
else:
|
||||
self.assertNotIn("glibc", requires)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Executable
+223
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end probe for the guards that must survive the mechanical split of
|
||||
# ecfs.rs, bucket_usecase.rs, multipart_usecase.rs and
|
||||
# lifecycle_transition_api_test.rs (rustfs/backlog#2765).
|
||||
#
|
||||
# It copies the working tree, simulates the split (moves real code into
|
||||
# storage/legacy_edge/, app/bucket_usecase/, app/multipart_usecase/ and
|
||||
# app/lifecycle_transition_api_test/), plants one forbidden line per rule in a
|
||||
# split file, and runs the real guard scripts there. Each planted line must be
|
||||
# reported with its split-file path, and nothing in the files that hold only
|
||||
# moved real code or ordinary depth-relative imports may be reported: that is
|
||||
# the "moved and broken -> detected" and "moved -> still passes" proof in one
|
||||
# run per guard. The real workspace is never modified.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${RUSTFS_PYTHON_BIN:-python3}"
|
||||
FX="$(mktemp -d)"
|
||||
trap 'rm -rf "$FX"' EXIT
|
||||
|
||||
failures=0
|
||||
checked=0
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1" >&2
|
||||
failures=$((failures + 1))
|
||||
}
|
||||
|
||||
# expect_reported <output-file> <path> <text> <description>
|
||||
expect_reported() {
|
||||
checked=$((checked + 1))
|
||||
if ! grep -F -- "$2" "$1" | grep -qF -- "$3"; then
|
||||
fail "$4: no report naming $2 with: $3"
|
||||
fi
|
||||
}
|
||||
|
||||
# expect_silent <output-file> <path> <description>
|
||||
expect_silent() {
|
||||
checked=$((checked + 1))
|
||||
if grep -qF -- "$2" "$1"; then
|
||||
fail "$3: $2 was reported: $(grep -F -- "$2" "$1" | head -3)"
|
||||
fi
|
||||
}
|
||||
|
||||
# Copy the tracked and untracked-but-not-ignored files of the working tree.
|
||||
(cd "$ROOT_DIR" && git ls-files -z --cached --others --exclude-standard) |
|
||||
"$PYTHON" -c '
|
||||
import os, shutil, sys
|
||||
root, dest = sys.argv[1], sys.argv[2]
|
||||
for rel in sys.stdin.buffer.read().decode().split("\0"):
|
||||
if not rel:
|
||||
continue
|
||||
src = os.path.join(root, rel)
|
||||
if not os.path.lexists(src):
|
||||
continue
|
||||
out = os.path.join(dest, rel)
|
||||
os.makedirs(os.path.dirname(out), exist_ok=True)
|
||||
shutil.copy2(src, out, follow_symlinks=False)
|
||||
' "$ROOT_DIR" "$FX"
|
||||
|
||||
APP="$FX/rustfs/src/app"
|
||||
STORAGE="$FX/rustfs/src/storage"
|
||||
mkdir -p "$APP/bucket_usecase" "$APP/multipart_usecase" "$APP/lifecycle_transition_api_test" \
|
||||
"$STORAGE/legacy_edge/tests"
|
||||
|
||||
# Real code moves: the TRACE hot span leaves bucket_usecase.rs with its
|
||||
# attributes, and the real import blocks of ecfs.rs and the lifecycle
|
||||
# transition tests are copied into split files.
|
||||
"$PYTHON" - "$FX" <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
fx = Path(sys.argv[1])
|
||||
bu = fx / "rustfs/src/app/bucket_usecase.rs"
|
||||
text = bu.read_text()
|
||||
marker = " pub async fn execute_list_objects_v2("
|
||||
assert text.count(marker) == 1, "execute_list_objects_v2 must be defined once"
|
||||
idx = text.index(marker)
|
||||
start = text.rindex("\n", 0, idx) + 1
|
||||
while True:
|
||||
prev = text.rindex("\n", 0, start - 1) + 1
|
||||
if text[prev:start].lstrip().startswith(("#[", "///")):
|
||||
start = prev
|
||||
else:
|
||||
break
|
||||
end = text.index("\n }\n", idx) + len("\n }\n")
|
||||
(fx / "rustfs/src/app/bucket_usecase/listing.rs").write_text(
|
||||
"impl DefaultBucketUsecase {\n" + text[start:end] + "}\n")
|
||||
bu.write_text(text[:start] + text[end:])
|
||||
|
||||
def imports(path):
|
||||
return "".join(l + "\n" for l in path.read_text().splitlines() if l.startswith("use crate::"))
|
||||
|
||||
(fx / "rustfs/src/storage/legacy_edge/edge_imports.rs").write_text(
|
||||
imports(fx / "rustfs/src/storage/ecfs.rs"))
|
||||
(fx / "rustfs/src/app/lifecycle_transition_api_test/imports.rs").write_text(
|
||||
imports(fx / "rustfs/src/app/lifecycle_transition_api_test.rs"))
|
||||
PY
|
||||
|
||||
# Ordinary depth-relative uses of the parent module in split files: none of
|
||||
# them reaches crate::app or the storage root, so no rule may report them.
|
||||
cat >"$APP/bucket_usecase/ordinary.rs" <<'EOF'
|
||||
use super::{DefaultBucketUsecase, ObjectInfo, ObjectOptions};
|
||||
use super::ECStore;
|
||||
use super::StorageError;
|
||||
fn ordinary() {
|
||||
super::tagging::helper();
|
||||
super::get_server_info();
|
||||
}
|
||||
EOF
|
||||
cp "$APP/bucket_usecase/ordinary.rs" "$APP/multipart_usecase/ordinary.rs"
|
||||
cp "$APP/bucket_usecase/ordinary.rs" "$APP/lifecycle_transition_api_test/ordinary.rs"
|
||||
cat >"$STORAGE/legacy_edge/tests/ordinary.rs" <<'EOF'
|
||||
mod tests {
|
||||
use super::{BucketMetadata, check_preconditions};
|
||||
}
|
||||
EOF
|
||||
|
||||
# One forbidden line per rule, each in a split file.
|
||||
cat >"$STORAGE/legacy_edge/bucket_configs.rs" <<'EOF'
|
||||
use crate::app::context::resolve_region;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::storage::core_storage_compat::Probe;
|
||||
use crate::startup_server::LayerProbe;
|
||||
EOF
|
||||
cat >"$APP/bucket_usecase/cors.rs" <<'EOF'
|
||||
use crate::storage::*;
|
||||
use crate::storage::access::ReqInfo;
|
||||
use super::super::StorageError;
|
||||
use super::super::ECStore;
|
||||
fn forbidden() {
|
||||
super::super::get_server_info();
|
||||
}
|
||||
EOF
|
||||
cat >"$APP/multipart_usecase/complete.rs" <<'EOF'
|
||||
use crate::storage::s3_api::multipart;
|
||||
use super::super::{quota, versioning_sys};
|
||||
use crate::app::usecase_storage_compat::Probe;
|
||||
EOF
|
||||
cat >"$APP/lifecycle_transition_api_test/put.rs" <<'EOF'
|
||||
use crate::storage::request_context::RequestContext;
|
||||
fn forbidden() {
|
||||
super::super::tagging::probe();
|
||||
}
|
||||
EOF
|
||||
cat >"$STORAGE/legacy_edge/tests/object_meta.rs" <<'EOF'
|
||||
use super::super::super::{BucketMetadata};
|
||||
mod tests {
|
||||
use super::super::super::super::{check_preconditions};
|
||||
}
|
||||
use crate::storage::core_storage_compat::TestProbe;
|
||||
EOF
|
||||
|
||||
# --- architecture rules ------------------------------------------------------
|
||||
ARCH_OUT="$FX/.arch.out"
|
||||
if (cd "$FX" && ./scripts/check_architecture_migration_rules.sh) >"$ARCH_OUT" 2>&1; then
|
||||
fail "architecture rules passed over planted violations"
|
||||
fi
|
||||
E=rustfs/src/storage/legacy_edge/bucket_configs.rs
|
||||
B=rustfs/src/app/bucket_usecase/cors.rs
|
||||
M=rustfs/src/app/multipart_usecase/complete.rs
|
||||
L=rustfs/src/app/lifecycle_transition_api_test/put.rs
|
||||
T=rustfs/src/storage/legacy_edge/tests/object_meta.rs
|
||||
expect_reported "$ARCH_OUT" "$E:1:" "storage runtime source reads must stay behind" "runtime-source bypass (ECFS set)"
|
||||
expect_reported "$ARCH_OUT" "$E:2:" "ECFS S3 routes must construct app usecases" "ECFS usecase bypass"
|
||||
expect_reported "$ARCH_OUT" "$E:3:" "must use relative core_storage_compat paths" "core_storage_compat (ECFS set)"
|
||||
expect_reported "$ARCH_OUT" "$B:1:" "instead of crate::storage::*" "storage wildcard (bucket split)"
|
||||
expect_reported "$ARCH_OUT" "$B:2:" "consume storage helper APIs through" "storage helper import (bucket split)"
|
||||
expect_reported "$ARCH_OUT" "$B:3:" "consume storage helper APIs through" "depth-aware super::super storage helper"
|
||||
expect_reported "$ARCH_OUT" "$B:4:" "runtime and data-usage facades must stay behind" "depth-aware super::super runtime facade import"
|
||||
expect_reported "$ARCH_OUT" "$B:6:" "runtime and data-usage facades must stay behind" "depth-aware super::super runtime facade path"
|
||||
expect_reported "$ARCH_OUT" "$M:1:" "consume S3 API helpers through" "s3_api bypass (multipart split)"
|
||||
expect_reported "$ARCH_OUT" "$M:2:" "consume storage helper APIs through" "depth-aware super::super owner import"
|
||||
expect_reported "$ARCH_OUT" "$M:3:" "local compatibility consumers must use relative" "usecase_storage_compat (multipart split)"
|
||||
expect_reported "$ARCH_OUT" "$L:1:" "completed storage helper consumers" "test storage helper (lifecycle split)"
|
||||
expect_reported "$ARCH_OUT" "$L:3:" "consume storage helper APIs through" "depth-aware super::super owner path"
|
||||
expect_reported "$ARCH_OUT" "$T:1:" "storage-owner tests must use storage_api test consumer" "ecfs_test import at file depth"
|
||||
expect_reported "$ARCH_OUT" "$T:3:" "storage-owner tests must use storage_api test consumer" "ecfs_test import in an inline tests module"
|
||||
expect_reported "$ARCH_OUT" "$T:5:" "local compatibility consumers must use relative" "ecfs_test core_storage_compat"
|
||||
for silent in rustfs/src/app/bucket_usecase/ordinary.rs rustfs/src/app/multipart_usecase/ordinary.rs \
|
||||
rustfs/src/app/lifecycle_transition_api_test/ordinary.rs rustfs/src/storage/legacy_edge/tests/ordinary.rs \
|
||||
rustfs/src/app/bucket_usecase/listing.rs rustfs/src/storage/legacy_edge/edge_imports.rs \
|
||||
rustfs/src/app/lifecycle_transition_api_test/imports.rs; do
|
||||
expect_silent "$ARCH_OUT" "$silent" "architecture rules over moved or ordinary code"
|
||||
done
|
||||
|
||||
# --- layer dependencies ------------------------------------------------------
|
||||
LAYER_OUT="$FX/.layer.out"
|
||||
if (cd "$FX" && ./scripts/check_layer_dependencies.sh) >"$LAYER_OUT" 2>&1; then
|
||||
fail "layer guard passed over a planted reverse dependency"
|
||||
fi
|
||||
expect_reported "$LAYER_OUT" "dep|$E|interface->composition|" "crate::startup_server::LayerProbe" "legacy_edge keeps the interface layer"
|
||||
expect_silent "$LAYER_OUT" "dep|rustfs/src/storage/legacy_edge/edge_imports.rs|" "moved ecfs imports stay legal"
|
||||
expect_silent "$LAYER_OUT" "dep|rustfs/src/app/lifecycle_transition_api_test/" "moved lifecycle transition tests stay excluded"
|
||||
expect_silent "$LAYER_OUT" "dep|rustfs/src/app/bucket_usecase/listing.rs|" "moved bucket usecase code keeps the app layer"
|
||||
|
||||
# --- logging hot span --------------------------------------------------------
|
||||
LOG_OUT="$FX/.logging.out"
|
||||
checked=$((checked + 1))
|
||||
if ! (cd "$FX" && ./scripts/check_logging_guardrails.sh) >"$LOG_OUT" 2>&1; then
|
||||
fail "logging guardrails failed after a clean move of the TRACE hot span: $(tail -3 "$LOG_OUT")"
|
||||
fi
|
||||
"$PYTHON" - "$APP/bucket_usecase/listing.rs" <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
path = Path(sys.argv[1])
|
||||
text = path.read_text()
|
||||
marker = '#[instrument(level = "trace"'
|
||||
assert text.count(marker) == 1, "the moved hot span must carry one TRACE attribute"
|
||||
path.write_text(text.replace(marker, '#[instrument(level = "debug"'))
|
||||
PY
|
||||
checked=$((checked + 1))
|
||||
if (cd "$FX" && ./scripts/check_logging_guardrails.sh) >"$LOG_OUT" 2>&1; then
|
||||
fail "logging guardrails passed after the moved hot span lost TRACE"
|
||||
elif ! grep -qF "execute_list_objects_v2 must remain TRACE-only" "$LOG_OUT"; then
|
||||
fail "logging guardrails failed for another reason: $(tail -3 "$LOG_OUT")"
|
||||
fi
|
||||
|
||||
if [[ $failures -gt 0 ]]; then
|
||||
echo "split guard rules probe: $failures of $checked checks failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "split guard rules probe: $checked checks passed"
|
||||
Reference in New Issue
Block a user