Compare commits

..

8 Commits

Author SHA1 Message Date
overtrue a8cecf6462 test(e2e): register verified Darwin test membership 2026-09-05 20:18:52 +08:00
overtrue 980f3abbd3 chore: merge main PR evidence guidance 2026-09-05 19:00:05 +08:00
overtrue e36650827b chore: integrate shared quick checks for E2E validation 2026-09-05 18:59:07 +08:00
overtrue 09c8e10d5e feat(test): verify the E2E server build and source identity 2026-09-05 18:58:01 +08:00
overtrue 0ff03c596c chore: merge main after ECStore compile repair 2026-09-05 18:46:20 +08:00
overtrue a74919db8e fix(ci): reject dependencies on required quick checks 2026-09-05 18:17:13 +08:00
overtrue e77c6f0ca5 fix(ci): install actionlint from its verified release 2026-09-05 17:42:14 +08:00
overtrue 3149411943 fix(ci): share quick checks and lint workflows 2026-09-05 17:37:20 +08:00
41 changed files with 1027 additions and 2394 deletions
+1 -1
View File
@@ -1,2 +1,2 @@
sha256-darwin=a881fd7d3f5cb94654221ca85b8b30cce1b95e608824a55a15339cbc294e6d34
sha256-darwin=85ee9b9e66e916dbf51857298637b6bcd1c23c2b066aaa4c171ce961c2d002b5
sha256-linux=a2933d83dfe74ffa03410a0959333a1c48288b8469ca9f17273d449d7510c24b
+2 -3
View File
@@ -3,10 +3,9 @@
.NOTPARALLEL: pre-commit pre-pr dev-check
.PHONY: setup-hooks
setup-hooks: ## Install the configured pre-commit hooks
setup-hooks: ## Set up git hooks
@echo "🔧 Setting up git hooks..."
pre-commit validate-config
pre-commit install
chmod +x .git/hooks/pre-commit
@echo "✅ Git hooks setup complete!"
.PHONY: doc-paths-check
+1
View File
@@ -38,6 +38,7 @@ script-tests: ## Run shell script tests
./scripts/test_python_bin.sh
./scripts/check_embedded_secrets.sh --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
+1
View File
@@ -98,6 +98,7 @@ runs:
shell: bash
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/test_e2e_binary.py
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py
+14 -10
View File
@@ -582,13 +582,15 @@ jobs:
install-build-packaging-tools: 'false'
- name: Build debug binary
run: cargo build -p rustfs --bins --features e2e-test-hooks
run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
- name: Upload debug binary
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-debug-binary
path: target/debug/rustfs
path: |
target/debug/rustfs
target/debug/rustfs.e2e.json
if-no-files-found: error
retention-days: 1
@@ -620,13 +622,15 @@ jobs:
install-build-packaging-tools: 'false'
- name: Build debug binary with rio-v2
run: cargo build -p rustfs --bins --features rio-v2,e2e-test-hooks
run: python3 scripts/e2e_binary.py build --bins --features rio-v2,e2e-test-hooks
- name: Upload debug binary
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: rustfs-debug-binary-rio-v2
path: target/debug/rustfs
path: |
target/debug/rustfs
target/debug/rustfs.e2e.json
if-no-files-found: error
retention-days: 1
@@ -775,7 +779,7 @@ jobs:
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
run: |
cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
--status-level all --final-status-level all --failure-output final
- name: Upload e2e smoke diagnostics
@@ -811,7 +815,7 @@ jobs:
RUSTFS_TEST_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
RUSTFS_TEST_PORT="${RUSTFS_TEST_PORT}" \
RUSTFS_TEST_LOG="${RUN_ROOT}/rustfs.log" \
./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/e2e-run.sh ./target/debug/rustfs "${RUN_ROOT}/data"
- name: Upload test logs
if: failure()
@@ -913,7 +917,7 @@ jobs:
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
# debug binary; each test spawns its own rustfs server on a random port.
- name: Run e2e full suite
run: cargo nextest run --profile e2e-full -p e2e_test
run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
- name: Upload junit
if: always()
@@ -974,7 +978,7 @@ jobs:
- name: Run end-to-end tests
run: |
s3s-e2e --version
./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
python3 scripts/e2e_binary.py run --features rio-v2,e2e-test-hooks -- ./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
- name: Upload test logs
if: failure()
@@ -1017,7 +1021,7 @@ jobs:
S3_PORT="${S3_PORT}" \
DATA_ROOT="${RUN_ROOT}" \
S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
./scripts/s3-tests/run.sh
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/s3-tests/run.sh
- name: Upload s3 test artifacts
if: always()
@@ -1099,7 +1103,7 @@ jobs:
S3_PORT="${S3_PORT}" \
DATA_ROOT="${RUN_ROOT}" \
S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
./scripts/s3-tests/run.sh
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- ./scripts/s3-tests/run.sh
- name: Upload s3 test artifacts
if: always()
+9 -11
View File
@@ -89,14 +89,10 @@ jobs:
- name: Verify awscurl
run: test -x "$AWSCURL_PATH"
# Build the rustfs binary once up front. The e2e tests spawn it as a
# child process (crates/e2e_test/src/common.rs) and will build it on
# demand otherwise, but a single explicit build avoids several parallel
# nextest test processes racing to build it at once.
# Build once and carry its source/binary identity into the test invocation.
- name: Build rustfs binary
run: |
cargo build -p rustfs --bins
: > target/debug/rustfs.features
python3 scripts/e2e_binary.py build --bins
- name: Verify replication e2e membership
env:
@@ -108,7 +104,7 @@ jobs:
- name: Run replication e2e nightly suite
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
run: cargo nextest run --profile e2e-repl-nightly -p e2e_test
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
- name: Upload nextest junit report
if: always()
@@ -144,8 +140,7 @@ jobs:
- name: Build rustfs binary
run: |
cargo build -p rustfs --bins --features e2e-test-hooks
: > target/debug/rustfs.features
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
- name: Verify cluster fault e2e membership
env:
@@ -157,7 +152,7 @@ jobs:
- name: Run cluster fault e2e nightly suite
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
run: cargo nextest run --profile e2e-nightly -p e2e_test
run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
- name: Upload cluster fault diagnostics
if: always()
@@ -198,6 +193,9 @@ jobs:
sudo apt-get install -y -qq iproute2
ss -tn state CLOSE-WAIT >/dev/null
- name: Build protocol server
run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"
# The suite owns fixed protocol ports and serializes its internal cases.
- name: Verify protocol e2e membership
env:
@@ -210,7 +208,7 @@ jobs:
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
run: >-
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
- name: Upload protocol diagnostics
if: always()
+2 -3
View File
@@ -98,12 +98,11 @@ jobs:
- name: Build current RustFS binary
run: |
cargo build --locked -p rustfs --bin rustfs
: > target/debug/rustfs.features
python3 scripts/e2e_binary.py build
- name: Run upgrade compatibility test
run: |
cargo test --locked -p e2e_test \
python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test \
"upgrade_compatibility_test::${{ matrix.test }}" \
-- --ignored --exact --nocapture
@@ -132,7 +132,7 @@ jobs:
s3api create-bucket --bucket "${RUSTFS_ODM_INTEROP_BUCKET}"
- name: Build the RustFS binary under test
run: cargo build --locked -p rustfs --bins
run: python3 scripts/e2e_binary.py build --bins
# The lane selects tests by module, so a rename would quietly shrink it.
# The committed digest in .config/e2e-odm-interop-selection.txt fails
@@ -143,7 +143,7 @@ jobs:
python3 ./scripts/check_test_wiring.py --check-profile e2e-odm-interop "${NEXTEST_LISTING}"
- name: Run the interop cases against MinIO
run: cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
- name: Build the MinIO interop report
if: always()
@@ -251,7 +251,7 @@ jobs:
- name: Build the RustFS binary under test
if: steps.credentials.outputs.present == 'true'
run: cargo build --locked -p rustfs --bins
run: python3 scripts/e2e_binary.py build --bins
# A filterset that matches nothing is valid, so the count is asserted
# rather than inferred from a green run.
@@ -272,7 +272,7 @@ jobs:
- name: Run the three-case minimum
if: steps.credentials.outputs.present == 'true'
run: |
cargo nextest run --profile e2e-odm-interop -p e2e_test \
python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test \
-E "${CLOUD_CASE_FILTER}" --no-tests=fail
- name: Build the ${{ matrix.provider }} interop report
+3 -3
View File
@@ -3,9 +3,9 @@
repos:
- repo: local
hooks:
- id: rustfs-fmt-check
name: Rust formatting
entry: cargo fmt --all --check
- id: rustfs-dev-check
name: rustfs dev-check
entry: make dev-check
language: system
types: [rust]
pass_filenames: false
+37 -11
View File
@@ -109,17 +109,24 @@ affected boundaries and risks. CI still runs its configured repository gates.
### 🔒 Git Pre-commit Hooks (optional)
The optional hook uses the checked-in `.pre-commit-config.yaml`. Install [pre-commit](https://pre-commit.com/#installation), then run this from the checkout or a linked worktree:
Git hooks are **not** versioned in this repository, so a fresh clone has no
active pre-commit hook. If you add your own `.git/hooks/pre-commit` (a good
choice is a one-liner that runs `make pre-commit`), you can mark it executable
with:
```bash
make setup-hooks
```
The hook runs `cargo fmt --all --check` when staged files include Rust source. It does not compile the workspace or run tests. Fix formatting with `cargo fmt --all`, inspect and stage the result, then commit again.
Or manually:
`pre-commit install` resolves Git's hook directory for linked worktrees and preserves an existing hook in migration mode. If you use `core.hooksPath`, keep that hook manager and integrate `pre-commit run` there; the installer refuses to silently replace that configuration.
```bash
chmod +x .git/hooks/pre-commit
```
A local hook provides early formatting feedback. With or without it, follow the verification tiers in `AGENTS.md`, run relevant behavioral tests, and satisfy the CI merge gates. `make pre-commit` and `make dev-check` remain explicit broader commands.
With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
applicable scoped checks, and reserve `make pre-pr` for broad cross-module
changes whose impact cannot be bounded by those checks.
### 📝 Formatting Configuration
@@ -131,11 +138,31 @@ fn_call_width = 90
single_line_let_else_max_width = 100
```
### 🚫 Commit Prevention
If you set up a pre-commit hook and your code doesn't meet the formatting requirements, the hook will:
1. **Block the commit** and show clear error messages
2. **Provide exact commands** to fix the issues
3. **Guide you through** the resolution process
Example output when formatting fails:
```
❌ Code formatting check failed!
💡 Please run 'cargo fmt --all' to format your code before committing.
🔧 Quick fix:
cargo fmt --all
git add .
git commit
```
### 🔄 Development Workflow
1. **Make your changes**
2. **Format your code**: `make fmt` or `cargo fmt --all`
3. **Select relevant checks** using the validation tier in `AGENTS.md`; use `make pre-commit` when its broader fast gate adds useful coverage
3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
4. **Commit your changes**: `git commit -m "your message"`
5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
6. **Run applicable scoped checks before opening/updating a PR**; consider
@@ -179,12 +206,11 @@ Configure your IDE to:
#### Pre-commit hook not running?
```bash
pre-commit validate-config
pre-commit run --all-files
# Inspect any configured hook manager; do not overwrite it.
git config --get core.hooksPath
# Install if no separate hook manager is configured.
make setup-hooks
# Check if hook is executable
ls -la .git/hooks/pre-commit
# Make it executable if needed
chmod +x .git/hooks/pre-commit
```
#### Formatting issues?
Generated
+13 -31
View File
@@ -315,7 +315,7 @@ dependencies = [
"strum",
"thiserror 2.0.20",
"uuid",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -508,7 +508,7 @@ dependencies = [
"arrow-select",
"flatbuffers",
"lz4_flex",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -2249,8 +2249,8 @@ dependencies = [
"liblzma",
"lz4",
"memchr",
"zstd 0.13.3",
"zstd-safe 7.3.0",
"zstd",
"zstd-safe",
]
[[package]]
@@ -4067,7 +4067,7 @@ dependencies = [
"uuid",
"walkdir",
"zip",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -5971,7 +5971,7 @@ dependencies = [
"lz4",
"snap",
"uuid",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -7658,7 +7658,7 @@ dependencies = [
"snap",
"tokio",
"twox-hash",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -9618,7 +9618,7 @@ dependencies = [
"x509-parser",
"zeroize",
"zip",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -10228,7 +10228,7 @@ dependencies = [
"thiserror 2.0.20",
"walkdir",
"zip",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -10395,7 +10395,7 @@ dependencies = [
"tracing-opentelemetry",
"tracing-subscriber",
"url",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -10985,7 +10985,7 @@ dependencies = [
"transform-stream",
"url",
"windows",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -14095,7 +14095,7 @@ dependencies = [
"typed-path",
"zeroize",
"zopfli",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -14128,16 +14128,7 @@ version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe 7.3.0",
]
[[package]]
name = "zstd"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e"
dependencies = [
"zstd-safe 8.0.0",
"zstd-safe",
]
[[package]]
@@ -14149,15 +14140,6 @@ dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-safe"
version = "8.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.1.0+zstd.1.5.7"
+5 -5
View File
@@ -199,10 +199,10 @@ serde_urlencoded = "0.7.1"
# matching stable releases are not available yet, while previous stable lines
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
# releases.
aes-gcm = { version = "0.11.1" }
argon2 = { version = "0.6.0" }
blake2 = "0.11.0"
chacha20poly1305 = { version = "0.11.0" }
aes-gcm = { version = "=0.11.1" }
argon2 = { version = "=0.6.0" }
blake2 = "=0.11.0"
chacha20poly1305 = { version = "=0.11.0" }
crc-fast = "1.10.0"
hmac = { version = "0.13.0" }
jsonwebtoken = { version = "11.0.0" }
@@ -343,7 +343,7 @@ windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.18" }
zip = "8.6.0"
zstd = "0.14.0"
zstd = "0.13.3"
# Observability and Metrics
metrics = "0.24.6"
-15
View File
@@ -130,21 +130,6 @@ Scanner cycle budget controls:
- timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
- this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
## Remote tier timeout environment variables
- `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS`
- remote tier TCP connect timeout.
- default is `10`.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
- `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS`
- remote tier request timeout through response headers.
- default is `86400` so large transition uploads keep a production-safe budget.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default. Very large values are accepted and act as a correspondingly long budget.
- `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS`
- maximum idle time between remote tier response-body chunks.
- default is `60`; the timer resets only when non-empty body data keeps progressing.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
## Drive timeout environment variables
- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
-32
View File
@@ -137,28 +137,6 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
/// Environment variable for remote tier TCP connect timeout in seconds.
pub const ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS";
/// Default remote tier TCP connect timeout in seconds.
pub const DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS: u64 = 10;
/// Environment variable for the remote tier request timeout in seconds.
///
/// This bounds upload/download request progress through response headers. The
/// default is intentionally large so multi-TiB transition uploads keep their
/// previous production budget while black-hole remotes no longer wait forever.
pub const ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS";
/// Default remote tier request timeout in seconds.
pub const DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS: u64 = 24 * 60 * 60;
/// Environment variable for remote tier response-body idle timeout in seconds.
///
/// The timer is re-armed on every non-empty response-body chunk, so slow but
/// progressing remotes can continue while silent response bodies are cancelled.
pub const ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS";
/// Default remote tier response-body idle timeout in seconds.
pub const DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: u64 = 60;
/// Request the object-transaction fencing contract used by storage-owned
/// cleanup receipts and lock-window optimizations.
///
@@ -834,16 +812,6 @@ mod remote_version_state_tests {
);
}
#[test]
fn remote_tier_timeout_env_names_are_stable() {
assert_eq!(super::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS");
assert_eq!(super::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS");
assert_eq!(
super::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
"RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS"
);
}
#[test]
fn data_movement_part_checksum_gate_uses_stable_environment_names() {
assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");
+33 -43
View File
@@ -1,7 +1,7 @@
# e2e_test
End-to-end test suite for RustFS. Each test spawns a **real `rustfs` binary**
(built on demand from the workspace) and drives it over the network with the
(built and identified before the test invocation) and drives it over the network with the
AWS SDK (`aws-sdk-s3`), raw HTTP (`reqwest` / `awscurl`), or a protocol client
(FTPS / WebDAV / SFTP). This is the black-box integration layer: exhaustive
end-to-end behavior lives here, unit behavior stays in the source crates
@@ -31,32 +31,28 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
## How to run
All commands assume repo root. `cargo test` triggers an on-demand build of the
`rustfs` binary from [`src/common.rs`](src/common.rs) (`rustfs_binary_path`) on
first use — the first invocation is slow, later ones reuse the binary.
All commands assume repo root and Python 3.9 or newer on Linux or macOS. Build the server once through the provenance entry point, then run the test command through the same script:
```bash
# Whole crate (default = ignored tests skipped)
cargo nextest run -p e2e_test
python3 scripts/e2e_binary.py build --features e2e-test-hooks
# Whole crate (ignored tests remain skipped)
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run -p e2e_test
# One module
cargo nextest run -p e2e_test -E 'test(list_objects_v2_pagination_test)'
# PR smoke subset (see "CI smoke subset" below)
cargo nextest run --profile e2e-smoke -p e2e_test
# ILM serial lane — ignored lifecycle tests, single-threaded (mirrors CI)
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run -p e2e_test -E 'test(list_objects_v2_pagination_test)'
# PR smoke subset
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test
```
The protocols suite has its own contract (fixed bind ports 90229301,
single-worker execution, feature-gated scheduling) documented in
[`src/protocols/README.md`](src/protocols/README.md). `RUSTFS_BUILD_FEATURES`
selects which features the spawned binary is built with; leave it unset to run
every protocol entry. Use the exact profile command under
[Troubleshooting](#troubleshooting) for CI-equivalent execution.
`build` records the source contents, HEAD, resolved Cargo features, profile, toolchain, and binary SHA-256 beside the executable in `rustfs.e2e.json`. `run` validates that identity before and after the command, preserves command failures, and removes its temporary run receipt on completion. The Rust harness checks that receipt before starting each server; it never compiles a server inside a test process. Source or binary changes during a run invalidate the result, even when the test command succeeds. Use an isolated worktree and keep it unchanged until the command finishes.
The additional `--features` arguments must match between `build` and `run`; Cargo defaults remain enabled. The wrapper supplies `RUSTFS_BUILD_FEATURES` from Cargo's resolved feature list, including features enabled by `full`. Protocol helpers require a subset of that list. `CARGO_TARGET_DIR` and `--profile release` are supported. An in-workspace target directory must be Git-ignored; tracked files are always included in the source identity. `build --bins` preserves CI lanes that compile all RustFS binary targets. For a downloaded artifact, copy both the executable and its sidecar, then use `run`; do not generate a new identity for an arbitrary prebuilt binary. `CARGO_BIN_EXE_rustfs` cannot override the verified executable.
Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; concurrent wrappers fail immediately. Use a private target directory and do not run ordinary Cargo builds against it while tests are active: Cargo does not honor this marker. Interrupted runs fail and terminate their command group. After an uncatchable kill, inspect the PID recorded in a leftover marker and remove it only after confirming its owner has stopped. Embedded file symlinks are hashed through their target; embedded directory symlinks are rejected because their contents cannot be enumerated safely by this entry point.
The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
### `#[ignore]` semantics
@@ -122,7 +118,7 @@ via `create_s3_client(idx)` / `create_all_clients()`. See
| `wait_for_server_ready` | Poll readiness before issuing requests |
| `create_s3_client` / `create_test_bucket` / `delete_test_bucket` | aws-sdk-s3 client + bucket lifecycle |
| `find_available_port` | Random free port (isolation primitive) |
| `rustfs_binary_path` / `_with_features` | Locate/build the binary; honors `RUSTFS_BUILD_FEATURES` |
| `rustfs_binary_path` / `_with_features` | Verify this run's binary receipt and required feature subset |
| `requested_rustfs_build_features` / `rustfs_build_feature_enabled` | Feature-gate a test to what the binary was built with |
| `execute_awscurl` / `awscurl_post` / `_get` / `_put` / `_delete` / `awscurl_post_sts_form_urlencoded` | Admin/STS API calls via `awscurl`; missing binaries are test failures |
| `replication_fast_env` | Env vars that shrink replication timers (from repl-4); pass to `start_rustfs_server_with_env` |
@@ -185,32 +181,26 @@ the wiring source of truth. Committed test-ID digests under
**Reproduce a CI failure locally** — run the exact profile/lane:
```bash
# Smoke (e2e-tests job) — includes the 20 fast replication tests
cargo nextest run --profile e2e-smoke -p e2e_test
# Full single-node merge/main lane
cargo nextest run --profile e2e-full -p e2e_test
# Cluster fault nightly lane
cargo nextest run --profile e2e-nightly -p e2e_test
# Replication nightly lane; awscurl is required for STS paths
cargo nextest run --profile e2e-repl-nightly -p e2e_test
# Fixed-port protocol nightly lane
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp \
cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
# ILM serial lane
# Smoke, full, and cluster lanes share a server with fault-test hooks.
python3 scripts/e2e_binary.py build --features e2e-test-hooks
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
# Replication nightly uses the default server; awscurl is required for STS.
python3 scripts/e2e_binary.py build
python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
# Protocol nightly owns fixed ports.
python3 scripts/e2e_binary.py build --features ftps,webdav,sftp
python3 scripts/e2e_binary.py run --features ftps,webdav,sftp -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
# The ILM serial lane does not use this server harness.
cargo nextest run -j1 --run-ignored ignored-only -p rustfs-scanner -p rustfs \
-E 'binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))'
# s3s-e2e black box
./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs-e2e-data
```
**Stale binary.** Tests build the `rustfs` binary once and reuse it. To avoid
rebuilding while iterating on tests, `common.rs` reuses an existing binary when
running *inside* the e2e test process even if sources changed
(`can_reuse_inside_e2e`, [`src/common.rs`](src/common.rs) line 98). Downside: if
you changed **server** code, force a rebuild with
`cargo build -p rustfs` (or `touch` a source file outside the reuse window)
before re-running, or CI's freshly built artifact will diverge from your local
one.
**Stale or unverified binary.** Re-run the matching `build` command after changing source or features, then invoke tests through `run`. A missing receipt, copied old executable, or mismatched build identity is a prerequisite failure. Bare Cargo invocations that start a server deliberately fail; unit tests that do not start a server can still run directly.
**Port already in use / orphan processes.** A hard-killed run can leak a
`rustfs` child holding its port. Find and kill it:
+123 -149
View File
@@ -31,7 +31,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serde_json;
use std::ffi::OsStr;
use std::fs as stdfs;
use std::io::ErrorKind;
use std::net::SocketAddr;
@@ -44,7 +43,6 @@ use tokio::net::TcpStream;
use tokio::time::sleep;
use tracing::{error, info, warn};
use uuid::Uuid;
use walkdir::WalkDir;
// Common constants for all E2E tests
pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin";
@@ -365,59 +363,75 @@ fn resolve_rustfs_binary_path(workspace: &Path, configured_target_dir: Option<&P
path
}
/// Resolve the RustFS binary relative to the workspace, optionally requesting build features.
/// Resolve the server verified by `scripts/e2e_binary.py run` for this test invocation.
/// Requested features are a required subset of the server's resolved Cargo features.
pub fn rustfs_binary_path_with_features(requested_features: Option<&str>) -> PathBuf {
if let Some(path) = std::env::var_os("CARGO_BIN_EXE_rustfs") {
return PathBuf::from(path);
}
let requested_features = requested_features.and_then(normalize_rustfs_build_features);
let workspace = workspace_root();
let configured_target_dir = std::env::var_os("CARGO_TARGET_DIR").map(PathBuf::from);
let binary_path = resolve_rustfs_binary_path(&workspace, configured_target_dir.as_deref());
let binary_path = std::env::var_os("CARGO_BIN_EXE_rustfs")
.map(PathBuf::from)
.unwrap_or_else(|| resolve_rustfs_binary_path(&workspace, configured_target_dir.as_deref()));
let receipt_path = std::env::var_os("RUSTFS_E2E_BINARY_RECEIPT").map(PathBuf::from);
receipt_path
.ok_or_else(|| std::io::Error::new(ErrorKind::NotFound, "missing E2E run receipt"))
.and_then(|receipt| verify_e2e_binary_receipt(&receipt, &workspace, &binary_path, requested_features))
.unwrap_or_else(|error| {
panic!(
"E2E server prerequisite failed: {error}. Build with `python3 scripts/e2e_binary.py build --features <features>` and run tests with `python3 scripts/e2e_binary.py run --features <features> -- cargo nextest run ...`"
)
})
}
let features_match = binary_features_match(&binary_path, requested_features.as_deref());
let source_is_newer = workspace_sources_newer_than_binary(&binary_path);
let can_reuse_inside_e2e = running_inside_e2e_test_binary() && requested_features.is_none() && features_match;
if binary_path.is_file() && features_match && (!source_is_newer || can_reuse_inside_e2e) {
if source_is_newer {
warn!(
"RustFS binary at {:?} appears older than workspace sources; reusing it inside cargo test to avoid nested builds",
binary_path
);
}
info!("Using existing RustFS binary at {:?}", binary_path);
return binary_path;
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct E2eBinaryReceipt {
schema: u32,
workspace: PathBuf,
binary: PathBuf,
size: u64,
modified_ns: u128,
features: Vec<String>,
}
fn verify_e2e_binary_receipt(
receipt_path: &Path,
workspace: &Path,
binary_path: &Path,
requested_features: Option<&str>,
) -> std::io::Result<PathBuf> {
let receipt: E2eBinaryReceipt = serde_json::from_slice(&stdfs::read(receipt_path)?)?;
let binary = binary_path.canonicalize()?;
let metadata = binary.metadata()?;
let modified_ns = metadata
.modified()?
.duration_since(std::time::UNIX_EPOCH)
.map_err(std::io::Error::other)?
.as_nanos();
// The runner hashes source and binary before/after the entire suite. Each
// nextest process checks only this invocation's path, features, and file stat.
if receipt.schema != 1
|| receipt.workspace != workspace.canonicalize()?
|| receipt.binary != binary
|| !metadata.is_file()
|| receipt.size != metadata.len()
|| receipt.modified_ns != modified_ns
{
return Err(std::io::Error::new(
ErrorKind::InvalidData,
"E2E server differs from this run's verified binary",
));
}
info!("Building RustFS binary to ensure it's up to date...");
build_rustfs_binary(requested_features.as_deref(), &binary_path);
info!("Using RustFS binary at {:?}", binary_path);
binary_path
}
fn workspace_sources_newer_than_binary(binary_path: &PathBuf) -> bool {
let Ok(binary_meta) = std::fs::metadata(binary_path) else {
return true;
};
let Ok(binary_modified) = binary_meta.modified() else {
return true;
};
let workspace = workspace_root();
let watch_roots = [
workspace.join("Cargo.toml"),
workspace.join("Cargo.lock"),
workspace.join("rustfs"),
workspace.join("crates"),
];
watch_roots.iter().any(|path| path_is_newer_than(binary_modified, path))
}
fn running_inside_e2e_test_binary() -> bool {
std::env::var("CARGO_PKG_NAME").is_ok_and(|value| value == "e2e_test")
if let Some(requested) = requested_features.and_then(normalize_rustfs_build_features)
&& requested
.split(',')
.any(|feature| !receipt.features.iter().any(|actual| actual == feature))
{
return Err(std::io::Error::new(
ErrorKind::InvalidInput,
"E2E server is missing a requested build feature",
));
}
Ok(binary)
}
pub fn requested_rustfs_build_features() -> Option<String> {
@@ -447,96 +461,6 @@ pub fn rustfs_build_feature_enabled(requested_features: Option<&str>, required_f
.any(|feature| feature.eq_ignore_ascii_case(RUSTFS_FULL_FEATURE) || feature.eq_ignore_ascii_case(required_feature))
}
fn rustfs_binary_features_stamp_path(binary_path: &Path) -> PathBuf {
binary_path.with_extension("features")
}
fn binary_features_match(binary_path: &Path, requested_features: Option<&str>) -> bool {
let stamp_path = rustfs_binary_features_stamp_path(binary_path);
let recorded = stdfs::read_to_string(stamp_path)
.ok()
.and_then(|value| normalize_rustfs_build_features(&value));
let requested = requested_features.and_then(normalize_rustfs_build_features);
match requested.as_deref() {
Some(features) => recorded.as_deref() == Some(features),
None => recorded.is_none(),
}
}
fn path_is_newer_than(binary_modified: std::time::SystemTime, path: &Path) -> bool {
if path.is_file() {
return std::fs::metadata(path)
.and_then(|meta| meta.modified())
.map(|modified| modified > binary_modified)
.unwrap_or(false);
}
if !path.is_dir() {
return false;
}
WalkDir::new(path)
.into_iter()
.filter_entry(|entry| {
let name = entry.file_name();
name != OsStr::new("target") && name != OsStr::new(".git")
})
.filter_map(Result::ok)
.filter(|entry| entry.file_type().is_file())
.any(|entry| {
std::fs::metadata(entry.path())
.and_then(|meta| meta.modified())
.map(|modified| modified > binary_modified)
.unwrap_or(false)
})
}
/// Build the RustFS binary using cargo
fn build_rustfs_binary(requested_features: Option<&str>, binary_path: &Path) {
let workspace = workspace_root();
info!("Building RustFS binary from workspace: {:?}", workspace);
let _profile = if cfg!(debug_assertions) {
info!("Building in debug mode");
"dev"
} else {
info!("Building in release mode");
"release"
};
let mut cmd = Command::new("cargo");
cmd.current_dir(&workspace).args(["build", "--bin", "rustfs"]);
if let Some(features) = requested_features {
cmd.arg("--features").arg(features);
info!("Building with features: {}", features);
}
if !cfg!(debug_assertions) {
cmd.arg("--release");
}
info!(
"Executing: cargo build --bin rustfs {}",
if cfg!(debug_assertions) { "" } else { "--release" }
);
let output = cmd.output().expect("Failed to execute cargo build command");
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
panic!("Failed to build RustFS binary. Error: {stderr}");
}
let stamp_path = rustfs_binary_features_stamp_path(binary_path);
if let Err(err) = stdfs::write(&stamp_path, requested_features.unwrap_or_default()) {
warn!("Failed to write RustFS feature stamp {:?}: {}", stamp_path, err);
}
info!("✅ RustFS binary built successfully");
}
fn awscurl_binary_path() -> PathBuf {
std::env::var_os("AWSCURL_PATH")
.map(PathBuf::from)
@@ -2073,16 +1997,66 @@ mod tests {
}
#[test]
fn binary_feature_stamp_matching_uses_normalized_features() {
let binary_path = std::env::temp_dir().join(format!("rustfs-feature-stamp-test-{}", Uuid::new_v4()));
let stamp_path = rustfs_binary_features_stamp_path(&binary_path);
fn explicit_binary_without_run_receipt_is_rejected() {
const CHILD_ENV: &str = "RUSTFS_E2E_RECEIPT_TEST_CHILD";
if std::env::var_os(CHILD_ENV).is_some() {
rustfs_binary_path_with_features(None);
return;
}
let executable = std::env::current_exe().expect("locate isolated test process");
let output = Command::new(&executable)
.args([
"--exact",
"common::tests::explicit_binary_without_run_receipt_is_rejected",
"--nocapture",
])
.env(CHILD_ENV, "1")
.env("CARGO_BIN_EXE_rustfs", &executable)
.env_remove("RUSTFS_E2E_BINARY_RECEIPT")
.output()
.expect("run the missing-receipt scenario with isolated environment variables");
assert!(!output.status.success(), "an explicit binary must not bypass run verification");
assert!(String::from_utf8_lossy(&output.stderr).contains("missing E2E run receipt"));
}
stdfs::write(&stamp_path, " SFTP, ftps ").expect("write feature stamp");
assert!(binary_features_match(&binary_path, Some("sftp,ftps")));
assert!(binary_features_match(&binary_path, Some(" SFTP, FTPS ")));
assert!(!binary_features_match(&binary_path, Some("sftp")));
stdfs::remove_file(stamp_path).ok();
#[test]
fn e2e_run_receipt_rejects_replaced_binary_and_missing_features() {
let directory = std::env::temp_dir().join(format!("rustfs-e2e-receipt-test-{}", Uuid::new_v4()));
stdfs::create_dir(&directory).expect("create receipt fixture");
let binary = directory.join("rustfs");
let receipt = directory.join("receipt.json");
stdfs::write(&binary, "server").expect("write fixture binary");
let metadata = binary.metadata().expect("stat fixture binary");
let record = serde_json::json!({
"schema": 1,
"workspace": directory.canonicalize().expect("canonical workspace"),
"binary": binary.canonicalize().expect("canonical binary"),
"size": metadata.len(),
"modified_ns": metadata.modified().expect("modified time").duration_since(std::time::UNIX_EPOCH).expect("positive timestamp").as_nanos(),
"features": ["default", "full", "ftps", "webdav", "sftp"]
});
stdfs::write(&receipt, serde_json::to_vec(&record).expect("serialize receipt")).expect("write receipt");
verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("sftp,webdav")).expect("resolved feature subset");
verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("full")).expect("full was actually requested");
assert_eq!(
verify_e2e_binary_receipt(&receipt, &directory, &binary, Some("rio-v2"))
.expect_err("full does not enable rio-v2")
.kind(),
ErrorKind::InvalidInput
);
let other = directory.join("old-server");
stdfs::write(&other, "server").expect("write alternate binary");
assert!(verify_e2e_binary_receipt(&receipt, &directory, &other, None).is_err());
stdfs::write(&binary, "different server").expect("replace fixture binary");
assert!(verify_e2e_binary_receipt(&receipt, &directory, &binary, None).is_err());
stdfs::remove_file(&receipt).expect("remove expired receipt");
assert_eq!(
verify_e2e_binary_receipt(&receipt, &directory, &binary, None)
.expect_err("expired receipt")
.kind(),
ErrorKind::NotFound
);
stdfs::remove_dir_all(directory).expect("remove receipt fixture");
}
/// Build a cluster environment struct in-memory (no ports, no processes) so
+3 -5
View File
@@ -17,15 +17,13 @@ Use the canonical CI-equivalent protocol command in the parent
For targeted debugging of the core suite only:
```bash
RUSTFS_BUILD_FEATURES=ftps,webdav,sftp cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
python3 scripts/e2e_binary.py build --features ftps,webdav,sftp
python3 scripts/e2e_binary.py run --features ftps,webdav,sftp -- cargo test --package e2e_test test_protocol_core_suite -- --test-threads=1 --nocapture
```
This targeted command does not cover the full `e2e-protocols` profile.
`RUSTFS_BUILD_FEATURES` controls which features the test rustfs binary is
built with. When this variable is set, the protocol test runner schedules
only entries whose protocol is present in the requested feature list. Leave
it unset to run every protocol entry.
`e2e_binary.py` supplies `RUSTFS_BUILD_FEATURES` from the verified server's resolved Cargo features. The protocol runner schedules only entries present in that feature list; helpers check that their required features are available without rebuilding the server.
`--test-threads=1` is required because every entry spawns a rustfs server
on fixed bind ports.
+3 -4
View File
@@ -167,10 +167,9 @@ pub mod bucket {
idle_guarded_body,
};
pub use crate::bucket::on_demand_migration::{
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome,
MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
decode_continuation_token, source_list_plan,
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
};
pub mod backfill {
pub use crate::bucket::on_demand_migration::backfill::{
@@ -25,13 +25,8 @@ use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use std::time::{Duration, Instant};
/// The continuation-token version used by ordinary progressing pages.
/// The only continuation-token envelope version this build reads and writes.
pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1;
const LIST_THROUGH_PROGRESS_TOKEN_VERSION: u32 = 2;
/// The sixteenth consecutive merged page without a key or new EOF fails.
/// This also bounds legitimate sparse listings; it is not a cycle detector.
pub const MAX_LIST_NO_PROGRESS_PAGES: u8 = 16;
/// Envelope marker. A bucket that is *not* merging hands out the local
/// listing's own marker, so the decoder needs a positive signal before it
@@ -116,10 +111,6 @@ pub struct ListThroughToken {
/// common prefix compares as itself, never as its members.
#[serde(default)]
pub last_key: Option<String>,
/// Consecutive empty truncated merged pages, present only in v2 tokens.
/// Ordinary v1 tokens retain their original serialized shape.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub no_progress: Option<u8>,
}
impl ListThroughToken {
@@ -132,7 +123,6 @@ impl ListThroughToken {
source: source.token,
source_done: source.done,
last_key,
no_progress: None,
}
}
@@ -180,21 +170,7 @@ pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, Lis
return Ok(ListThroughCursor::Local(decoded.to_string()));
}
match value.get("v").and_then(serde_json::Value::as_u64) {
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {
// v1 readers reject this field even when it is null or zero.
if value.get("no_progress").is_some() {
return Err(ListThroughTokenError::Malformed);
}
}
Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => {
if !value
.get("no_progress")
.and_then(serde_json::Value::as_u64)
.is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count))
{
return Err(ListThroughTokenError::Malformed);
}
}
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {}
Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
None => return Err(ListThroughTokenError::Malformed),
}
@@ -312,8 +288,6 @@ pub enum ListPageError {
Empty,
#[error("truncated listing repeats a continuation token")]
Repeated,
#[error("listing exhausted its consecutive no-progress page budget")]
NoProgress(MergeSide),
}
pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> {
@@ -378,7 +352,6 @@ pub struct MergeOutcome {
#[derive(Debug)]
pub struct ListThroughMerger {
max_keys: usize,
no_progress: Option<u8>,
last_key: Option<String>,
local: SideState,
source: SideState,
@@ -398,7 +371,6 @@ impl ListThroughMerger {
};
Self {
max_keys,
no_progress: token.and_then(|token| token.no_progress),
last_key,
local,
source,
@@ -464,18 +436,13 @@ impl ListThroughMerger {
Ok(())
}
/// `issue_progress_tokens` allows a v1 chain to start carrying a budget.
/// An existing v2 budget is always enforced, including on reader-only nodes.
/// Borrowing lets a source failure re-merge the fetched local buffers.
pub fn finish(&self, issue_progress_tokens: bool) -> Result<MergeOutcome, ListPageError> {
pub fn finish(self) -> MergeOutcome {
let Self {
max_keys,
no_progress,
last_key,
local,
source,
} = self;
let max_keys = *max_keys;
// A side with more pages behind it can only be trusted up to the last
// key it handed over: past that horizon the other side's entries could
@@ -541,44 +508,12 @@ impl ListThroughMerger {
let source_left = !source.disabled && (!source_cursor.done || consumed_source < source.entries.len());
let is_truncated = local_left || source_left;
let reached_eof = (!local.start.done && local_cursor.done) || (!source.start.done && source_cursor.done);
let next_no_progress = if !is_truncated || !picks.is_empty() || reached_eof {
None
} else if max_keys == 0 {
// A zero-sized request cannot consume entries. Preserve an existing
// budget without spending it or starting a new one.
*no_progress
} else if issue_progress_tokens || no_progress.is_some() {
let count = no_progress.unwrap_or(0).saturating_add(1);
if count >= MAX_LIST_NO_PROGRESS_PAGES {
// An empty truncated side closes the merge horizon. Local
// failure takes precedence; disabling the source cannot fix it.
let side = if local.more && local.entries.is_empty() {
MergeSide::Local
} else if !source.disabled && source.more && source.entries.is_empty() {
MergeSide::Source
} else {
MergeSide::Local
};
return Err(ListPageError::NoProgress(side));
}
Some(count)
} else {
None
};
let last_key = consumed_key.or_else(|| last_key.clone());
Ok(MergeOutcome {
let last_key = consumed_key.or(last_key);
MergeOutcome {
picks,
is_truncated,
next_token: is_truncated.then(|| {
let mut token = ListThroughToken::new(local_cursor, source_cursor, last_key);
if let Some(count) = next_no_progress {
token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION;
token.no_progress = Some(count);
}
token
}),
})
next_token: is_truncated.then(|| ListThroughToken::new(local_cursor, source_cursor, last_key)),
}
}
}
@@ -706,7 +641,7 @@ mod tests {
.push_page(fetch.side, kept, truncated, next)
.expect("reference provider pages must advance");
}
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(outcome.is_truncated, outcome.next_token.is_some());
if outcome.is_truncated {
assert_ne!(outcome.next_token, token, "every truncated merged page must make progress");
@@ -789,7 +724,7 @@ mod tests {
.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None)
.expect("local EOF is valid");
assert_eq!(merger.next_fetch(), None);
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(outcome.picks.len(), 1);
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
@@ -805,7 +740,6 @@ mod tests {
source: Some("source-1".to_string()),
source_done: false,
last_key: Some("a".to_string()),
no_progress: None,
};
let mut merger = ListThroughMerger::new(1, Some(&resume));
merger.disable_source();
@@ -817,7 +751,7 @@ mod tests {
Some("local-2".to_string()),
)
.expect("local cursor advances");
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("truncated page carries a token");
assert_eq!(token.source.as_deref(), Some("source-1"), "the source cursor must not move");
@@ -896,7 +830,7 @@ mod tests {
.expect("opaque cursor advances regardless of sort order");
}
assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget");
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert!(outcome.picks.is_empty());
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("empty progressing page has a cursor");
@@ -906,7 +840,7 @@ mod tests {
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None)
.expect("source EOF");
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(
outcome.picks,
vec![MergePick {
@@ -953,7 +887,7 @@ mod tests {
Err(ListPageError::Repeated)
);
merger.disable_source();
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(
outcome.picks,
vec![MergePick {
@@ -1045,8 +979,8 @@ mod tests {
let encoded = token.encode();
assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token))));
let bumped = encoded.replace("\"v\":1", "\"v\":3");
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(3)));
let bumped = encoded.replace("\"v\":1", "\"v\":2");
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(2)));
let extra = encoded.replace("{", "{\"x\":1,");
assert_eq!(decode_continuation_token(&extra), Err(ListThroughTokenError::Malformed));
@@ -1058,257 +992,6 @@ mod tests {
assert_eq!(decode_continuation_token(no_version), Err(ListThroughTokenError::Malformed));
}
fn progress_token(count: Option<u8>, local_done: bool, source_done: bool) -> ListThroughToken {
let mut token = ListThroughToken::new(
SideCursor {
token: None,
done: local_done,
},
SideCursor {
token: Some("A".into()),
done: source_done,
},
Some("last-key".into()),
);
if let Some(count) = count {
token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION;
token.no_progress = Some(count);
}
token
}
fn push_empty_pages(merger: &mut ListThroughMerger, side: MergeSide) {
for _ in 0..MAX_LIST_FETCHES_PER_SIDE {
let fetch = merger.next_fetch().expect("empty truncated side must be fetched");
assert_eq!(fetch.side, side);
let next = format!("{}:next", fetch.token.unwrap_or_default());
merger
.push_page(side, vec![], true, Some(next))
.expect("opaque cursor advances");
}
}
#[test]
fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() {
let token = progress_token(None, true, false);
assert_eq!(
token.encode(),
r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
);
for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
let token = progress_token(Some(count), true, false);
assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
}
for version in [1, 2] {
for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
let encoded = format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#);
assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
}
}
for encoded in [
r#"{"t":"odm-list","v":1,"no_progress":1}"#,
r#"{"t":"odm-list","v":2}"#,
r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#,
] {
assert_eq!(decode_continuation_token(encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
}
}
#[test]
fn reader_only_nodes_do_not_start_a_budget_but_mixed_readers_preserve_one() {
let mut token = progress_token(None, true, false);
for _ in 0..MAX_LIST_NO_PROGRESS_PAGES {
let mut merger = ListThroughMerger::new(2, Some(&token));
push_empty_pages(&mut merger, MergeSide::Source);
token = merger
.finish(false)
.expect("reader-only v1 behavior")
.next_token
.expect("truncated cursor");
assert_eq!(token.v, 1);
assert_eq!(token.no_progress, None);
}
for count in 1..=MAX_LIST_NO_PROGRESS_PAGES {
let mut merger = ListThroughMerger::new(2, Some(&token));
push_empty_pages(&mut merger, MergeSide::Source);
assert!(merger.next_fetch().is_none(), "the per-request two-fetch limit stays intact");
let outcome = merger.finish(count % 2 == 1);
if count == MAX_LIST_NO_PROGRESS_PAGES {
assert_eq!(outcome, Err(ListPageError::NoProgress(MergeSide::Source)));
break;
}
token = outcome.expect("budget not exhausted").next_token.expect("truncated cursor");
assert_eq!(token.no_progress, Some(count));
let ListThroughCursor::Merged(decoded) = decode_continuation_token(&token.encode()).expect("round-trip v2") else {
panic!("merged cursor expected");
};
token = *decoded;
}
}
#[test]
fn objects_and_common_prefixes_reset_a_budget_at_the_boundary() {
for entry in [ListEntryKey::object("result"), ListEntryKey::prefix("result/")] {
for issue_tokens in [false, true] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Source, vec![], true, Some("B".into()))
.expect("empty advancing page");
merger
.push_page(MergeSide::Source, vec![entry.clone()], true, Some("C".into()))
.expect("real progress");
let outcome = merger
.finish(issue_tokens)
.expect("real progress does not exhaust the budget");
assert_eq!(
outcome.picks,
vec![MergePick {
side: MergeSide::Source,
index: 0
}]
);
let next = outcome.next_token.expect("source remains truncated");
assert_eq!(next.last_key.as_deref(), Some(entry.name.as_str()));
assert_eq!(next.v, 1);
assert_eq!(next.no_progress, None);
assert!(!next.encode().contains("no_progress"));
}
}
}
#[test]
fn only_a_new_eof_transition_resets_the_empty_page_budget() {
for finished_side in [MergeSide::Local, MergeSide::Source] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
if finished_side == MergeSide::Local {
merger
.push_page(MergeSide::Local, vec![], false, None)
.expect("new local EOF");
push_empty_pages(&mut merger, MergeSide::Source);
} else {
push_empty_pages(&mut merger, MergeSide::Local);
merger
.push_page(MergeSide::Source, vec![], false, None)
.expect("new source EOF");
}
let next = merger
.finish(false)
.expect("new EOF is progress")
.next_token
.expect("other side truncated");
assert_eq!(next.no_progress, None);
assert_eq!(next.v, 1);
assert_eq!(next.local_done, finished_side == MergeSide::Local);
assert_eq!(next.source_done, finished_side == MergeSide::Source);
let mut merger = ListThroughMerger::new(2, Some(&next));
let remaining = if finished_side == MergeSide::Local {
MergeSide::Source
} else {
MergeSide::Local
};
push_empty_pages(&mut merger, remaining);
let next = merger
.finish(true)
.expect("a new budget starts")
.next_token
.expect("truncated");
assert_eq!(next.no_progress, Some(1), "an already-done side cannot reset every page");
}
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger.push_page(MergeSide::Source, vec![], false, None).expect("final EOF");
let outcome = merger.finish(false).expect("EOF succeeds at the budget boundary");
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
}
#[test]
fn filtered_duplicates_cannot_reset_the_no_progress_budget() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
for next in ["B", "C"] {
let entries = [ListEntryKey::object("last-key"), ListEntryKey::object("earlier")]
.into_iter()
.filter(|entry| merger.accepts(&entry.name))
.collect::<Vec<_>>();
assert!(entries.is_empty(), "both provider entries were already consumed");
merger
.push_page(MergeSide::Source, entries, true, Some(next.into()))
.expect("advancing cursor");
}
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source)));
}
#[test]
fn no_progress_is_attributed_to_local_when_source_cannot_unblock_it() {
for source_mode in ["disabled", "done", "empty", "data"] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, source_mode == "done");
let mut merger = ListThroughMerger::new(2, Some(&resume));
if source_mode == "disabled" {
merger.disable_source();
}
push_empty_pages(&mut merger, MergeSide::Local);
match source_mode {
"empty" => push_empty_pages(&mut merger, MergeSide::Source),
"data" => merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("source")], false, None)
.expect("source data"),
_ => {}
}
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Local)), "{source_mode}");
}
}
#[test]
fn source_budget_failure_remerges_local_objects_and_prefixes_without_refetching() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Local, vec![ListEntryKey::object("local")], true, Some("L1".into()))
.expect("local object");
merger
.push_page(MergeSide::Local, vec![ListEntryKey::prefix("prefix/")], true, Some("L2".into()))
.expect("local prefix");
push_empty_pages(&mut merger, MergeSide::Source);
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source)));
merger.disable_source();
assert!(merger.next_fetch().is_none(), "fallback does not perform another fetch");
let outcome = merger.finish(false).expect("local data makes progress");
assert_eq!(
outcome.picks,
vec![
MergePick {
side: MergeSide::Local,
index: 0
},
MergePick {
side: MergeSide::Local,
index: 1
}
]
);
let token = outcome.next_token.expect("remaining local page");
assert_eq!(token.local.as_deref(), Some("L2"));
assert_eq!(token.source.as_deref(), Some("A"));
assert_eq!(token.last_key.as_deref(), Some("prefix/"));
assert_eq!(token.no_progress, None);
assert_eq!(token.v, 1);
}
#[test]
fn a_zero_sized_merge_preserves_an_existing_budget() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(0, Some(&resume));
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], true, Some("B".into()))
.expect("source page");
let outcome = merger.finish(false).expect("a zero-sized request cannot consume entries");
assert!(outcome.picks.is_empty());
assert_eq!(outcome.next_token.expect("unconsumed source").no_progress, resume.no_progress);
}
#[test]
fn a_plain_local_marker_stays_local() {
assert_eq!(
@@ -40,10 +40,9 @@ pub use config::{
SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
};
pub use list_through::{
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome, MergePick,
MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
decode_continuation_token, source_list_plan,
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
};
pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
pub use pull::{
@@ -37,7 +37,7 @@ use crate::services::tier::{
use bytes::Bytes;
use http::StatusCode;
use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_error_response::to_error_response,
@@ -320,27 +320,6 @@ pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Erro
}
}
fn transition_timeout_from_env(env_key: &str, default_secs: u64) -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64(env_key, default_secs))
}
pub(crate) fn transition_client_timeouts_from_env() -> TransitionClientTimeouts {
TransitionClientTimeouts::new(
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
),
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
),
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
),
)
}
/// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
///
/// Credential, bucket, and endpoint validation run in this order because the
@@ -371,7 +350,6 @@ pub(crate) async fn new_s3_compatible_warm_backend(
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let timeouts = transition_client_timeouts_from_env();
let opts = Options {
creds,
secure: u.scheme() == "https",
@@ -384,7 +362,7 @@ pub(crate) async fn new_s3_compatible_warm_backend(
// Run the SSRF guard after the host-presence check so a host-less endpoint
// keeps this constructor's stable error text.
(params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
let client = TransitionClient::new_with_timeouts(&endpoint, opts, params.provider_tag, timeouts).await?;
let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
@@ -26,7 +26,7 @@ use crate::services::tier::{
tier_config::TierS3,
warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options, endpoint_authority, transition_client_timeouts_from_env,
build_transition_put_options, endpoint_authority,
},
};
use http::HeaderMap;
@@ -139,7 +139,6 @@ impl WarmBackendS3 {
} else {
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
}
let timeouts = transition_client_timeouts_from_env();
let opts = Options {
creds,
secure: u.scheme() == "https",
@@ -148,7 +147,7 @@ impl WarmBackendS3 {
..Default::default()
};
let endpoint = endpoint_authority(&u)?;
let client = TransitionClient::new_with_timeouts(&endpoint, opts, tier_type, timeouts).await?;
let client = TransitionClient::new(&endpoint, opts, tier_type).await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
+4 -84
View File
@@ -2452,9 +2452,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let write_quorum = fi.write_quorum(self.default_write_quorum());
let read_quorum = fi.read_quorum(self.default_read_quorum());
// Release the registry guard before recovery and cleanup read it again:
// a queued topology writer would otherwise deadlock those nested reads.
let disks = self.get_disks_internal().await;
let disks = self.disks.read().await;
let disks = disks.clone();
// let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
@@ -6742,87 +6743,6 @@ mod tests {
.await;
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_multipart_releases_disk_snapshot_before_cleanup() {
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-topology-lock-bucket";
let object = "object";
let body = vec![0x65; 4096];
make_bucket_on_all(&disk_stores, bucket).await;
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &body, &ObjectOptions::default()).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
for dir in &temp_dirs {
assert!(
dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
"the test must create real upload staging on every disk"
);
}
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication);
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
// Hold a separate read gate so the real writer queues even when completion
// correctly releases its snapshot guard. Polling Pending proves admission
// to Tokio's write-preferring queue before the cleanup attempts another read.
let read_gate = set_disks.disks.read().await;
let writer = set_disks.disks.write();
tokio::pin!(writer);
assert!(matches!(
futures::poll!(tokio::task::unconstrained(writer.as_mut())),
std::task::Poll::Pending
));
assert!(
set_disks.disks.try_read().is_err(),
"the pending writer must already block new readers before the cleanup resumes"
);
drop(read_gate);
barrier.release();
let writer_guard = tokio::time::timeout(Duration::from_secs(5), writer)
.await
.expect("a queued topology writer must not deadlock with multipart cleanup's disk snapshot");
// A reconnect can publish the same handles; this test isolates admission
// order without changing the disks that contain the committed object.
drop(writer_guard);
tokio::time::timeout(Duration::from_secs(10), complete)
.await
.expect("multipart cleanup must finish after the topology writer releases")
.expect("completion task should not panic")
.expect("completion should preserve the successful object commit");
let mut reader = tokio::time::timeout(
Duration::from_secs(10),
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
)
.await
.expect("GET should finish after completion")
.expect("the completed object should remain readable");
let mut observed_body = Vec::new();
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut observed_body))
.await
.expect("the completed object body should finish streaming")
.expect("the completed object body should be readable");
assert_eq!(observed_body, body);
assert!(matches!(
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
Err(StorageError::InvalidUploadID(..))
));
for dir in &temp_dirs {
assert!(
!dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
"successful completion must remove its upload staging from every disk"
);
}
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_releases_object_lock_before_cleanup_and_keeps_upload_lock() {
+44 -460
View File
@@ -43,10 +43,6 @@ const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer";
const ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT: &str = "Expiration cannot specify both Days and Date";
const ERR_LIFECYCLE_MULTIPLE_TRANSITIONS: &str = "Only one Transition action per lifecycle rule is supported";
const ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS: &str =
"Only one NoncurrentVersionTransition action per lifecycle rule is supported";
const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str =
"'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer";
const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
@@ -365,12 +361,6 @@ impl Lifecycle for BucketLifecycleConfiguration {
{
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS));
}
if expiration.days.is_some() && expiration.date.is_some() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT,
));
}
if let Some(expiration_date) = &expiration.date {
let date = OffsetDateTime::from(expiration_date.clone());
if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
@@ -404,20 +394,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
}
if let Some(transitions) = &r.transitions {
if transitions.len() > 1 {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, ERR_LIFECYCLE_MULTIPLE_TRANSITIONS));
}
for transition in transitions {
TransitionOps::validate(transition)?;
}
}
if let Some(noncurrent_transitions) = &r.noncurrent_version_transitions {
if noncurrent_transitions.len() > 1 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS,
));
}
for transition in noncurrent_transitions {
NoncurrentVersionTransitionOps::validate(transition)?;
}
@@ -492,8 +473,6 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
async fn eval(&self, obj: &ObjectOpts) -> Event {
// A single-object lookup cannot prove how many newer historical versions
// survive. Count-dependent actions wait for the complete-group evaluator.
self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await
}
@@ -557,8 +536,23 @@ impl Lifecycle for BucketLifecycleConfiguration {
return Event::default();
};
if let Some(event) = obj.restored_copy_expiry(now) {
events.push(event);
if let Some(restore_expires) = obj.restore_expires
&& restore_expires.unix_timestamp() != 0
&& now.unix_timestamp() > restore_expires.unix_timestamp()
{
let mut action = IlmAction::DeleteRestoredAction;
if !obj.is_latest {
action = IlmAction::DeleteRestoredVersionAction;
}
events.push(Event {
action,
due: Some(now),
rule_id: "".into(),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
}
if let Some(ref lc_rules) = self.filter_rules(obj).await {
@@ -617,12 +611,17 @@ impl Lifecycle for BucketLifecycleConfiguration {
continue;
}
if !obj.is_latest
&& let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
&& let Some(retain_newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions
&& newer_noncurrent_versions < usize::try_from(retain_newer_noncurrent_versions).unwrap_or(usize::MAX)
{
continue;
}
if !obj.is_latest
&& let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
&& let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
&& noncurrent_version_expiration
.newer_noncurrent_versions
.is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
{
if let Some(successor_mod_time) = obj.successor_mod_time {
let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
@@ -652,11 +651,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
&& let Some(noncurrent_version_transition) = rule
.noncurrent_version_transitions
.as_ref()
.filter(|transitions| transitions.len() == 1)
.and_then(|transitions| transitions.first())
&& noncurrent_version_transition
.newer_noncurrent_versions
.is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
&& let Some(storage_class) = noncurrent_version_transition.storage_class.as_ref()
&& !storage_class.as_str().is_empty()
&& !obj.delete_marker
@@ -740,11 +735,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
if obj.transition_status != TRANSITION_COMPLETE
&& let Some(transition) = rule
.transitions
.as_ref()
.filter(|transitions| transitions.len() == 1)
.and_then(|transitions| transitions.first())
&& let Some(transition) = rule.transitions.as_ref().and_then(|transitions| transitions.first())
&& let Some(storage_class) = transition.storage_class.as_ref()
&& !storage_class.as_str().is_empty()
{
@@ -767,15 +758,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
if !events.is_empty() {
// Eligible expiration takes precedence over transition, even when a
// failed transition has an earlier deadline. Within each action class,
// prefer the earliest deadline using a deterministic total order.
// Select the winning event using a strict total order (MinIO semantics):
// the earliest `due` wins, and ties break toward delete-type actions. A
// missing `due` is treated as UNIX_EPOCH. This replaces a hand-written
// `sort_by` comparator that was not a strict weak ordering (it could return
// `Ordering::Less` for both `(a, b)` and `(b, a)`), which panics on the
// repository toolchain and did not deterministically pick the earliest event.
let event = events
.iter()
.min_by_key(|event| {
(
ilm_action_priority_rank(&event.action),
event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(),
ilm_action_priority_rank(&event.action),
)
})
.cloned()
@@ -1048,27 +1042,6 @@ impl ObjectOpts {
pub fn expired_object_deletemarker(&self) -> bool {
self.delete_marker && self.is_latest && self.num_versions == 1
}
pub(crate) fn restored_copy_expiry(&self, now: OffsetDateTime) -> Option<Event> {
let restore_expires = self.restore_expires?;
// Restore metadata alone does not prove that a durable remote copy exists.
if self.transition_status != TRANSITION_COMPLETE
|| restore_expires.unix_timestamp() == 0
|| now.unix_timestamp() <= restore_expires.unix_timestamp()
{
return None;
}
let action = if self.is_latest {
IlmAction::DeleteRestoredAction
} else {
IlmAction::DeleteRestoredVersionAction
};
expiration_action_has_valid_target(action, self.version_id, self.is_latest, self.delete_marker).then(|| Event {
action,
due: Some(now),
..Default::default()
})
}
}
/// Returns whether an expiry action has enough identity to target the object
@@ -1091,8 +1064,11 @@ pub fn expiration_action_has_valid_target(
}
}
/// Eligible logical expiration takes precedence over transition and restore-copy
/// cleanup. Deadlines break ties within an action class.
/// Total-order rank for lifecycle actions used to break `due` ties.
///
/// Delete-type actions rank before every other action so that, when two events
/// share the same `due`, a delete wins (MinIO semantics). The concrete numeric
/// values only matter relative to each other.
fn ilm_action_priority_rank(action: &IlmAction) -> u8 {
match action {
IlmAction::DeleteAllVersionsAction
@@ -4183,392 +4159,6 @@ mod tests {
assert_eq!(event.action, IlmAction::NoneAction);
}
mod adversarial_regressions {
use super::*;
use s3s::dto::NoncurrentVersionExpiration;
fn run(test: impl std::future::Future<Output = ()>) {
with_default_ilm_process_time(|| {
tokio::runtime::Builder::new_current_thread()
.build()
.expect("lifecycle regression runtime should build")
.block_on(test);
});
}
fn noncurrent_object() -> ObjectOpts {
ObjectOpts {
name: "logs/object".to_string(),
mod_time: Some(datetime!(2020-01-01 00:00:00 UTC)),
successor_mod_time: Some(datetime!(2020-01-02 00:00:00 UTC)),
version_id: Some(Uuid::from_u128(1)),
size: 1024 * 1024,
..Default::default()
}
}
#[test]
#[serial]
fn noncurrent_transition_retains_the_requested_newer_versions() {
run(async {
let mut rule = enabled_rule(None, None, Some("retain-two-hot-versions"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = Arc::new(BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
});
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid noncurrent transition policy");
let objects = (0..4)
.map(|index| ObjectOpts {
mod_time: Some(datetime!(2020-01-05 00:00:00 UTC) - Duration::days(index)),
successor_mod_time: (index > 0).then_some(datetime!(2020-01-06 00:00:00 UTC) - Duration::days(index)),
version_id: Some(Uuid::from_u128(u128::try_from(index + 1).expect("small version index"))),
is_latest: index == 0,
num_versions: 4,
..noncurrent_object()
})
.collect::<Vec<_>>();
let actions = crate::Evaluator::new(lc)
.eval(&objects)
.await
.expect("complete version chain should evaluate")
.into_iter()
.map(|event| event.action)
.collect::<Vec<_>>();
assert_eq!(
actions,
[
IlmAction::NoneAction,
IlmAction::NoneAction,
IlmAction::NoneAction,
IlmAction::TransitionVersionAction
],
"the two newest noncurrent versions must remain in their current storage class"
);
});
}
#[test]
#[serial]
fn noncurrent_transition_checks_count_age_and_single_object_context() {
run(async {
let mut rule = enabled_rule(None, None, Some("retain-two"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(3),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid counted transition");
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
for (newer, expected) in [
(0, IlmAction::NoneAction),
(1, IlmAction::NoneAction),
(2, IlmAction::TransitionVersionAction),
(3, IlmAction::TransitionVersionAction),
] {
assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
}
assert_eq!(
lc.eval_inner(&object, datetime!(2020-01-04 00:00:00 UTC), 2).await.action,
IlmAction::NoneAction,
"the retention count does not replace the age condition"
);
assert_eq!(
lc.eval(&object).await.action,
IlmAction::NoneAction,
"a single-object lookup must not assume a complete version history"
);
for retain in [None, Some(0), Some(-1), Some(i32::MAX)] {
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition exists")[0]
.newer_noncurrent_versions = retain;
let expected = if matches!(retain, None | Some(0)) {
IlmAction::TransitionVersionAction
} else {
IlmAction::NoneAction
};
assert_eq!(lc.eval_inner(&object, now, 2).await.action, expected, "retention: {retain:?}");
}
});
}
#[test]
#[serial]
fn noncurrent_expiration_and_transition_have_independent_retention_counts() {
run(async {
let mut rule = enabled_rule(None, None, Some("independent-counts"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(90),
newer_noncurrent_versions: Some(4),
});
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(30),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid independent retention limits");
let object = noncurrent_object();
let now = datetime!(2020-05-01 00:00:00 UTC);
for (newer, expected) in [
(1, IlmAction::NoneAction),
(2, IlmAction::TransitionVersionAction),
(3, IlmAction::TransitionVersionAction),
(4, IlmAction::DeleteVersionAction),
] {
assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
}
});
}
#[test]
#[serial]
fn expiration_retention_does_not_skip_an_independent_transition() {
run(async {
let mut rule = enabled_rule(None, None, Some("transition-then-expire"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
let transition_only = lc.eval_inner(&object, now, 0).await;
assert_eq!(transition_only.action, IlmAction::TransitionVersionAction);
lc.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(90),
newer_noncurrent_versions: Some(2),
});
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid combined policy");
let combined = lc.eval_inner(&object, now, 0).await;
assert_eq!(combined.action, transition_only.action, "retention limits expiration, not transition");
assert_eq!(combined.storage_class, transition_only.storage_class);
});
}
#[test]
#[serial]
fn current_transition_rejects_multiple_stages_in_any_order() {
run(async {
let mut rule = enabled_rule(None, None, Some("two-current-transitions"));
rule.transitions = Some(vec![
Transition {
date: Some(datetime!(2020-03-01 00:00:00 UTC).into()),
days: None,
storage_class: Some(TransitionStorageClass::from_static("COLD")),
},
Transition {
date: Some(datetime!(2020-01-03 00:00:00 UTC).into()),
days: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
},
]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = ObjectOpts {
is_latest: true,
..noncurrent_object()
};
let now = datetime!(2020-01-10 00:00:00 UTC);
for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
lc.rules[0].status = ExpirationStatus::from_static(status);
for _ in 0..2 {
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("multiple transition stages must be rejected");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_TRANSITIONS);
assert_eq!(
lc.eval_inner(&object, now, 0).await.action,
IlmAction::NoneAction,
"legacy multi-stage configurations must not silently execute their first stage"
);
lc.rules[0]
.transitions
.as_mut()
.expect("transition array is present")
.reverse();
}
}
lc.rules[0]
.transitions
.as_mut()
.expect("transition array is present")
.remove(0);
lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("one stage is supported");
let event = lc.eval_inner(&object, now, 0).await;
assert_eq!(event.action, IlmAction::TransitionAction);
assert_eq!(event.storage_class, "WARM");
});
}
#[test]
#[serial]
fn noncurrent_transition_rejects_multiple_stages_in_any_order() {
run(async {
let mut rule = enabled_rule(None, None, Some("two-noncurrent-transitions"));
rule.noncurrent_version_transitions = Some(vec![
NoncurrentVersionTransition {
noncurrent_days: Some(30),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("COLD")),
},
NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
},
]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
lc.rules[0].status = ExpirationStatus::from_static(status);
for _ in 0..2 {
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("multiple noncurrent transition stages must be rejected");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS);
assert_eq!(
lc.eval_inner(&object, now, 0).await.action,
IlmAction::NoneAction,
"legacy multi-stage configurations must not silently execute their first stage"
);
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition array is present")
.reverse();
}
}
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition array is present")
.remove(0);
lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("one stage is supported");
let event = lc.eval_inner(&object, now, 0).await;
assert_eq!(event.action, IlmAction::TransitionVersionAction);
assert_eq!(event.storage_class, "WARM");
});
}
#[test]
#[serial]
fn expiration_rejects_simultaneous_days_and_date() {
run(async {
let mut lc = BucketLifecycleConfiguration {
rules: vec![enabled_rule(
Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
None,
Some("ambiguous-expiry"),
)],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("a single Days expiration is valid");
lc.rules[0].expiration.as_mut().expect("expiration is present").date =
Some(datetime!(2099-01-01 00:00:00 UTC).into());
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("Days and Date are mutually exclusive; accepting both silently overrides Days");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT);
});
}
#[test]
#[serial]
fn overdue_transition_does_not_starve_permanent_expiration() {
run(async {
let mut rule = enabled_rule(
Some(LifecycleExpiration {
days: Some(90),
..Default::default()
}),
None,
Some("archive-then-delete"),
);
rule.transitions = Some(vec![Transition {
days: Some(30),
date: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid transition and expiration policy");
let object = ObjectOpts {
is_latest: true,
version_id: None,
transition_status: TRANSITION_PENDING.to_string(),
..noncurrent_object()
};
let before_expiration = lc.eval_inner(&object, datetime!(2020-02-15 00:00:00 UTC), 0).await;
assert_eq!(before_expiration.action, IlmAction::TransitionAction);
let overdue = lc.eval_inner(&object, datetime!(2020-05-01 00:00:00 UTC), 0).await;
assert_eq!(
overdue.action,
IlmAction::DeleteAction,
"an unavailable tier must not prevent permanent expiration indefinitely"
);
});
}
}
/// Property-based tests for the rule evaluator (backlog#1148 ilm-14,
/// follow-up to backlog#1030 / rustfs#4455).
///
@@ -4579,7 +4169,7 @@ mod tests {
///
/// * `eval_inner` never panics and is deterministic for a fixed input;
/// * the winning event matches an independently recomputed candidate set:
/// eligible expiration wins over transition, then earliest `due` wins (the
/// earliest `due` wins, ties break toward delete-class actions (the
/// `min_by_key` selection that replaced the rustfs#4455 comparator);
/// * `expected_expiry_time` is monotonically non-decreasing in `days` and
/// always lands on the processing boundary, both at production defaults
@@ -4868,8 +4458,8 @@ mod tests {
/// consider for a live current version under `selection`-shaped rules
/// (expiration and first-transition only, no filters): expiration
/// fires when `now >= due`, transition when `now > due` and the object
/// has not already transitioned. Eligible expiration wins over transition;
/// the earliest deadline wins within the selected action class.
/// has not already transitioned. Selection semantics under test:
/// earliest due wins, ties prefer delete-class.
fn oracle_candidates(lc: &BucketLifecycleConfiguration, obj: &ObjectOpts, now: OffsetDateTime) -> Vec<Candidate> {
let mod_time = obj.mod_time.expect("selection strategy always sets mod_time");
let mut candidates = Vec::new();
@@ -4958,8 +4548,8 @@ mod tests {
/// Differential test of winner selection (the rustfs#4455 fix):
/// for a live current version under randomized expiration and
/// transition rules, `eval_inner`'s winner must carry the
/// earliest expiration from the independently recomputed candidate
/// set, or the earliest transition when no expiration is eligible,
/// minimum `(due, rank)` of the independently recomputed
/// candidate set — earliest due wins, ties prefer delete-class —
/// and must be `NoneAction` exactly when that set is empty.
#[test]
#[serial]
@@ -4988,13 +4578,7 @@ mod tests {
// Oracle and evaluator must observe the same (pinned) time env.
let (event, expected) = with_production_time_env(|| {
let candidates = oracle_candidates(&lc, &obj, now);
let expected = candidates
.iter()
.filter(|(_, rank)| *rank == 0)
.min()
.copied()
.or_else(|| candidates.into_iter().min());
let expected = oracle_candidates(&lc, &obj, now).into_iter().min();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
+7 -93
View File
@@ -116,10 +116,13 @@ impl Evaluator {
break 'top_loop;
}
}
// Restore expiry removes only the temporary local copy; the
// retained logical version and its remote data remain intact.
IlmAction::DeleteAction | IlmAction::DeleteVersionAction if self.is_object_locked(obj) => {
event = obj.restored_copy_expiry(now).unwrap_or_default();
IlmAction::DeleteAction
| IlmAction::DeleteRestoredAction
| IlmAction::DeleteVersionAction
| IlmAction::DeleteRestoredVersionAction
if self.is_object_locked(obj) =>
{
event = Event::default();
}
_ => {}
}
@@ -203,95 +206,6 @@ mod tests {
use super::*;
use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
#[tokio::test]
async fn adversarial_restore_expiry_survives_legal_hold() {
let mut policy = (*latest_expiration_lifecycle()).clone();
policy.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::DISABLED);
let policy = Arc::new(policy);
policy
.validate(&lock_enabled_without_default_retention())
.await
.expect("valid disabled lifecycle rule");
let mut objects = [true, false].map(|is_latest| ObjectOpts {
is_latest,
num_versions: 2,
mod_time: Some(
OffsetDateTime::from_unix_timestamp(if is_latest { 1_200_000 } else { 1_000_000 })
.expect("fixed version timestamp"),
),
successor_mod_time: (!is_latest)
.then(|| OffsetDateTime::from_unix_timestamp(1_200_000).expect("fixed successor timestamp")),
transition_status: crate::TRANSITION_COMPLETE.to_string(),
restore_expires: Some(OffsetDateTime::from_unix_timestamp(2_000_000).expect("fixed expired restore timestamp")),
..current_object_opts(ReplicationStatusType::Completed)
});
let evaluator = Evaluator::new(policy).with_lock_retention(Some(lock_enabled_without_default_retention()));
let expected = [IlmAction::DeleteRestoredAction, IlmAction::DeleteRestoredVersionAction];
let unlocked = evaluator
.eval(&objects)
.await
.expect("unlocked restored versions should evaluate");
assert_eq!(unlocked.iter().map(|event| event.action).collect::<Vec<_>>(), expected);
for object in &mut objects {
object
.user_defined
.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
}
let locked = evaluator
.eval(&objects)
.await
.expect("locked restored versions should evaluate");
assert_eq!(
locked.iter().map(|event| event.action).collect::<Vec<_>>(),
expected,
"expiring a restored local copy preserves the retained logical version and remote object"
);
let mut expiring_policy = (*latest_expiration_lifecycle()).clone();
expiring_policy.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
});
let expiring_evaluator =
Evaluator::new(Arc::new(expiring_policy)).with_lock_retention(Some(lock_enabled_without_default_retention()));
let locked = expiring_evaluator
.eval(&objects)
.await
.expect("locked expired versions should evaluate");
assert_eq!(
locked.iter().map(|event| event.action).collect::<Vec<_>>(),
expected,
"blocked logical expiration must still allow an eligible restore-copy cleanup"
);
for status in [ReplicationStatusType::Pending, ReplicationStatusType::Failed] {
for object in &mut objects {
object.replication_status = status.clone();
}
for evaluator in [&evaluator, &expiring_evaluator] {
let events = evaluator.eval(&objects).await.expect("pending replication should evaluate");
assert!(events.iter().all(|event| event.action == IlmAction::NoneAction));
}
}
for object in &mut objects {
object.replication_status = ReplicationStatusType::Completed;
}
for transition_status in ["", crate::TRANSITION_PENDING, "unknown"] {
for object in &mut objects {
object.transition_status = transition_status.to_string();
}
for evaluator in [&evaluator, &expiring_evaluator] {
let events = evaluator.eval(&objects).await.expect("incomplete transition should evaluate");
assert!(
events.iter().all(|event| event.action == IlmAction::NoneAction),
"restore metadata cannot authorize cleanup without a completed transition"
);
}
}
}
fn expired_marker_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
+48 -204
View File
@@ -120,10 +120,18 @@ impl TransitionClient {
let h = resp.headers().clone();
let mut body = resp.into_body();
let body_vec = if let Some(limit) = max_response_bytes {
self.collect_response_body(resp.into_body(), limit).await?
collect_response_body(body, limit).await?
} else {
self.collect_response_body_unbounded(resp.into_body()).await?
let mut body_vec = Vec::new();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
body_vec
};
Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
}
@@ -135,7 +143,7 @@ mod bounded_response_tests {
use crate::{
api_get_options::GetObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, collect_response_body},
transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body},
};
use http_body_util::Full;
use hyper::body::Bytes;
@@ -167,31 +175,7 @@ mod bounded_response_tests {
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
fn test_options() -> Options {
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
}
}
async fn client_for_endpoint(endpoint: &str, timeouts: TransitionClientTimeouts) -> TransitionClient {
TransitionClient::new_with_timeouts(endpoint, test_options(), "", timeouts)
.await
.expect("fixture client should build")
}
async fn bounded_get_fixture_with_timeouts(
body: &'static [u8],
timeouts: TransitionClientTimeouts,
) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
@@ -225,14 +209,27 @@ mod bounded_response_tests {
stream.write_all(body).await.expect("fixture should write response body");
request
});
let client = client_for_endpoint(&endpoint, timeouts).await;
let client = TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
)
.await
.expect("fixture client should build");
Some((client, request))
}
async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
bounded_get_fixture_with_timeouts(body, TransitionClientTimeouts::default()).await
}
#[tokio::test]
async fn real_transport_accepts_the_exact_closed_range_length() {
let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else {
@@ -295,7 +292,24 @@ mod bounded_response_tests {
.local_addr()
.expect("listener local address should be available")
.to_string();
let client = client_for_endpoint(&endpoint, TransitionClientTimeouts::default()).await;
let client = TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
)
.await
.expect("fixture client should build");
let mut opts = GetObjectOptions::default();
opts.headers
.insert("range".to_string(), "bytes=0-18446744073709551615".to_string());
@@ -312,176 +326,6 @@ mod bounded_response_tests {
.is_err()
);
}
#[tokio::test]
async fn connection_refused_returns_without_waiting_for_the_request_timeout() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
drop(listener);
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(5), Duration::from_secs(1)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let result = tokio::time::timeout(Duration::from_secs(2), client.get_object_inner("bucket", "probe", &opts))
.await
.expect("connection refused should return before the broader request timeout");
assert!(result.is_err(), "connection refused must fail instead of hanging");
}
#[tokio::test]
async fn response_header_stall_returns_timed_out() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_millis(50), Duration::from_secs(1)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let err = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect_err("response header stalls must be bounded");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
#[tokio::test]
async fn response_body_idle_stall_returns_timed_out() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
stream
.write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 7\r\nConnection: close\r\n\r\nRu")
.await
.expect("fixture should write the first body chunk");
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_millis(50)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let err = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect_err("body stalls after partial progress must be bounded");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
#[tokio::test]
async fn response_body_idle_timer_resets_on_progress() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
stream
.write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 7\r\nConnection: close\r\n\r\n")
.await
.expect("fixture should write response headers");
for byte in b"RustFS!" {
stream.write_all(&[*byte]).await.expect("fixture should write body progress");
tokio::time::sleep(Duration::from_millis(20)).await;
}
});
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_millis(10), Duration::from_secs(1), Duration::from_millis(100)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let (_, _, mut reader) = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect("continuous body progress must not be killed by the idle timer");
let mut body = Vec::new();
reader
.read_to_end(&mut body)
.await
.expect("bounded response should be readable");
assert_eq!(body, b"RustFS!");
fixture.await.expect("fixture should join");
}
}
#[derive(Default)]
+10 -82
View File
@@ -27,6 +27,7 @@ use crate::{
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body},
};
use http::{HeaderMap, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
@@ -123,9 +124,14 @@ impl TransitionClient {
}
//let mut list_bucket_result = ListBucketV2Result::default();
let body_vec = self
.collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let mut list_bucket_result = match quick_xml::de::from_str::<ListBucketV2Result>(&String::from_utf8_lossy(&body_vec)) {
Ok(result) => result,
Err(err) => {
@@ -208,9 +214,7 @@ impl TransitionClient {
let resp_status = resp.status();
let headers = resp.headers().clone();
let body = self
.collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let body = collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE).await?;
if resp_status != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(
resp_status,
@@ -424,30 +428,6 @@ fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Er
#[cfg(test)]
mod tests {
use super::*;
use crate::{
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClientTimeouts},
};
use std::time::Duration;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
};
fn timeout_test_options() -> Options {
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
}
}
#[test]
fn list_versions_xml_preserves_versions_and_delete_markers() {
@@ -545,56 +525,4 @@ mod tests {
assert_eq!(parsed.common_prefixes.len(), 1);
assert_eq!(parsed.common_prefixes[0].prefix, "subdir/");
}
#[tokio::test]
async fn list_objects_v2_body_stall_returns_timed_out() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one list request");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 512\r\nConnection: close\r\n\r\n<ListBucketResult><Name>warm")
.await
.expect("fixture should write a partial list response");
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = TransitionClient::new_with_timeouts(
&endpoint,
timeout_test_options(),
"",
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_millis(50)),
)
.await
.expect("fixture client should build");
client
.bucket_loc_cache
.lock()
.expect("location cache should lock")
.set("bucket", "us-east-1");
let err = client
.list_objects_v2_query("bucket", "", "", false, false, "", "", 1, HeaderMap::new())
.await
.expect_err("a stalled ListObjectsV2 body must be bounded");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
}
@@ -18,6 +18,7 @@
#![allow(clippy::all)]
use http::{HeaderMap, HeaderName, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Bytes;
use s3s::S3ErrorCode;
use std::collections::HashMap;
@@ -246,9 +247,14 @@ impl TransitionClient {
// Parse the CreateMultipartUpload response for the UploadId. Returning a
// default (empty) result here made every multipart transition fail at the
// first UploadPart with "UploadID cannot be empty" (rustfs/rustfs#4811).
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::other(e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let initiate_multipart_upload_result =
quick_xml::de::from_str::<InitiateMultipartUploadResult>(&String::from_utf8_lossy(&body_vec))
.map_err(|e| std::io::Error::other(format!("failed to parse CreateMultipartUpload response: {e}")))?;
+9 -3
View File
@@ -19,6 +19,7 @@
#![allow(clippy::all)]
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use rustfs_utils::HashAlgorithm;
@@ -350,9 +351,14 @@ impl TransitionClient {
)
.await?;
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
process_remove_multi_objects_response(
ReaderImpl::Body(Bytes::from(body_vec)),
bucket_name,
+11 -72
View File
@@ -19,6 +19,7 @@
#![allow(clippy::all)]
use http::{HeaderMap, HeaderValue, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
@@ -118,9 +119,14 @@ impl TransitionClient {
let resp_status = resp.status();
let h = resp.headers().clone();
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let resperr = http_resp_to_error_response(resp_status, &h, body_vec, bucket_name, "");
warn!("bucket exists, resperr: {:?}", resperr);
@@ -164,13 +170,11 @@ impl TransitionClient {
let resp_status = resp.status();
let h = resp.headers().clone();
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let body_vec = collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE).await?;
parse_bucket_versioning_response(resp_status, &h, body_vec, bucket_name)
}
Err(err) => Err(err),
Err(err) => Err(std::io::Error::other(err)),
}
}
@@ -270,14 +274,8 @@ impl TransitionClient {
#[cfg(test)]
mod tests {
use super::parse_bucket_versioning_response;
use crate::{
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts},
};
use http::{HeaderMap, StatusCode};
use s3s::dto::BucketVersioningStatus;
use std::time::Duration;
use tokio::{io::AsyncReadExt, net::TcpListener};
#[test]
fn parses_bucket_versioning_statuses_mfa_delete_and_unversioned_state() {
@@ -340,63 +338,4 @@ mod tests {
assert_eq!(strict_err.kind(), std::io::ErrorKind::InvalidData);
}
}
#[tokio::test]
async fn get_bucket_versioning_preserves_request_timeout_kind() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one versioning request");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = TransitionClient::new_with_timeouts(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_millis(50), Duration::from_secs(1)),
)
.await
.expect("fixture client should build");
client
.bucket_loc_cache
.lock()
.expect("location cache should lock")
.set("bucket", "us-east-1");
let err = client
.get_bucket_versioning("bucket")
.await
.expect_err("a stalled versioning request must time out");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
}
+10 -5
View File
@@ -26,6 +26,7 @@ use crate::{
transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
};
use http::Request;
use http_body_util::BodyExt;
use hyper::StatusCode;
use hyper::body::Body;
use hyper::body::Bytes;
@@ -85,7 +86,7 @@ impl TransitionClient {
let req = self.get_bucket_location_request(bucket_name)?;
let mut resp = self.doit(req).await?;
location = process_bucket_location_response(self, resp, bucket_name, &self.tier_type).await?;
location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
{
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
bucket_loc_cache.set(bucket_name, &location);
@@ -197,7 +198,6 @@ impl TransitionClient {
}
async fn process_bucket_location_response(
client: &TransitionClient,
mut resp: http::Response<Incoming>,
bucket_name: &str,
tier_type: &str,
@@ -237,9 +237,14 @@ async fn process_bucket_location_response(
}
//}
let body_vec = client
.collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let mut location = "".to_string();
if tier_type == "huaweicloud" {
if let Ok(body_str) = String::from_utf8(body_vec) {
+41 -328
View File
@@ -41,7 +41,7 @@ use http::{
request::{Builder, Request},
};
use http_body::Body;
use http_body_util::BodyExt;
use http_body_util::{BodyExt, LengthLimitError, Limited};
use hyper::body::Bytes;
use hyper::body::Incoming;
use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
@@ -67,12 +67,10 @@ use s3s::dto::Owner;
use s3s::dto::ReplicationStatus;
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use std::error::Error as StdError;
use std::io::Cursor;
use std::pin::Pin;
use std::sync::atomic::{AtomicI32, Ordering};
use std::task::{Context, Poll};
use std::time::Duration as StdDuration;
use std::{
collections::HashMap,
sync::{Arc, Mutex},
@@ -81,108 +79,28 @@ use time::Duration;
use time::OffsetDateTime;
use tokio::io::BufReader;
use tokio::io::{AsyncRead, AsyncReadExt};
use tracing::{debug, error, trace, warn};
use tracing::{debug, error, warn};
use url::{Url, form_urlencoded};
use uuid::Uuid;
const C_USER_AGENT: &str = "RustFS (linux; x86)";
pub const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024;
const EVENT_TIER_REMOTE_TRANSPORT: &str = "tier_remote_transport";
const LOG_COMPONENT_S3_CLIENT: &str = "s3_client";
const LOG_SUBSYSTEM_TIER: &str = "tier";
const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT];
fn response_body_exceeds_limit_error() -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit")
}
fn remote_tier_timeout_error(message: &'static str) -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::TimedOut, message)
}
fn source_chain_has_io_kind(error: &(dyn StdError + 'static), kind: std::io::ErrorKind) -> bool {
let mut current = Some(error);
while let Some(error) = current {
if error
.downcast_ref::<std::io::Error>()
.is_some_and(|io_error| io_error.kind() == kind)
{
return true;
}
current = error.source();
}
false
}
fn transition_transport_error(err: hyper_util::client::legacy::Error) -> std::io::Error {
if source_chain_has_io_kind(&err, std::io::ErrorKind::TimedOut) {
return remote_tier_timeout_error("remote tier connection timed out");
}
std::io::Error::other(err)
}
async fn next_response_body_data<B>(
mut body: Pin<&mut B>,
idle_timeout: Option<StdDuration>,
) -> Result<Option<Bytes>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
let next_nonempty_data = async {
loop {
let Some(frame) = std::future::poll_fn(|cx| body.as_mut().poll_frame(cx)).await else {
return Ok(None);
};
let frame = frame.map_err(std::io::Error::other)?;
let Ok(data) = frame.into_data() else {
continue;
};
if !data.is_empty() {
return Ok(Some(data));
}
}
};
if let Some(idle_timeout) = idle_timeout {
tokio::time::timeout(idle_timeout, next_nonempty_data)
.await
.map_err(|_| remote_tier_timeout_error("remote tier response body stalled"))?
} else {
next_nonempty_data.await
}
}
async fn collect_response_body_inner<B>(
body: B,
limit: Option<usize>,
idle_timeout: Option<StdDuration>,
) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
let mut body_vec = Vec::new();
let mut body = std::pin::pin!(body);
while let Some(data) = next_response_body_data(body.as_mut(), idle_timeout).await? {
let Some(new_len) = body_vec.len().checked_add(data.len()) else {
return Err(response_body_exceeds_limit_error());
};
if limit.is_some_and(|limit| new_len > limit) {
return Err(response_body_exceeds_limit_error());
}
body_vec.extend_from_slice(&data);
}
Ok(body_vec)
}
pub async fn collect_response_body<B>(body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
{
collect_response_body_inner(body, Some(limit), None).await
let body = Limited::new(body, limit).collect().await.map_err(|err| {
if err.is::<LengthLimitError>() {
std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit")
} else {
std::io::Error::other(err)
}
})?;
Ok(body.to_bytes().to_vec())
}
const C_UNKNOWN: i32 = -1;
@@ -278,62 +196,6 @@ pub struct TransitionClient {
pub trailing_header_support: bool,
pub max_retries: i64,
pub tier_type: String,
pub timeouts: TransitionClientTimeouts,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TransitionClientTimeouts {
pub connect_timeout: StdDuration,
pub request_timeout: StdDuration,
pub response_body_idle_timeout: StdDuration,
}
impl TransitionClientTimeouts {
pub const fn new(
connect_timeout: StdDuration,
request_timeout: StdDuration,
response_body_idle_timeout: StdDuration,
) -> Self {
Self {
connect_timeout,
request_timeout,
response_body_idle_timeout,
}
}
fn validate(self) -> Result<Self, std::io::Error> {
if self.connect_timeout.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"remote tier connect timeout must be greater than zero",
));
}
if self.request_timeout.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"remote tier request timeout must be greater than zero",
));
}
if self.response_body_idle_timeout.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"remote tier response body idle timeout must be greater than zero",
));
}
Ok(self)
}
}
impl Default for TransitionClientTimeouts {
fn default() -> Self {
Self {
connect_timeout: StdDuration::from_secs(rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS),
request_timeout: StdDuration::from_secs(rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS),
response_body_idle_timeout: StdDuration::from_secs(
rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
),
}
}
}
#[derive(Debug, Default)]
@@ -426,28 +288,12 @@ async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
impl TransitionClient {
pub async fn new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
Self::private_new(endpoint, opts, tier_type, TransitionClientTimeouts::default()).await
let client = Self::private_new(endpoint, opts, tier_type).await?;
Ok(client)
}
/// Builds a transition client with explicit transport timeout budgets.
///
/// [`Self::new`] keeps the historical constructor surface and uses the
/// production defaults from [`TransitionClientTimeouts::default`].
pub async fn new_with_timeouts(
endpoint: &str,
opts: Options,
tier_type: &str,
timeouts: TransitionClientTimeouts,
) -> Result<TransitionClient, std::io::Error> {
Self::private_new(endpoint, opts, tier_type, timeouts).await
}
async fn private_new(
endpoint: &str,
opts: Options,
tier_type: &str,
timeouts: TransitionClientTimeouts,
) -> Result<TransitionClient, std::io::Error> {
async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
if rustls::crypto::CryptoProvider::get_default().is_none() {
// No default provider is set yet; try to install aws-lc-rs.
// `install_default` can only fail if another thread races us and installs a provider
@@ -460,19 +306,15 @@ impl TransitionClient {
}
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
let timeouts = timeouts.validate()?;
let tls = build_tls_config().await?;
let mut http = HttpConnector::new();
http.enforce_http(false);
http.set_connect_timeout(Some(timeouts.connect_timeout));
let https = hyper_rustls::HttpsConnectorBuilder::new()
.with_tls_config(tls)
.https_or_http()
.enable_http1()
.enable_http2()
.wrap_connector(http);
.build();
let http_client = Client::builder(TokioExecutor::new()).build(https);
let mut client = TransitionClient {
@@ -495,7 +337,6 @@ impl TransitionClient {
trailing_header_support: opts.trailing_headers,
max_retries: opts.max_retries,
tier_type: tier_type.to_string(),
timeouts,
};
{
@@ -660,43 +501,29 @@ impl TransitionClient {
}
pub async fn doit(&self, req: Request<s3s::Body>) -> Result<Response<Incoming>, std::io::Error> {
let req_method;
let req_uri;
let resp;
let http_client = self.http_client.clone();
let req_method = req.method().clone();
let resp = tokio::time::timeout(self.timeouts.request_timeout, http_client.request(req)).await;
{
req_method = req.method().clone();
req_uri = req.uri().clone();
debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string());
resp = http_client.request(req);
}
let resp = resp.await;
debug!("http_client url: {} {}", req_method, req_uri);
if let Err(err) = resp {
error!("http_client call error: {:?}", err);
return Err(std::io::Error::other(err));
}
let resp = match resp {
Ok(Ok(resp)) => resp,
Ok(Err(err)) => {
let err = transition_transport_error(err);
error!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
error_kind = ?err.kind(),
"remote tier request failed"
);
return Err(err);
}
Err(_) => {
warn!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
timeout_ms = self.timeouts.request_timeout.as_millis(),
"remote tier request timed out before response headers"
);
return Err(remote_tier_timeout_error("remote tier request timed out before response headers"));
}
Ok(r) => r,
Err(_) => return Err(std::io::Error::other("Unexpected error in response")),
};
trace!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
status = %resp.status(),
"remote tier response received"
);
debug!(status = %resp.status(), "remote tier response received");
//let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
//debug!("http_resp_body: {}", String::from_utf8(b).unwrap());
@@ -710,15 +537,7 @@ impl TransitionClient {
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string();
warn!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
status = %status,
request_id,
"remote tier request rejected"
);
warn!(status = %status, request_id, "remote tier request rejected");
}
Ok(resp)
}
@@ -762,9 +581,7 @@ impl TransitionClient {
let resp_status = resp.status();
let h = resp.headers().clone();
let body_vec = self
.collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE)
.await?;
let body_vec = collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE).await?;
let parsed_error =
http_resp_to_error_response(resp_status, &h, body_vec, &metadata.bucket_name, &metadata.object_name);
let routing_region = parsed_error.region;
@@ -818,22 +635,6 @@ impl TransitionClient {
Err(std::io::Error::other("remote tier request did not produce a response"))
}
pub async fn collect_response_body<B>(&self, body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
collect_response_body_inner(body, Some(limit), Some(self.timeouts.response_body_idle_timeout)).await
}
pub async fn collect_response_body_unbounded<B>(&self, body: B) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
collect_response_body_inner(body, None, Some(self.timeouts.response_body_idle_timeout)).await
}
async fn new_request(
&self,
method: &http::Method,
@@ -1703,17 +1504,12 @@ pub struct CreateBucketConfiguration {
mod tests {
use super::{
MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body,
collect_response_body_inner, signer_error_to_io_error, to_object_info_for_provider, validate_header_values,
with_rustls_init_guard,
signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard,
};
use crate::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
use futures::stream;
use http::{HeaderMap, HeaderValue, Request};
use http_body::Frame;
use http_body_util::{Full, StreamBody};
use http::{HeaderMap, HeaderValue};
use http_body_util::Full;
use hyper::body::Bytes;
use std::time::Duration as StdDuration;
use tokio::net::TcpListener;
use uuid::Uuid;
#[tokio::test]
@@ -1744,77 +1540,6 @@ mod tests {
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
#[tokio::test]
async fn empty_data_frames_do_not_reset_the_body_idle_timeout() {
let frames = stream::unfold((), |_| async {
tokio::time::sleep(StdDuration::from_millis(10)).await;
Some((Ok::<_, std::io::Error>(Frame::data(Bytes::new())), ()))
});
let body = StreamBody::new(Box::pin(frames));
let err = tokio::time::timeout(
StdDuration::from_millis(200),
collect_response_body_inner(body, Some(1), Some(StdDuration::from_millis(50))),
)
.await
.expect("the collector should enforce its own body idle timeout")
.expect_err("empty frames must not count as body progress");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
}
#[tokio::test]
async fn public_body_collector_accepts_non_unpin_bodies() {
let body = StreamBody::new(stream::once(async { Ok::<_, std::io::Error>(Frame::data(Bytes::from_static(b"ok"))) }));
let collected = collect_response_body(body, 2)
.await
.expect("the public collector should pin non-Unpin bodies internally");
assert_eq!(collected, b"ok");
}
#[tokio::test]
async fn https_endpoints_reach_the_transport_connector() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let accepted = tokio::spawn(async move {
let (stream, _) = tokio::time::timeout(StdDuration::from_secs(1), listener.accept())
.await
.expect("HTTPS connector should reach the TCP listener")
.expect("fixture should accept the HTTPS connection");
drop(stream);
});
let client = super::TransitionClient::new_with_timeouts(
&endpoint,
super::Options {
secure: true,
..Default::default()
},
"",
super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::from_secs(1), StdDuration::from_secs(1)),
)
.await
.expect("fixture client should build");
let request = Request::builder()
.uri(format!("https://{endpoint}/"))
.body(s3s::Body::empty())
.expect("fixture request should build");
client
.doit(request)
.await
.expect_err("the fixture closes before completing the TLS handshake");
accepted.await.expect("fixture should join");
}
#[test]
fn rustls_guard_converts_panics_to_io_errors() {
let err = with_rustls_init_guard(|| -> Result<(), std::io::Error> { panic!("missing provider") })
@@ -1848,18 +1573,6 @@ mod tests {
assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
}
#[test]
fn transition_timeouts_reject_zero_budgets() {
for timeouts in [
super::TransitionClientTimeouts::new(StdDuration::ZERO, StdDuration::from_secs(1), StdDuration::from_secs(1)),
super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::ZERO, StdDuration::from_secs(1)),
super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::from_secs(1), StdDuration::ZERO),
] {
let err = timeouts.validate().expect_err("zero timeout budgets must fail closed");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
}
}
#[test]
fn validate_header_values_returns_header_name_for_non_utf8_values() {
let mut headers = HeaderMap::new();
-10
View File
@@ -7,16 +7,6 @@ On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** t
The module is on by default (rustfs/backlog#2163); set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off (`rustfs/src/module_switches.rs`). With the switch off, the runtime never intervenes on a read and the admin `PUT` route refuses with `OnDemandMigrationDisabled`. Reads of the configuration and of the status endpoint keep working while the switch is off, so a disabled deployment can still be inspected. The switch only decides whether the module may act at all: a bucket with no `on-demand-migration.json` is never resolved by the runtime and makes no source call, so turning the module on changes nothing for buckets you have not configured.
## List continuation token rollout
`RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false`; unset or invalid boolean values also keep it off. It controls only whether a v1 listing may first issue a v2 continuation token after an empty truncated merged page. Every node with this reader support accepts existing v2 tokens and continues their budget even with the switch off. Ordinary pages that consume an object or common prefix retain the original v1 token shape.
Leave the switch off while deploying v2 reader support to every node that can receive a continuation request, including nodes behind other load-balancer routes. Then set it to `true` in each node's environment and restart those nodes to enable issuance. A v1-only binary rejects v2 with `400 InvalidArgument` before the source-error policy runs; neither `not_found` nor turning off list-through makes that old reader compatible. With issuance still off, a new v1 chain retains the existing limitation: an empty source cursor cycle spanning requests can continue indefinitely. The default rollout does not claim to fix that chain until issuance is enabled.
An active v2 budget rejects the sixteenth consecutive merged page that consumes no new object/common prefix and reaches no new end-of-list state. The first fifteen empty pages can be resumed; with the existing two-fetch-per-side limit, that interval costs at most 32 fetches per side, including the failing request. A key, common prefix, or a newly exhausted side on the sixteenth request succeeds and resets the budget. A side that was already exhausted does not reset it again. This is a resource bound, not proof of a cursor cycle: an unusually long but valid empty source-page chain also reaches the limit. Tokens are unsigned base64 JSON, so this budget applies to clients that continue with the returned token unchanged; replaying or editing a token can reset it, and it is not a malicious-client defense or a global request quota. The two-fetch-per-side request limit and existing source rate limiter still apply. A source failure follows `policy.source_error`: `propagate` returns `424 SourceUnavailable` with `invalid_pagination`; `not_found` returns the fetched local listing with `x-rustfs-on-demand-migration-list: local_only`. A blocking local-side failure returns `InternalError`, without silently discarding local entries.
For rollback, first turn issuance off on every node. Keep v2-capable readers available for outstanding v2 chains: switching issuance off does not erase their budgets, and tokens have no expiration that proves those chains have drained. Route those continuations to compatible readers or have clients explicitly restart their listings before restoring v1-only binaries. Restarting a listing is a new scan and can repeat entries. Do not roll back readers while assuming the issuance switch makes existing v2 tokens disappear.
## Positioning
| Capability | Direction | What it moves | Where the authoritative copy is | When to use it instead |
@@ -29,18 +29,6 @@
Both knobs are read by the RustFS process that owns the replication target, at client build time; restart the server after changing them.
### Remote tier transport timeouts
Remote tier S3-compatible clients use separate transport budgets. These settings do not change bucket or site replication clients.
| Variable | Default | Meaning |
| --- | --- | --- |
| `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS` | `10` | Maximum time to establish the remote tier TCP connection. |
| `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS` | `86400` | Maximum time for a remote tier request to reach response headers. The long default preserves large transition-upload headroom. |
| `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS` | `60` | Maximum time without a non-empty response-body chunk. Empty HTTP/2 frames do not count as progress. |
All three values must be positive integers. Zero fails tier client initialization instead of silently disabling the boundary. An invalid integer is logged and falls back to the default; very large values are accepted and provide a correspondingly long effective budget. The values are read when the tier client is built; recreate or reload the tier configuration after changing them.
## Before changing any of this
Follow the SOP in `docs/postmortems/2026-09-03-replication-checksum-default-regression.md`: inventory the target-side rules the current default satisfies, run the outbound target matrix, and document any new knob here in the same PR.
-10
View File
@@ -22,16 +22,6 @@
| `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` |
| Dual-key internal metadata helpers (`insert_bytes` / `get_bytes`) | `crates/utils/src/http/metadata_compat.rs` |
## Lifecycle rule limits and evaluation
Each lifecycle rule supports at most one `Transition` and one `NoncurrentVersionTransition`. A version can make one initial transition; chaining additional tiers after it reaches `complete` is not supported. Splitting stages across overlapping rules does not enable a transition chain. `PutBucketLifecycleConfiguration` rejects multiple entries in either transition array with `InvalidArgument`, including in disabled rules. Existing stored multi-entry arrays are not executed; replace each with a single intended destination. Independent expiration actions in the rule remain eligible.
`Expiration.Days` and `Expiration.Date` are mutually exclusive. A request containing both is rejected instead of silently selecting the date. When expiration and transition are both eligible, expiration takes precedence; a failed earlier transition does not keep an expired object indefinitely. Deadlines select the earliest action within the same action class.
Noncurrent expiration and transition have independent `NewerNoncurrentVersions` limits. A transition with a positive limit waits for a complete version-group evaluation to establish that enough newer noncurrent versions remain. Single-object evaluation, including the current manual transition and immediate-enqueue paths, conservatively defers these counted transitions to the lifecycle scanner. An unmet expiration retention limit does not suppress a separately eligible transition.
An expired restored local copy can be cleaned up under Object Lock because the retained logical version and remote data remain intact. Cleanup requires a completed transition and still waits for pending or failed replication. The storage layer revalidates the source identity and restore metadata before removing the local copy; restore headers alone do not authorize cleanup.
## Free-version recovery controls
The dedicated free-version recovery loop is enabled by default and is independent of the data scanner and heal switches. Setting `RUSTFS_SCANNER_ENABLED=false` does not stop this repair loop. Set `RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED=false` before process startup to disable only the dedicated persisted-marker walk. That setting does not disable lifecycle workers or prevent another scanner path from discovering a free version, and it can leave remote cleanup markers pending for longer, so use it as a break-glass pressure control rather than a cleanup mechanism.
+13 -316
View File
@@ -26,8 +26,8 @@ use super::storage_api::bucket_usecase::ECStore;
use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
use super::storage_api::bucket_usecase::StorageObjectOptions;
use super::storage_api::bucket_usecase::bucket::on_demand_migration::{
BucketOdmState, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
BucketOdmState, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, MergeSide,
OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan,
};
use super::storage_api::bucket_usecase::bucket::versioning_sys::BucketVersioningSys;
@@ -51,9 +51,6 @@ type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
/// yet, so the only class RustFS can vouch for is the default one.
const SOURCE_STORAGE_CLASS: &str = "STANDARD";
/// Enable only after every node serving continuation requests can read v2.
const ENV_LIST_PROGRESS_TOKENS: &str = "RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS";
/// Concurrent local metadata probes when a versioned bucket has to check
/// source-only keys for a shadowing delete marker.
const DELETE_MARKER_PROBE_CONCURRENCY: usize = 32;
@@ -267,18 +264,7 @@ pub(crate) async fn merged_list_objects_v2(
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some));
}
let issue_progress_tokens = rustfs_utils::get_env_bool(ENV_LIST_PROGRESS_TOKENS, false);
let outcome = match merger.finish(issue_progress_tokens) {
Ok(outcome) => outcome,
Err(ListPageError::NoProgress(MergeSide::Source)) => {
degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?;
merger
.finish(issue_progress_tokens)
.map_err(|error| S3Error::with_message(S3ErrorCode::InternalError, error.to_string()))?
}
Err(error) => return Err(S3Error::with_message(S3ErrorCode::InternalError, error.to_string())),
};
drop(merger);
let outcome = merger.finish();
let mut objects = Vec::with_capacity(outcome.picks.len());
let mut prefixes = Vec::new();
let mut source_only_keys = Vec::new();
@@ -444,8 +430,7 @@ mod tests {
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{
FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig,
SourceCredentials, TlsConfig,
FilterConfig, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, SourceCredentials, TlsConfig,
};
use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response};
use crate::app::storage_api::test::StoragePutObjReader;
@@ -463,7 +448,6 @@ mod tests {
source: Some("source-2".to_string()),
source_done: false,
last_key: Some("k".to_string()),
no_progress: None,
}
}
@@ -531,24 +515,6 @@ mod tests {
assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted));
}
#[test]
fn a_v2_token_keeps_the_local_cursor_when_list_through_is_turned_off() {
let mut resume = token(Some("local-2"), false);
resume.v = 2;
resume.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 1);
let encoded = resume.encode();
let decoded = decode_list_cursor(Some(&encoded)).expect("a v2 envelope decodes");
assert_eq!(decoded.as_ref(), Some(&resume));
assert!(matches!(
local_cursor(Some(&encoded), decoded.as_ref()),
LocalListCursor::Token(Some(local)) if local == "local-2"
));
resume.local_done = true;
let encoded = resume.encode();
let decoded = decode_list_cursor(Some(&encoded)).expect("v2 with local EOF decodes");
assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted));
}
#[test]
fn a_plain_local_token_is_passed_through_and_a_tampered_one_is_rejected() {
assert!(
@@ -585,30 +551,14 @@ mod tests {
/// Serves exactly the scripted S3 pages and joins every connection before
/// returning. A source retry or unexpected operation fails the test.
async fn scripted_list_source(pages: Vec<String>) -> (String, tokio_util::task::AbortOnDropHandle<Vec<String>>) {
let (endpoint, server, _) = list_source(pages.into_iter()).await;
(endpoint, server)
}
async fn list_source(
pages: impl Iterator<Item = String> + Send + 'static,
) -> (
String,
tokio_util::task::AbortOnDropHandle<Vec<String>>,
tokio_util::sync::CancellationToken,
) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind listing source");
let address = listener.local_addr().expect("listing source address");
let stop = tokio_util::sync::CancellationToken::new();
let server_stop = stop.clone();
let server = tokio::spawn(async move {
let mut requests = Vec::new();
for body in pages {
let (mut stream, _) = tokio::select! {
_ = server_stop.cancelled() => break,
accepted = listener.accept() => accepted.expect("accept source listing"),
};
let (mut stream, _) = listener.accept().await.expect("accept source listing");
let mut request = Vec::new();
let mut chunk = [0; 4096];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
@@ -638,7 +588,7 @@ mod tests {
}
requests
});
(format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server), stop)
(format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server))
}
fn source_xml(next: Option<&str>, truncated: bool, key: Option<&str>) -> String {
@@ -666,12 +616,12 @@ mod tests {
}
}
async fn source_policy_input(
endpoint: String,
async fn source_policy_request(
pages: Vec<String>,
policy: SourceErrorPolicy,
resume_source: Option<&str>,
filter_prefix: Option<&str>,
) -> (ListThroughTestState, ListObjectsV2Input) {
) -> (S3Result<S3Response<ListObjectsV2Output>>, Vec<String>) {
let store = shared_gating_ecstore().await;
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
let bucket = format!("odm-list-{}", uuid::Uuid::new_v4().simple());
@@ -688,8 +638,9 @@ mod tests {
)
.await
.expect("seed real local listing");
let (endpoint, server) = scripted_list_source(pages).await;
let sys = OnDemandMigrationSys::get();
let state_guard = ListThroughTestState {
let _state_guard = ListThroughTestState {
bucket: bucket.clone(),
module_enabled: sys.is_module_enabled(),
};
@@ -734,7 +685,6 @@ mod tests {
source: Some(source.into()),
source_done: false,
last_key: None,
no_progress: None,
};
base64_simd::STANDARD.encode_to_string(token.encode().as_bytes())
});
@@ -751,10 +701,6 @@ mod tests {
request_payer: None,
start_after: None,
};
(state_guard, input)
}
async fn execute_source_list(input: ListObjectsV2Input) -> S3Result<S3Response<ListObjectsV2Output>> {
let request = S3Request {
input,
method: http::Method::GET,
@@ -766,23 +712,12 @@ mod tests {
service: None,
trailing_headers: None,
};
tokio::time::timeout(
let result = tokio::time::timeout(
Duration::from_secs(10),
DefaultBucketUsecase::from_global().execute_list_objects_v2(request),
)
.await
.expect("listing must complete within its bounded source budget")
}
async fn source_policy_request(
pages: Vec<String>,
policy: SourceErrorPolicy,
resume_source: Option<&str>,
filter_prefix: Option<&str>,
) -> (S3Result<S3Response<ListObjectsV2Output>>, Vec<String>) {
let (endpoint, server) = scripted_list_source(pages).await;
let (_state_guard, input) = source_policy_input(endpoint, policy, resume_source, filter_prefix).await;
let result = execute_source_list(input).await;
.expect("listing must complete within its bounded source budget");
let requests = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("source connections must finish")
@@ -907,244 +842,6 @@ mod tests {
});
}
#[test]
#[serial_test::serial]
fn list_through_cross_request_empty_cursor_cycle_obeys_policy() {
run_large_stack_test("list-through-cross-request-cursor-cycle", || async {
temp_env::async_with_vars(
[
(ENV_LIST_PROGRESS_TOKENS, Some("true")),
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
("HTTP_PROXY", None),
("HTTPS_PROXY", None),
("ALL_PROXY", None),
("http_proxy", None),
("https_proxy", None),
("all_proxy", None),
("NO_PROXY", Some("*")),
("no_proxy", Some("*")),
],
async {
for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] {
let pages = ["B", "C", "A"].map(|next| source_xml(Some(next), true, None));
let (endpoint, server, stop) = list_source(pages.into_iter().cycle()).await;
let (_state_guard, mut input) = source_policy_input(endpoint, policy, Some("A"), None).await;
let mut seen = std::collections::HashSet::from([input
.continuation_token
.clone()
.expect("the first request resumes source cursor A")]);
let mut client_requests = 0;
let mut empty_pages = 0;
let terminal = tokio::time::timeout(Duration::from_secs(30), async {
loop {
client_requests += 1;
let response = match execute_source_list(input.clone()).await {
Ok(response) => response,
Err(error) => break Err(error),
};
if response.headers.contains_key("x-rustfs-on-demand-migration-list") {
break Ok(response);
}
let output = response.output;
assert!(output.contents.as_ref().is_none_or(Vec::is_empty));
assert!(output.common_prefixes.as_ref().is_none_or(Vec::is_empty));
assert_eq!(output.key_count, Some(0));
assert_eq!(output.is_truncated, Some(true));
let next = output
.next_continuation_token
.expect("a truncated page must carry its cursor");
assert!(
seen.insert(next.clone()),
"a cross-request source cursor cycle must not return an identical empty merged token"
);
empty_pages += 1;
input.continuation_token = Some(next);
}
})
.await
.expect("a source cursor cycle must terminate within a bounded client pagination chain");
assert_eq!(empty_pages, usize::from(MAX_LIST_NO_PROGRESS_PAGES - 1));
assert_eq!(client_requests, usize::from(MAX_LIST_NO_PROGRESS_PAGES));
assert_source_policy_result(terminal, policy);
stop.cancel();
let requests = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("cyclic source server must stop")
.expect("cyclic source server must not panic");
assert_eq!(requests.len(), 2 * client_requests, "the sixteenth empty page exhausts the budget");
for (index, request) in requests.iter().enumerate() {
let source_cursor = ["A", "B", "C"][index % 3];
assert!(
request.contains(&format!("continuation-token={source_cursor}")),
"the real SDK must follow the returned source cursor: {request}"
);
}
}
},
)
.await;
});
}
#[test]
#[serial_test::serial]
fn list_through_default_rollout_continues_v2_without_issuing_it_from_v1() {
run_large_stack_test("list-through-reader-first-rollout", || async {
temp_env::async_with_vars(
[
(ENV_LIST_PROGRESS_TOKENS, None),
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
("HTTP_PROXY", None),
("HTTPS_PROXY", None),
("ALL_PROXY", None),
("http_proxy", None),
("https_proxy", None),
("all_proxy", None),
("NO_PROXY", Some("*")),
("no_proxy", Some("*")),
],
async {
for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] {
let pages = ["B", "C", "A"].map(|next| source_xml(Some(next), true, None));
let (endpoint, server, stop) = list_source(pages.into_iter().cycle()).await;
let (_state_guard, mut input) = source_policy_input(endpoint, policy, Some("A"), None).await;
let original = input.continuation_token.clone();
for _ in 0..3 {
let response = execute_source_list(input.clone()).await.expect("reader-only v1 behavior");
assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
assert_eq!(response.output.key_count, Some(0));
assert_eq!(response.output.is_truncated, Some(true));
let next = response.output.next_continuation_token.expect("resumable empty page");
let raw = base64_simd::STANDARD.decode_to_vec(&next).expect("base64 continuation token");
let decoded = std::str::from_utf8(&raw).expect("JSON token");
let token = decode_list_cursor(Some(decoded)).expect("v1 reader").expect("merged token");
assert_eq!(token.v, 1, "the default rollout cannot begin issuing v2");
assert_eq!(token.no_progress, None);
assert!(!decoded.contains("no_progress"), "ordinary v1 wire shape stays unchanged");
input.continuation_token = Some(next);
}
assert_eq!(input.continuation_token, original, "default rollout retains the known v1 limitation");
let raw = base64_simd::STANDARD
.decode_to_vec(input.continuation_token.as_ref().expect("v1 token"))
.expect("base64 continuation token");
let mut token = decode_list_cursor(Some(std::str::from_utf8(&raw).expect("JSON token")))
.expect("v1 reader")
.expect("merged token");
token.v = 2;
token.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 2);
input.continuation_token = Some(base64_simd::STANDARD.encode_to_string(token.encode().as_bytes()));
let response = execute_source_list(input.clone()).await.expect("reader-only node resumes v2");
assert_eq!(response.output.key_count, Some(0));
assert_eq!(response.output.is_truncated, Some(true));
let next = response.output.next_continuation_token.expect("last allowed empty cursor");
let raw = base64_simd::STANDARD.decode_to_vec(&next).expect("base64 continuation token");
let token = decode_list_cursor(Some(std::str::from_utf8(&raw).expect("JSON token")))
.expect("v2 reader")
.expect("merged token");
assert_eq!(token.v, 2);
assert_eq!(token.no_progress, Some(MAX_LIST_NO_PROGRESS_PAGES - 1));
input.continuation_token = Some(next);
assert_source_policy_result(execute_source_list(input).await, policy);
stop.cancel();
let requests = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("cyclic source server must stop")
.expect("source server must not panic");
assert_eq!(requests.len(), 10, "five handler requests each fetched two source pages");
for (index, request) in requests.iter().enumerate() {
let cursor = ["A", "B", "C"][index % 3];
assert!(request.contains(&format!("continuation-token={cursor}")), "{request}");
}
}
},
)
.await;
});
}
#[test]
#[serial_test::serial]
fn list_through_empty_advancing_pages_resume_across_handler_requests() {
run_large_stack_test("list-through-resumable-empty-pages", || async {
temp_env::async_with_vars(
[
(ENV_LIST_PROGRESS_TOKENS, Some("true")),
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
("HTTP_PROXY", None),
("HTTPS_PROXY", None),
("ALL_PROXY", None),
("http_proxy", None),
("https_proxy", None),
("all_proxy", None),
("NO_PROXY", Some("*")),
("no_proxy", Some("*")),
],
async {
for filter_prefix in [None, Some("photos/2024/")] {
let source_key = filter_prefix.map_or("a-source", |_| "photos/2024/a-source");
let (endpoint, server) = scripted_list_source(vec![
source_xml(Some("A"), true, None),
source_xml(Some("B"), true, None),
source_xml(Some("C"), true, None),
source_xml(None, false, Some(source_key)),
])
.await;
let (_state_guard, mut input) =
source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, filter_prefix).await;
let first = execute_source_list(input.clone())
.await
.expect("valid empty pages must remain resumable");
assert!(!first.headers.contains_key("x-rustfs-on-demand-migration-list"));
assert!(first.output.contents.as_ref().is_none_or(Vec::is_empty));
assert!(first.output.common_prefixes.as_ref().is_none_or(Vec::is_empty));
assert_eq!(first.output.key_count, Some(0));
assert_eq!(first.output.is_truncated, Some(true));
input.continuation_token = Some(first.output.next_continuation_token.expect("empty advancing cursor"));
let second = execute_source_list(input)
.await
.expect("a progressing empty chain must reach its data");
assert!(!second.headers.contains_key("x-rustfs-on-demand-migration-list"));
let output = second.output;
let objects = output
.contents
.unwrap_or_default()
.into_iter()
.map(|object| object.key.expect("listed object key"))
.collect::<Vec<_>>();
let prefixes = output
.common_prefixes
.unwrap_or_default()
.into_iter()
.map(|prefix| prefix.prefix.expect("listed common prefix"))
.collect::<Vec<_>>();
if filter_prefix.is_some() {
assert_eq!(objects, vec!["z-local"]);
assert_eq!(prefixes, vec!["photos/"]);
} else {
assert_eq!(objects, vec!["a-source", "z-local"]);
assert!(prefixes.is_empty());
}
assert_eq!(output.key_count, Some(2));
assert_eq!(output.is_truncated, Some(false));
assert!(output.next_continuation_token.is_none());
let requests = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("finite source connections must finish")
.expect("finite source server must not panic");
assert_eq!(requests.len(), 4);
assert!(!requests[0].contains("continuation-token="));
for (request, cursor) in requests[1..].iter().zip(["A", "B", "C"]) {
assert!(request.contains(&format!("continuation-token={cursor}")), "{request}");
}
}
},
)
.await;
});
}
fn assert_source_policy_result(result: S3Result<S3Response<ListObjectsV2Output>>, policy: SourceErrorPolicy) {
match policy {
SourceErrorPolicy::Propagate => {
+4 -4
View File
@@ -634,8 +634,8 @@ pub(crate) mod bucket {
};
#[cfg(test)]
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig,
PathStyle, Provider, SourceConfig, SourceCredentials, TlsConfig,
BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OnDemandMigrationConfig, PathStyle, Provider, SourceConfig,
SourceCredentials, TlsConfig,
};
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OdmStateError, OnDemandMigrationSys, PolicyConfig,
@@ -643,8 +643,8 @@ pub(crate) mod bucket {
commit_inline, idle_guarded_body,
};
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SourceListPlan, decode_continuation_token, source_list_plan,
ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, MergeSide,
SOURCE_LIST_MAX_RATE_WAIT, SourceListPlan, decode_continuation_token, source_list_plan,
};
}
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Build an identified E2E server and verify it around one test invocation."""
import argparse
from contextlib import contextmanager
import hashlib
import json
import os
from pathlib import Path
import stat
import signal
import subprocess
import sys
import tempfile
ROOT = Path(__file__).resolve().parent.parent
RECEIPT_ENV = "RUSTFS_E2E_BINARY_RECEIPT"
def feature_set(value):
return sorted(set(part.strip() for part in value.split(",") if part.strip()))
def file_hash(path):
digest = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def source_identity():
head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
tracked = subprocess.check_output(["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"], cwd=ROOT)
paths = set(tracked.decode("utf-8").rstrip("\0").split("\0")) - {""}
# RustEmbed consumes ignored console assets as well as tracked Rust sources.
static_dir = ROOT / "rustfs/static"
if static_dir.is_symlink():
raise ValueError("The embedded static directory must not be a symlink")
if static_dir.is_dir():
for path in static_dir.rglob("*"):
if path.is_symlink() and path.is_dir():
raise ValueError(f"Unsupported embedded directory symlink: {path}")
if not path.is_dir():
paths.add(str(path.relative_to(ROOT)))
elif static_dir.exists():
paths.add("rustfs/static")
digest = hashlib.sha256()
digest.update(b"static-present\0" if static_dir.is_dir() else b"static-absent\0")
for name in sorted(paths):
path = ROOT / name
digest.update(name.encode("utf-8") + b"\0")
try:
metadata = path.lstat()
except FileNotFoundError:
digest.update(b"deleted\0")
continue
if stat.S_ISLNK(metadata.st_mode):
digest.update(b"symlink\0" + os.fsencode(os.readlink(path)) + b"\0")
if path.is_dir():
target = path.resolve()
if ROOT not in target.parents:
raise ValueError(f"Directory link escapes the source inventory: {name}")
# Directory aliases such as .claude/skills share already-hashed inputs.
for child in target.rglob("*"):
if child.is_dir() and not child.is_symlink():
continue
if child.is_dir() or str(child.relative_to(ROOT)) not in paths:
raise ValueError(f"Directory link contains an unrecorded input: {child}")
digest.update(b"directory\0" + str(target.relative_to(ROOT)).encode("utf-8") + b"\0")
continue
elif not stat.S_ISREG(metadata.st_mode):
raise ValueError(f"Unsupported build input: {name}")
digest.update(str(metadata.st_mode & 0o111).encode() + b"\0")
digest.update(file_hash(path).encode() + b"\0")
return {"head": head, "sha256": digest.hexdigest()}
def sidecar_path(binary):
return binary.with_name(binary.name + ".e2e.json")
def validate_target_directory(target_dir):
if target_dir == ROOT or target_dir in ROOT.parents:
raise ValueError("CARGO_TARGET_DIR must not contain the source workspace")
if ROOT in target_dir.parents:
ignored = subprocess.run(["git", "check-ignore", "--quiet", "--no-index", str(target_dir.relative_to(ROOT))], cwd=ROOT)
if ignored.returncode != 0:
raise ValueError("An in-workspace CARGO_TARGET_DIR must be Git-ignored; use target/ or an external directory")
@contextmanager
def exclusive_binary(binary):
marker = binary.with_name(binary.name + ".e2e.lock")
try:
descriptor = os.open(marker, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
except FileExistsError as error:
raise ValueError(f"Another E2E build/run owns {marker}; do not share a target directory between concurrent runs") from error
try:
identity = os.fstat(descriptor)
with os.fdopen(descriptor, "w") as lock:
lock.write(f"pid={os.getpid()}\n")
yield
finally:
current = marker.stat()
if (current.st_dev, current.st_ino) != (identity.st_dev, identity.st_ino):
raise ValueError("The E2E ownership marker changed during the command")
marker.unlink()
def terminate_command(process):
if process.poll() is not None:
return
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait()
def build(binary, target_dir, profile, requested, all_bins):
sidecar = sidecar_path(binary)
sidecar.unlink(missing_ok=True)
before = source_identity()
command = ["cargo", "build", "--locked", "-p", "rustfs", "--target-dir", str(target_dir), "--message-format=json-render-diagnostics"]
command.extend(["--bins"] if all_bins else ["--bin", "rustfs"])
if requested:
command.extend(["--features", ",".join(requested)])
if profile == "release":
command.append("--release")
artifact = None
with subprocess.Popen(command, cwd=ROOT, stdout=subprocess.PIPE, text=True, start_new_session=True) as process:
try:
for line in process.stdout:
message = json.loads(line)
if message.get("reason") == "compiler-message":
print(message["message"].get("rendered", ""), end="", file=sys.stderr)
if message.get("reason") == "compiler-artifact" and message.get("target", {}).get("name") == "rustfs" and "bin" in message.get("target", {}).get("kind", []):
artifact = message
if process.wait() != 0:
raise ValueError("RustFS build failed; no E2E identity was recorded")
except BaseException:
terminate_command(process)
raise
if not artifact or Path(artifact.get("executable", "")).resolve() != binary:
raise ValueError("Cargo did not produce the requested RustFS executable")
if source_identity() != before:
raise ValueError("Build inputs changed during compilation; finish preparing embedded assets and rebuild in an isolated worktree")
record = {
"schema": 1,
"source": before,
"requested_features": requested,
"features": sorted(artifact["features"]),
"profile": profile,
"rustc": subprocess.check_output(["rustc", "-Vv"], text=True),
"binary_sha256": file_hash(binary),
}
sidecar.write_text(json.dumps(record, sort_keys=True) + "\n")
print(f"Built E2E server: {binary}\nIdentity: {sidecar}", file=sys.stderr)
def verify(binary, profile, requested):
record = json.loads(sidecar_path(binary).read_text())
if not isinstance(record, dict) or set(record) != {"schema", "source", "requested_features", "features", "profile", "rustc", "binary_sha256"} or type(record["schema"]) is not int or record["schema"] != 1:
raise ValueError("Missing or unsupported E2E binary identity; run the build command")
if not isinstance(record["rustc"], str) or not record["rustc"].strip():
raise ValueError("Missing E2E build toolchain identity")
if record["requested_features"] != requested or record["profile"] != profile:
raise ValueError("E2E binary build features/profile differ from this test invocation")
if not isinstance(record["features"], list) or not all(isinstance(item, str) for item in record["features"]) or not set(requested) <= set(record["features"]):
raise ValueError("Invalid resolved E2E binary features")
if record["source"] != source_identity():
raise ValueError("E2E binary was built from different inputs; rebuild before testing")
if record["binary_sha256"] != file_hash(binary):
raise ValueError("E2E binary content differs from its build identity")
return record
def run(binary, profile, requested, command):
if not command:
raise ValueError("run requires a test command after --")
override = os.environ.get("CARGO_BIN_EXE_rustfs")
if override and Path(override).resolve() != binary:
raise ValueError("CARGO_BIN_EXE_rustfs selects a different server; use --binary explicitly")
record = verify(binary, profile, requested)
metadata = binary.stat()
with tempfile.TemporaryDirectory(prefix="rustfs-e2e-receipt-") as directory:
receipt = Path(directory) / "receipt.json"
receipt.write_text(json.dumps({
"schema": 1,
"workspace": str(ROOT),
"binary": str(binary),
"size": metadata.st_size,
"modified_ns": metadata.st_mtime_ns,
"features": record["features"],
}))
env = dict(os.environ, CARGO_BIN_EXE_rustfs=str(binary), RUSTFS_BUILD_FEATURES=",".join(record["features"]))
env[RECEIPT_ENV] = str(receipt)
with subprocess.Popen(command, cwd=ROOT, env=env, start_new_session=True) as process:
try:
status = process.wait()
except (KeyboardInterrupt, SystemExit):
terminate_command(process)
raise
try:
if verify(binary, profile, requested) != record:
raise ValueError("E2E build identity changed during testing")
except (OSError, ValueError, subprocess.SubprocessError) as error:
print(f"E2E validation invalidated: {error}", file=sys.stderr)
return status if status else 1
return status
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=("build", "run"))
parser.add_argument("--features", default="", help="additional Cargo features; defaults remain enabled")
parser.add_argument("--profile", choices=("debug", "release"), default="debug")
parser.add_argument("--binary", type=Path, help="prebuilt server path for run")
parser.add_argument("--bins", action="store_true", help="build all RustFS binary targets, preserving the CI build matrix")
# Parse the child command separately so its options are never interpreted here.
args = sys.argv[1:]
separator = args.index("--") if "--" in args else len(args)
command = args[separator + 1:] if separator < len(args) else []
options = parser.parse_args(args[:separator])
target_dir = Path(os.environ.get("CARGO_TARGET_DIR", ROOT / "target")).resolve()
binary = (options.binary or target_dir / options.profile / ("rustfs.exe" if os.name == "nt" else "rustfs")).resolve()
try:
validate_target_directory(target_dir)
requested = feature_set(options.features)
if options.mode == "build":
binary.parent.mkdir(parents=True, exist_ok=True)
with exclusive_binary(binary):
if options.mode == "build":
if options.binary or command:
raise ValueError("build does not accept --binary or a child command")
build(binary, target_dir, options.profile, requested, options.bins)
return 0
if options.bins:
raise ValueError("--bins is a build option")
return run(binary, options.profile, requested, command)
except (OSError, ValueError, subprocess.SubprocessError) as error:
print(f"E2E prerequisite failed: {error}", file=sys.stderr)
return 1
if __name__ == "__main__":
signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(128 + signum))
raise SystemExit(main())
+12 -3
View File
@@ -14,7 +14,12 @@ NC='\033[0m' # No Color
# Default values
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TARGET_DIR="$PROJECT_ROOT/target/debug"
CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$PROJECT_ROOT/target}"
if [[ "$CARGO_TARGET_DIR" != /* ]]; then
CARGO_TARGET_DIR="$PROJECT_ROOT/$CARGO_TARGET_DIR"
fi
export CARGO_TARGET_DIR
TARGET_DIR="$CARGO_TARGET_DIR/debug"
RUSTFS_BINARY="$TARGET_DIR/rustfs"
DATA_DIR="$TARGET_DIR/rustfs_test_data"
RUSTFS_PID=""
@@ -94,7 +99,7 @@ build_rustfs() {
print_info "Building RustFS..."
cd "$PROJECT_ROOT"
if ! cargo build --bin rustfs --features "$RUSTFS_BUILD_FEATURES"; then
if ! python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"; then
print_error "Failed to build RustFS"
exit 1
fi
@@ -115,6 +120,10 @@ check_dependencies() {
missing_tools+=("curl")
fi
if ! command -v python3 >/dev/null 2>&1; then
missing_tools+=("python3")
fi
if ! command -v cargo >/dev/null 2>&1; then
missing_tools+=("cargo")
fi
@@ -203,7 +212,7 @@ run_tests() {
print_info "Test command: ${test_cmd[*]}"
if "${test_cmd[@]}"; then
if python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- "${test_cmd[@]}"; then
print_success "All tests passed!"
return 0
else
+13 -12
View File
@@ -243,9 +243,10 @@ run_quick_e2e_steps() {
return
fi
run_step "e2e-reliability-disk-fault" cargo test --package e2e_test reliability_disk_fault_test -- --nocapture
run_step "e2e-heal-erasure-disk-rebuild" cargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture
run_step "e2e-namespace-lock-quorum" cargo test --package e2e_test namespace_lock_quorum_test -- --nocapture
run_step "build-e2e-server" python3 scripts/e2e_binary.py build
run_step "e2e-reliability-disk-fault" python3 scripts/e2e_binary.py run -- cargo test --package e2e_test reliability_disk_fault_test -- --nocapture
run_step "e2e-heal-erasure-disk-rebuild" python3 scripts/e2e_binary.py run -- cargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture
run_step "e2e-namespace-lock-quorum" python3 scripts/e2e_binary.py run -- cargo test --package e2e_test namespace_lock_quorum_test -- --nocapture
}
run_quick_profile() {
@@ -313,15 +314,15 @@ write_blackbox_matrix() {
{
printf 'profile\tscenario\tgate\tcommand\tfixture_env\tstatus\n'
printf 'quick\tsingle-node disk fault read/write\tblack-box\tcargo test --package e2e_test reliability_disk_fault_test -- --nocapture\tnone\t%s\n' "$e2e_status"
printf 'quick\theal degraded erasure disk rebuild\tblack-box\tcargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture\tnone\t%s\n' "$e2e_status"
printf 'quick\tnamespace lock quorum under EC ops\tblack-box\tcargo test --package e2e_test namespace_lock_quorum_test -- --nocapture\tnone\t%s\n' "$e2e_status"
printf 'quick\tsingle-node disk fault read/write\tblack-box\tpython3 scripts/e2e_binary.py run -- cargo test --package e2e_test reliability_disk_fault_test -- --nocapture\tnone\t%s\n' "$e2e_status"
printf 'quick\theal degraded erasure disk rebuild\tblack-box\tpython3 scripts/e2e_binary.py run -- cargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture\tnone\t%s\n' "$e2e_status"
printf 'quick\tnamespace lock quorum under EC ops\tblack-box\tpython3 scripts/e2e_binary.py run -- cargo test --package e2e_test namespace_lock_quorum_test -- --nocapture\tnone\t%s\n' "$e2e_status"
printf 'full\tlegacy bitrot read fixture restore\tfixture\tcargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture\tRUSTFS_LEGACY_TEST_ROOT,RUSTFS_LEGACY_TEST_DISK\t%s\n' "$legacy_status"
printf 'full\tMinIO generated encrypted read and negative restore fixture\tfixture\tcargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture\tRUSTFS_MINIO_FIXTURE_ROOT,RUSTFS_MINIO_STATIC_KMS_KEY_B64\t%s\n' "$minio_status"
printf 'full\tS3 multipart range versioning delete subset\tblack-box\tenv TESTEXPR=\"multipart or range or versioning or delete\" DEPLOY_MODE=build MAXFAIL=0 ./scripts/s3-tests/run.sh\tnone\t%s\n' "$s3_status"
printf 'destructive\tdistributed cluster concurrency\tblack-box\tcargo test --package e2e_test cluster_concurrency_test -- --nocapture\tnone\t%s\n' "$destructive_status"
printf 'destructive\tstale multipart cleanup cluster\tblack-box\tcargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture\tnone\t%s\n' "$destructive_status"
printf 'destructive\tdelete marker migration semantics\tblack-box\tcargo test --package e2e_test delete_marker_migration_semantics_test -- --nocapture\tnone\t%s\n' "$destructive_status"
printf 'destructive\tdistributed cluster concurrency\tblack-box\tpython3 scripts/e2e_binary.py run -- cargo test --package e2e_test cluster_concurrency_test -- --nocapture\tnone\t%s\n' "$destructive_status"
printf 'destructive\tstale multipart cleanup cluster\tblack-box\tpython3 scripts/e2e_binary.py run -- cargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture\tnone\t%s\n' "$destructive_status"
printf 'destructive\tdelete marker migration semantics\tblack-box\tpython3 scripts/e2e_binary.py run -- cargo test --package e2e_test delete_marker_migration_semantics_test -- --nocapture\tnone\t%s\n' "$destructive_status"
} >"$BLACKBOX_MATRIX"
}
@@ -566,9 +567,9 @@ run_destructive_profile() {
return
fi
run_step "e2e-cluster-concurrency" cargo test --package e2e_test cluster_concurrency_test -- --nocapture
run_step "e2e-stale-multipart-cleanup-cluster" cargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture
run_step "e2e-delete-marker-migration-semantics" cargo test --package e2e_test delete_marker_migration_semantics_test -- --nocapture
run_step "e2e-cluster-concurrency" python3 scripts/e2e_binary.py run -- cargo test --package e2e_test cluster_concurrency_test -- --nocapture
run_step "e2e-stale-multipart-cleanup-cluster" python3 scripts/e2e_binary.py run -- cargo test --package e2e_test stale_multipart_cleanup_cluster_test -- --nocapture
run_step "e2e-delete-marker-migration-semantics" python3 scripts/e2e_binary.py run -- cargo test --package e2e_test delete_marker_migration_semantics_test -- --nocapture
}
run_fuzz_profile() {
+263
View File
@@ -0,0 +1,263 @@
#!/usr/bin/env python3
"""Exercise the E2E build/run boundary without compiling RustFS."""
import json
import os
from pathlib import Path
import shutil
import signal
import subprocess
import sys
import tempfile
import unittest
class BinaryProvenanceTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
(self.root / "scripts").mkdir()
shutil.copy(Path(__file__).with_name("e2e_binary.py"), self.root / "scripts/e2e_binary.py")
(self.root / "Cargo.toml").write_text("[workspace]\n")
(self.root / "source.rs").write_text("original source\n")
(self.root / ".gitignore").write_text("/target/\n/rustfs/static/\n")
(self.root / ".agents/skills").mkdir(parents=True)
(self.root / ".agents/skills/SKILL.md").write_text("tracked instructions\n")
(self.root / ".claude").mkdir()
(self.root / ".claude/skills").symlink_to("../.agents/skills", target_is_directory=True)
subprocess.run(["git", "init", "-q", str(self.root)], check=True)
for args in (["add", "."], ["-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "fixture"]):
subprocess.run(["git", "-C", str(self.root), *args], check=True)
self.commands = self.root / "target/commands"
self.commands.mkdir(parents=True)
cargo = self.commands / "cargo"
cargo.write_text(f"#!{sys.executable}\n" + '''import json, os, pathlib, sys
if os.environ.get("FAKE_BUILD_FAIL"):
raise SystemExit(23)
args = sys.argv[1:]
target = pathlib.Path(args[args.index("--target-dir") + 1])
binary = target / ("release" if "--release" in args else "debug") / "rustfs"
binary.parent.mkdir(parents=True, exist_ok=True)
binary.write_text("#!/bin/sh\\nexit 0\\n")
binary.chmod(0o755)
features = ["default", "ftps", "webdav"]
if "--features" in args:
features.extend(args[args.index("--features") + 1].split(","))
if "full" in features:
features.extend(["sftp", "swift", "metrics-gpu", "pyroscope"])
print(json.dumps({"reason": "compiler-artifact", "target": {"name": "rustfs", "kind": ["bin"]}, "executable": str(binary), "features": sorted(set(features))}))
if os.environ.get("FAKE_BUILD_MUTATE"):
pathlib.Path("source.rs").write_text("changed during build")
''')
cargo.chmod(0o755)
rustc = self.commands / "rustc"
rustc.write_text("#!/bin/sh\nprintf 'rustc fixture\\nhost: fixture\\n'\n")
rustc.chmod(0o755)
self.env = dict(os.environ, PATH=f"{self.commands}{os.pathsep}{os.environ['PATH']}")
for name in ("CARGO_TARGET_DIR", "CARGO_BIN_EXE_rustfs", "RUSTFS_BUILD_FEATURES", "RUSTFS_E2E_BINARY_RECEIPT"):
self.env.pop(name, None)
self.binary = self.root / "target/debug/rustfs"
self.sidecar = self.binary.with_name("rustfs.e2e.json")
def invoke(self, *args, env=None):
return subprocess.run([sys.executable, str(self.root / "scripts/e2e_binary.py"), *args], cwd=self.root, env=env or self.env, text=True, capture_output=True)
def build(self, features=""):
result = self.invoke("build", "--features", features)
self.assertEqual(result.returncode, 0, result.stderr)
def run_code(self, code="pass", features="", env=None):
return self.invoke("run", "--features", features, "--", sys.executable, "-c", code, env=env)
def test_build_run_and_receipt_cleanup(self):
self.build("full,e2e-test-hooks")
result = self.run_code("import os,pathlib; print(os.environ['RUSTFS_E2E_BINARY_RECEIPT']); assert pathlib.Path(os.environ['CARGO_BIN_EXE_rustfs']).is_file(); assert 'sftp' in os.environ['RUSTFS_BUILD_FEATURES']", "e2e-test-hooks,full")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertFalse(Path(result.stdout.strip()).exists(), "run receipts must not survive their command")
self.assertIn("sftp", json.loads(self.sidecar.read_text())["features"])
def test_source_changes_are_not_hidden_by_timestamps_or_head(self):
self.build()
path = self.root / "source.rs"
old = path.stat()
path.write_text("different bytes\n")
os.utime(path, ns=(old.st_atime_ns, old.st_mtime_ns))
self.assertNotEqual(self.run_code().returncode, 0)
def test_deleted_untracked_and_ignored_embedded_inputs(self):
for mutation in ("delete", "untracked", "static"):
with self.subTest(mutation=mutation):
self.build()
path = self.root / "source.rs"
if mutation == "delete":
path.unlink()
elif mutation == "untracked":
(self.root / "new.rs").write_text("new source")
else:
static = self.root / "rustfs/static"
static.mkdir(parents=True)
(static / "index.html").write_text("embedded content")
self.assertNotEqual(self.run_code().returncode, 0)
path.write_text("original source\n")
def test_wrong_binary_features_and_manifest_fail_closed(self):
self.build("sftp")
self.assertNotEqual(self.run_code(features="webdav").returncode, 0)
self.binary.write_text("old server")
self.assertNotEqual(self.run_code(features="sftp").returncode, 0)
self.sidecar.write_text("{}")
self.assertNotEqual(self.run_code(features="sftp").returncode, 0)
self.sidecar.unlink()
self.assertNotEqual(self.run_code(features="sftp").returncode, 0)
def test_build_failure_or_source_race_does_not_leave_a_receipt(self):
for failure in ("FAKE_BUILD_FAIL", "FAKE_BUILD_MUTATE"):
self.build()
result = self.invoke("build", env=dict(self.env, **{failure: "1"}))
self.assertNotEqual(result.returncode, 0)
self.assertFalse(self.sidecar.exists())
def test_child_failure_and_changes_during_run_fail(self):
self.build()
failed = self.run_code("raise SystemExit(37)")
self.assertEqual(failed.returncode, 37, failed.stderr)
for code in ("import pathlib; pathlib.Path('source.rs').write_text('changed while testing')", "import pathlib; pathlib.Path('target/debug/rustfs').write_text('different server')"):
self.build()
self.assertNotEqual(self.run_code(code).returncode, 0)
def test_override_cannot_select_an_unverified_server(self):
self.build()
result = self.run_code(env=dict(self.env, CARGO_BIN_EXE_rustfs="/some/old/server"))
self.assertNotEqual(result.returncode, 0)
def test_artifact_moves_between_clean_checkouts(self):
self.build()
with tempfile.TemporaryDirectory() as destination:
clone = Path(destination) / "clone"
subprocess.run(["git", "clone", "-q", str(self.root), str(clone)], check=True)
(clone / "target/debug").mkdir(parents=True)
shutil.copy2(self.binary, clone / "target/debug/rustfs")
shutil.copy2(self.sidecar, clone / "target/debug/rustfs.e2e.json")
result = subprocess.run([sys.executable, str(clone / "scripts/e2e_binary.py"), "run", "--", sys.executable, "-c", "pass"], cwd=clone, env=self.env, text=True, capture_output=True)
self.assertEqual(result.returncode, 0, result.stderr)
def test_target_directory_and_profile_are_explicit(self):
env = dict(self.env, CARGO_TARGET_DIR="target/custom")
built = self.invoke("build", "--profile", "release", env=env)
self.assertEqual(built.returncode, 0, built.stderr)
run = self.invoke("run", "--profile", "release", "--", sys.executable, "-c", "pass", env=env)
self.assertEqual(run.returncode, 0, run.stderr)
self.assertNotEqual(self.invoke("run", "--", sys.executable, "-c", "pass", env=env).returncode, 0)
def test_target_directory_cannot_hide_source_inputs(self):
for target in (str(self.root), str(self.root / "crates"), str(self.root.parent)):
with self.subTest(target=target):
result = self.invoke("build", env=dict(self.env, CARGO_TARGET_DIR=target))
self.assertNotEqual(result.returncode, 0)
self.assertIn("CARGO_TARGET_DIR", result.stderr)
tracked = self.root / "target/tracked.rs"
tracked.write_text("tracked build input")
subprocess.run(["git", "add", "-f", "target/tracked.rs"], cwd=self.root, check=True)
self.build()
tracked.write_text("changed tracked build input")
self.assertNotEqual(self.run_code().returncode, 0)
def test_unsupported_embedded_directory_links_fail_closed(self):
self.build()
destination = self.root / "target/embedded-assets"
destination.mkdir()
(destination / "index.html").write_text("untracked embedded input")
static = self.root / "rustfs/static"
static.mkdir(parents=True)
(static / "linked-assets").symlink_to(destination, target_is_directory=True)
self.assertNotEqual(self.run_code().returncode, 0)
def test_directory_aliases_cannot_hide_unrecorded_inputs(self):
self.build()
target = self.root / ".agents/skills/SKILL.md"
target.write_text("changed instructions\n")
self.assertNotEqual(self.run_code().returncode, 0)
self.build()
(target.parent / ".gitignore").write_text("hidden.rs\n")
(target.parent / "hidden.rs").write_text("ignored build input\n")
result = self.invoke("build")
self.assertNotEqual(result.returncode, 0)
self.assertIn("unrecorded input", result.stderr)
alias = self.root / ".claude/skills"
alias.unlink()
with tempfile.TemporaryDirectory() as external:
alias.symlink_to(external, target_is_directory=True)
result = self.invoke("build")
self.assertNotEqual(result.returncode, 0)
self.assertIn("escapes the source inventory", result.stderr)
def test_directory_alias_indirection_is_part_of_the_identity(self):
for name in ("first", "second"):
directory = self.root / name
directory.mkdir()
(directory / "input.rs").write_text(name)
selection = self.root / "target/selection"
selection.symlink_to(self.root / "first", target_is_directory=True)
(self.root / "source-alias").symlink_to("target/selection", target_is_directory=True)
self.build()
selection.unlink()
selection.symlink_to(self.root / "second", target_is_directory=True)
self.assertNotEqual(self.run_code().returncode, 0)
def test_existing_embedded_files_and_symlink_targets_are_hashed(self):
static = self.root / "rustfs/static"
static.mkdir(parents=True)
index = static / "index.html"
index.write_text("embedded version one")
external = self.root / "target/embedded-file"
external.write_text("linked version one")
(static / "linked.html").symlink_to(external)
self.build()
index.write_text("embedded version two")
self.assertNotEqual(self.run_code().returncode, 0)
self.build()
external.write_text("linked version two")
self.assertNotEqual(self.run_code().returncode, 0)
def test_each_run_hashes_binary_twice_and_never_calls_cargo(self):
script = self.root / "scripts/e2e_binary.py"
script.write_text(script.read_text().replace("def file_hash(path):\n", "def file_hash(path):\n if path.name == 'rustfs':\n with (ROOT / 'target/hash-count').open('a') as count:\n count.write('hash\\n')\n"))
self.build()
count = self.root / "target/hash-count"
count.write_text("")
result = self.run_code(env=dict(self.env, FAKE_BUILD_FAIL="1"))
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(count.read_text().splitlines(), ["hash", "hash"])
def test_concurrent_build_or_run_is_rejected(self):
self.build()
command = [sys.executable, str(self.root / "scripts/e2e_binary.py"), "run", "--", sys.executable, "-c", "print('ready', flush=True); input()"]
with subprocess.Popen(command, cwd=self.root, env=self.env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as process:
self.assertEqual(process.stdout.readline().strip(), "ready")
try:
for args in (("build", "--features", "sftp"), ("run", "--", sys.executable, "-c", "pass")):
rejected = self.invoke(*args)
self.assertNotEqual(rejected.returncode, 0)
self.assertIn("Another E2E build/run", rejected.stderr)
finally:
output, error = process.communicate("\n", timeout=10)
self.assertEqual(process.returncode, 0, error + output)
self.assertFalse(self.binary.with_name("rustfs.e2e.lock").exists())
def test_interruption_cleans_receipt_and_releases_ownership(self):
self.build()
for signum in (signal.SIGINT, signal.SIGTERM):
command = [sys.executable, str(self.root / "scripts/e2e_binary.py"), "run", "--", sys.executable, "-c", "import os; print(os.environ['RUSTFS_E2E_BINARY_RECEIPT'], flush=True); input()"]
with subprocess.Popen(command, cwd=self.root, env=self.env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as process:
receipt = Path(process.stdout.readline().strip())
self.assertTrue(receipt.is_file())
process.send_signal(signum)
process.communicate(timeout=10)
self.assertNotEqual(process.returncode, 0)
self.assertFalse(receipt.exists())
self.assertFalse(self.binary.with_name("rustfs.e2e.lock").exists())
if __name__ == "__main__":
unittest.main()