Compare commits

..

2 Commits

Author SHA1 Message Date
Zhengchao An 48f0f04e0c fix(tests): satisfy new clippy lints 2026-09-05 17:23:20 +08:00
Zhengchao An cef1b21638 fix(object): simplify SSE config not-found match 2026-09-05 16:58:44 +08:00
33 changed files with 1459 additions and 2483 deletions
+2 -3
View File
@@ -3,10 +3,9 @@
.NOTPARALLEL: pre-commit pre-pr dev-check
.PHONY: setup-hooks
setup-hooks: ## Install the configured pre-commit hooks
setup-hooks: ## Set up git hooks
@echo "🔧 Setting up git hooks..."
pre-commit validate-config
pre-commit install
chmod +x .git/hooks/pre-commit
@echo "✅ Git hooks setup complete!"
.PHONY: doc-paths-check
-115
View File
@@ -1,115 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Quick Checks
description: Run the shared compile-free RustFS quality checks.
runs:
using: composite
steps:
- name: Install quality tools
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
with:
tool: |
ripgrep@15.2.0
shellcheck@0.11.0
- name: Install actionlint
shell: bash
run: |
actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
curl --fail --location --silent --show-error \
--output "$actionlint_dir/actionlint.tar.gz" \
https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
rm "$actionlint_dir/actionlint.tar.gz"
echo "$actionlint_dir" >> "$GITHUB_PATH"
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: rustfmt
- name: Check workflow syntax and shell scripts
shell: bash
run: shellcheck --version && actionlint
- name: Check code formatting
shell: bash
run: cargo fmt --all --check
- name: Check unsafe code allowances
shell: bash
run: ./scripts/check_unsafe_code_allowances.sh
- name: Check layered dependencies
shell: bash
run: ./scripts/check_layer_dependencies.sh
- name: Check architecture migration rules
shell: bash
run: ./scripts/check_architecture_migration_rules.sh
- name: Check logging guardrails
shell: bash
run: ./scripts/check_logging_guardrails.sh
- name: Check error other(format!) ratchet
shell: bash
run: ./scripts/check_error_other_format_ratchet.sh
- name: Check tokio io-uring feature guard
shell: bash
run: ./scripts/check_no_tokio_io_uring.sh
- name: Check extension schema boundaries
shell: bash
run: ./scripts/check_extension_schema_boundaries.sh
- name: Check body-cache whitelist guard
shell: bash
run: ./scripts/check_body_cache_whitelist.sh
- name: Check s3s footprint ratchet
shell: bash
run: ./scripts/check_s3s_footprint.sh
- name: Check cryptographic capability wording
shell: bash
run: ./scripts/check_fips_wording.sh
- name: Check no embedded secret material
shell: bash
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
shell: bash
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
shell: bash
run: ./scripts/check_no_planning_docs.sh
- name: Check CI paths stay in sync
shell: bash
run: ./scripts/check_ci_paths_sync.sh
- name: Check io_uring lane --lib precondition
shell: bash
run: ./scripts/check_uring_lane_lib_only.sh
+5 -5
View File
@@ -10,16 +10,16 @@ Use N/A when there is no related issue.
## Summary of Changes
<!--
Describe the concrete problem and resulting behavior. For a behavior change, name the input or state that triggers it and the expected outcome. Explain any new dependency or abstraction that the change needs.
Briefly explain what changed and why reviewers should accept it.
Focus on behavior, compatibility, and review-relevant context.
-->
## Verification
<!--
Give 13 concrete pieces of evidence for the changed behavior: the test or command, its observed result, and the regression it catches. For a bug fix, record a failing-before/passing-after check or explain why it was unavailable.
List the commands or checks you ran, for example:
- `make pre-commit`
Identify the tested commit and any local changes. When testing a prebuilt binary or external service, include its source/version and artifact identity; a successful run against a different build is not evidence for this change.
List relevant checks not run and the remaining risk. Use the validation tier in AGENTS.md; do not run broader checks solely to fill this section. For documentation-only changes, list the applicable documentation checks. Use N/A only when verification is not applicable.
Use N/A only when verification is not applicable.
-->
## Impact
+89 -6
View File
@@ -12,10 +12,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Reports the existing required checks for paths excluded by ci.yml.
# Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared
# action to keep validation coverage aligned. Keep this paths list in sync with
# ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh.
# Companion to ci.yml for required status checks.
#
# ci.yml skips docs-only pull requests via paths-ignore, but the branch ruleset
# requires a check named "Test and Lint" — without this workflow a docs-only PR
# would wait on it forever. This workflow triggers on exactly the paths ci.yml
# ignores and reports success under the same job name. Mixed PRs trigger both
# workflows and the real check still gates: a required check with any failing
# run blocks the merge.
# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
#
# "Quick Checks" is mirrored here ahead of the ruleset change that will make it
# required too (rustfs/backlog#1599). Until that change lands this job is
# inert; mirroring it first is what lets the ruleset change happen without
# stranding docs-only PRs on a check nobody reports.
#
# Keep the paths list below in sync with the pull_request paths-ignore list
# in ci.yml, and keep the quick-checks steps below byte-identical to the
# quick-checks job in ci.yml.
name: Continuous Integration (docs only)
@@ -45,6 +59,19 @@ permissions:
contents: read
jobs:
# Deliberately NOT a bare `echo`. Once "Quick Checks" becomes a required
# check, ci.yml gates every expensive job behind it, so a mixed PR reports
# two check runs with this name: the real one (45-51s) and this companion.
# GitHub has no written contract for how it picks between same-named
# required check runs ("latest wins" vs "any failure blocks"), so instead of
# relying on ordering we make both runs execute the same commands against
# the same merge ref — their conclusions are then necessarily identical and
# the choice does not matter. Keep these steps byte-identical to the
# quick-checks job in ci.yml (a guard script that asserts this, and the paths
# sync below, is tracked in rustfs/backlog#1603).
#
# For a genuinely docs-only PR this adds no strictness (no code changed, so
# fmt and the guards always pass) and costs ~50s of ubuntu-latest.
quick-checks:
name: Quick Checks
runs-on: ubuntu-latest
@@ -55,8 +82,64 @@ jobs:
with:
persist-credentials: false
- name: Run shared quick checks
uses: ./.github/actions/quick-checks
- name: Install ripgrep
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
with:
tool: ripgrep@15.2.0
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: rustfmt
- name: Check code formatting
run: cargo fmt --all --check
- name: Check unsafe code allowances
run: ./scripts/check_unsafe_code_allowances.sh
- name: Check layered dependencies
run: ./scripts/check_layer_dependencies.sh
- name: Check architecture migration rules
run: ./scripts/check_architecture_migration_rules.sh
- name: Check logging guardrails
run: ./scripts/check_logging_guardrails.sh
- name: Check tokio io-uring feature guard
run: ./scripts/check_no_tokio_io_uring.sh
- name: Check extension schema boundaries
run: ./scripts/check_extension_schema_boundaries.sh
- name: Check body-cache whitelist guard
run: ./scripts/check_body_cache_whitelist.sh
- name: Check s3s footprint ratchet
run: ./scripts/check_s3s_footprint.sh
- name: Check cryptographic capability wording
run: ./scripts/check_fips_wording.sh
- name: Check no embedded secret material
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
- name: Check CI paths stay in sync
run: ./scripts/check_ci_paths_sync.sh
- name: Check io_uring lane --lib precondition
run: ./scripts/check_uring_lane_lib_only.sh
test-and-lint:
name: Test and Lint
+67 -3
View File
@@ -100,7 +100,12 @@ jobs:
- name: Typos check with custom config file
uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
# Fail early with compile-free checks shared with docs-only CI.
# Fast, compile-free checks that fail early so contributors get feedback in
# ~1 minute instead of waiting for the full test job.
#
# These steps are mirrored byte-for-byte in ci-docs-only.yml so that a mixed
# PR, which reports two check runs named "Quick Checks", cannot get one red
# and one green. Edit both jobs together.
quick-checks:
name: Quick Checks
if: github.event_name != 'pull_request' || github.event.action != 'closed'
@@ -112,8 +117,67 @@ jobs:
with:
persist-credentials: false
- name: Run shared quick checks
uses: ./.github/actions/quick-checks
- name: Install ripgrep
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
with:
tool: ripgrep@15.2.0
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
with:
components: rustfmt
- name: Check code formatting
run: cargo fmt --all --check
- name: Check unsafe code allowances
run: ./scripts/check_unsafe_code_allowances.sh
- name: Check layered dependencies
run: ./scripts/check_layer_dependencies.sh
- name: Check architecture migration rules
run: ./scripts/check_architecture_migration_rules.sh
- name: Check logging guardrails
run: ./scripts/check_logging_guardrails.sh
- name: Check error other(format!) ratchet
run: ./scripts/check_error_other_format_ratchet.sh
- name: Check tokio io-uring feature guard
run: ./scripts/check_no_tokio_io_uring.sh
- name: Check extension schema boundaries
run: ./scripts/check_extension_schema_boundaries.sh
- name: Check body-cache whitelist guard
run: ./scripts/check_body_cache_whitelist.sh
- name: Check s3s footprint ratchet
run: ./scripts/check_s3s_footprint.sh
- name: Check cryptographic capability wording
run: ./scripts/check_fips_wording.sh
- name: Check no embedded secret material
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
- name: Check CI paths stay in sync
run: ./scripts/check_ci_paths_sync.sh
- name: Check io_uring lane --lib precondition
run: ./scripts/check_uring_lane_lib_only.sh
test-and-lint:
name: Test and Lint
+3 -3
View File
@@ -3,9 +3,9 @@
repos:
- repo: local
hooks:
- id: rustfs-fmt-check
name: Rust formatting
entry: cargo fmt --all --check
- id: rustfs-dev-check
name: rustfs dev-check
entry: make dev-check
language: system
types: [rust]
pass_filenames: false
+37 -11
View File
@@ -109,17 +109,24 @@ affected boundaries and risks. CI still runs its configured repository gates.
### 🔒 Git Pre-commit Hooks (optional)
The optional hook uses the checked-in `.pre-commit-config.yaml`. Install [pre-commit](https://pre-commit.com/#installation), then run this from the checkout or a linked worktree:
Git hooks are **not** versioned in this repository, so a fresh clone has no
active pre-commit hook. If you add your own `.git/hooks/pre-commit` (a good
choice is a one-liner that runs `make pre-commit`), you can mark it executable
with:
```bash
make setup-hooks
```
The hook runs `cargo fmt --all --check` when staged files include Rust source. It does not compile the workspace or run tests. Fix formatting with `cargo fmt --all`, inspect and stage the result, then commit again.
Or manually:
`pre-commit install` resolves Git's hook directory for linked worktrees and preserves an existing hook in migration mode. If you use `core.hooksPath`, keep that hook manager and integrate `pre-commit run` there; the installer refuses to silently replace that configuration.
```bash
chmod +x .git/hooks/pre-commit
```
A local hook provides early formatting feedback. With or without it, follow the verification tiers in `AGENTS.md`, run relevant behavioral tests, and satisfy the CI merge gates. `make pre-commit` and `make dev-check` remain explicit broader commands.
With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
applicable scoped checks, and reserve `make pre-pr` for broad cross-module
changes whose impact cannot be bounded by those checks.
### 📝 Formatting Configuration
@@ -131,11 +138,31 @@ fn_call_width = 90
single_line_let_else_max_width = 100
```
### 🚫 Commit Prevention
If you set up a pre-commit hook and your code doesn't meet the formatting requirements, the hook will:
1. **Block the commit** and show clear error messages
2. **Provide exact commands** to fix the issues
3. **Guide you through** the resolution process
Example output when formatting fails:
```
❌ Code formatting check failed!
💡 Please run 'cargo fmt --all' to format your code before committing.
🔧 Quick fix:
cargo fmt --all
git add .
git commit
```
### 🔄 Development Workflow
1. **Make your changes**
2. **Format your code**: `make fmt` or `cargo fmt --all`
3. **Select relevant checks** using the validation tier in `AGENTS.md`; use `make pre-commit` when its broader fast gate adds useful coverage
3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
4. **Commit your changes**: `git commit -m "your message"`
5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
6. **Run applicable scoped checks before opening/updating a PR**; consider
@@ -179,12 +206,11 @@ Configure your IDE to:
#### Pre-commit hook not running?
```bash
pre-commit validate-config
pre-commit run --all-files
# Inspect any configured hook manager; do not overwrite it.
git config --get core.hooksPath
# Install if no separate hook manager is configured.
make setup-hooks
# Check if hook is executable
ls -la .git/hooks/pre-commit
# Make it executable if needed
chmod +x .git/hooks/pre-commit
```
#### Formatting issues?
Generated
+13 -31
View File
@@ -315,7 +315,7 @@ dependencies = [
"strum",
"thiserror 2.0.20",
"uuid",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -508,7 +508,7 @@ dependencies = [
"arrow-select",
"flatbuffers",
"lz4_flex",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -2249,8 +2249,8 @@ dependencies = [
"liblzma",
"lz4",
"memchr",
"zstd 0.13.3",
"zstd-safe 7.3.0",
"zstd",
"zstd-safe",
]
[[package]]
@@ -4067,7 +4067,7 @@ dependencies = [
"uuid",
"walkdir",
"zip",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -5971,7 +5971,7 @@ dependencies = [
"lz4",
"snap",
"uuid",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -7658,7 +7658,7 @@ dependencies = [
"snap",
"tokio",
"twox-hash",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -9618,7 +9618,7 @@ dependencies = [
"x509-parser",
"zeroize",
"zip",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -10228,7 +10228,7 @@ dependencies = [
"thiserror 2.0.20",
"walkdir",
"zip",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -10395,7 +10395,7 @@ dependencies = [
"tracing-opentelemetry",
"tracing-subscriber",
"url",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -10985,7 +10985,7 @@ dependencies = [
"transform-stream",
"url",
"windows",
"zstd 0.14.0",
"zstd",
]
[[package]]
@@ -14095,7 +14095,7 @@ dependencies = [
"typed-path",
"zeroize",
"zopfli",
"zstd 0.13.3",
"zstd",
]
[[package]]
@@ -14128,16 +14128,7 @@ version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe 7.3.0",
]
[[package]]
name = "zstd"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e"
dependencies = [
"zstd-safe 8.0.0",
"zstd-safe",
]
[[package]]
@@ -14149,15 +14140,6 @@ dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-safe"
version = "8.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.1.0+zstd.1.5.7"
+5 -5
View File
@@ -199,10 +199,10 @@ serde_urlencoded = "0.7.1"
# matching stable releases are not available yet, while previous stable lines
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
# releases.
aes-gcm = { version = "0.11.1" }
argon2 = { version = "0.6.0" }
blake2 = "0.11.0"
chacha20poly1305 = { version = "0.11.0" }
aes-gcm = { version = "=0.11.1" }
argon2 = { version = "=0.6.0" }
blake2 = "=0.11.0"
chacha20poly1305 = { version = "=0.11.0" }
crc-fast = "1.10.0"
hmac = { version = "0.13.0" }
jsonwebtoken = { version = "11.0.0" }
@@ -343,7 +343,7 @@ windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.18" }
zip = "8.6.0"
zstd = "0.14.0"
zstd = "0.13.3"
# Observability and Metrics
metrics = "0.24.6"
-15
View File
@@ -130,21 +130,6 @@ Scanner cycle budget controls:
- timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
- this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
## Remote tier timeout environment variables
- `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS`
- remote tier TCP connect timeout.
- default is `10`.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
- `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS`
- remote tier request timeout through response headers.
- default is `86400` so large transition uploads keep a production-safe budget.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default. Very large values are accepted and act as a correspondingly long budget.
- `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS`
- maximum idle time between remote tier response-body chunks.
- default is `60`; the timer resets only when non-empty body data keeps progressing.
- must be positive; zero fails tier client initialization, while an invalid integer is logged and falls back to the default.
## Drive timeout environment variables
- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
-32
View File
@@ -137,28 +137,6 @@ pub const DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED: bool = false;
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_WRITE);
const _: () = assert!(!DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED);
/// Environment variable for remote tier TCP connect timeout in seconds.
pub const ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS";
/// Default remote tier TCP connect timeout in seconds.
pub const DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS: u64 = 10;
/// Environment variable for the remote tier request timeout in seconds.
///
/// This bounds upload/download request progress through response headers. The
/// default is intentionally large so multi-TiB transition uploads keep their
/// previous production budget while black-hole remotes no longer wait forever.
pub const ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS";
/// Default remote tier request timeout in seconds.
pub const DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS: u64 = 24 * 60 * 60;
/// Environment variable for remote tier response-body idle timeout in seconds.
///
/// The timer is re-armed on every non-empty response-body chunk, so slow but
/// progressing remotes can continue while silent response bodies are cancelled.
pub const ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: &str = "RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS";
/// Default remote tier response-body idle timeout in seconds.
pub const DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS: u64 = 60;
/// Request the object-transaction fencing contract used by storage-owned
/// cleanup receipts and lock-window optimizations.
///
@@ -834,16 +812,6 @@ mod remote_version_state_tests {
);
}
#[test]
fn remote_tier_timeout_env_names_are_stable() {
assert_eq!(super::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS");
assert_eq!(super::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS, "RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS");
assert_eq!(
super::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
"RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS"
);
}
#[test]
fn data_movement_part_checksum_gate_uses_stable_environment_names() {
assert_eq!(super::ENV_DATA_MOVEMENT_PART_CHECKSUMS_WRITE, "RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE");
+3 -4
View File
@@ -167,10 +167,9 @@ pub mod bucket {
idle_guarded_body,
};
pub use crate::bucket::on_demand_migration::{
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome,
MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
decode_continuation_token, source_list_plan,
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
};
pub mod backfill {
pub use crate::bucket::on_demand_migration::backfill::{
@@ -25,13 +25,8 @@ use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use std::time::{Duration, Instant};
/// The continuation-token version used by ordinary progressing pages.
/// The only continuation-token envelope version this build reads and writes.
pub const LIST_THROUGH_TOKEN_VERSION: u32 = 1;
const LIST_THROUGH_PROGRESS_TOKEN_VERSION: u32 = 2;
/// The sixteenth consecutive merged page without a key or new EOF fails.
/// This also bounds legitimate sparse listings; it is not a cycle detector.
pub const MAX_LIST_NO_PROGRESS_PAGES: u8 = 16;
/// Envelope marker. A bucket that is *not* merging hands out the local
/// listing's own marker, so the decoder needs a positive signal before it
@@ -116,10 +111,6 @@ pub struct ListThroughToken {
/// common prefix compares as itself, never as its members.
#[serde(default)]
pub last_key: Option<String>,
/// Consecutive empty truncated merged pages, present only in v2 tokens.
/// Ordinary v1 tokens retain their original serialized shape.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub no_progress: Option<u8>,
}
impl ListThroughToken {
@@ -132,7 +123,6 @@ impl ListThroughToken {
source: source.token,
source_done: source.done,
last_key,
no_progress: None,
}
}
@@ -180,21 +170,7 @@ pub fn decode_continuation_token(decoded: &str) -> Result<ListThroughCursor, Lis
return Ok(ListThroughCursor::Local(decoded.to_string()));
}
match value.get("v").and_then(serde_json::Value::as_u64) {
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {
// v1 readers reject this field even when it is null or zero.
if value.get("no_progress").is_some() {
return Err(ListThroughTokenError::Malformed);
}
}
Some(version) if version == u64::from(LIST_THROUGH_PROGRESS_TOKEN_VERSION) => {
if !value
.get("no_progress")
.and_then(serde_json::Value::as_u64)
.is_some_and(|count| (1..u64::from(MAX_LIST_NO_PROGRESS_PAGES)).contains(&count))
{
return Err(ListThroughTokenError::Malformed);
}
}
Some(version) if version == u64::from(LIST_THROUGH_TOKEN_VERSION) => {}
Some(version) => return Err(ListThroughTokenError::UnsupportedVersion(version.min(u64::from(u32::MAX)) as u32)),
None => return Err(ListThroughTokenError::Malformed),
}
@@ -312,8 +288,6 @@ pub enum ListPageError {
Empty,
#[error("truncated listing repeats a continuation token")]
Repeated,
#[error("listing exhausted its consecutive no-progress page budget")]
NoProgress(MergeSide),
}
pub(crate) fn validate_list_page(is_truncated: bool, token: Option<&str>, next_token: Option<&str>) -> Result<(), ListPageError> {
@@ -378,7 +352,6 @@ pub struct MergeOutcome {
#[derive(Debug)]
pub struct ListThroughMerger {
max_keys: usize,
no_progress: Option<u8>,
last_key: Option<String>,
local: SideState,
source: SideState,
@@ -398,7 +371,6 @@ impl ListThroughMerger {
};
Self {
max_keys,
no_progress: token.and_then(|token| token.no_progress),
last_key,
local,
source,
@@ -464,18 +436,13 @@ impl ListThroughMerger {
Ok(())
}
/// `issue_progress_tokens` allows a v1 chain to start carrying a budget.
/// An existing v2 budget is always enforced, including on reader-only nodes.
/// Borrowing lets a source failure re-merge the fetched local buffers.
pub fn finish(&self, issue_progress_tokens: bool) -> Result<MergeOutcome, ListPageError> {
pub fn finish(self) -> MergeOutcome {
let Self {
max_keys,
no_progress,
last_key,
local,
source,
} = self;
let max_keys = *max_keys;
// A side with more pages behind it can only be trusted up to the last
// key it handed over: past that horizon the other side's entries could
@@ -541,44 +508,12 @@ impl ListThroughMerger {
let source_left = !source.disabled && (!source_cursor.done || consumed_source < source.entries.len());
let is_truncated = local_left || source_left;
let reached_eof = (!local.start.done && local_cursor.done) || (!source.start.done && source_cursor.done);
let next_no_progress = if !is_truncated || !picks.is_empty() || reached_eof {
None
} else if max_keys == 0 {
// A zero-sized request cannot consume entries. Preserve an existing
// budget without spending it or starting a new one.
*no_progress
} else if issue_progress_tokens || no_progress.is_some() {
let count = no_progress.unwrap_or(0).saturating_add(1);
if count >= MAX_LIST_NO_PROGRESS_PAGES {
// An empty truncated side closes the merge horizon. Local
// failure takes precedence; disabling the source cannot fix it.
let side = if local.more && local.entries.is_empty() {
MergeSide::Local
} else if !source.disabled && source.more && source.entries.is_empty() {
MergeSide::Source
} else {
MergeSide::Local
};
return Err(ListPageError::NoProgress(side));
}
Some(count)
} else {
None
};
let last_key = consumed_key.or_else(|| last_key.clone());
Ok(MergeOutcome {
let last_key = consumed_key.or(last_key);
MergeOutcome {
picks,
is_truncated,
next_token: is_truncated.then(|| {
let mut token = ListThroughToken::new(local_cursor, source_cursor, last_key);
if let Some(count) = next_no_progress {
token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION;
token.no_progress = Some(count);
}
token
}),
})
next_token: is_truncated.then(|| ListThroughToken::new(local_cursor, source_cursor, last_key)),
}
}
}
@@ -706,7 +641,7 @@ mod tests {
.push_page(fetch.side, kept, truncated, next)
.expect("reference provider pages must advance");
}
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(outcome.is_truncated, outcome.next_token.is_some());
if outcome.is_truncated {
assert_ne!(outcome.next_token, token, "every truncated merged page must make progress");
@@ -789,7 +724,7 @@ mod tests {
.push_page(MergeSide::Local, vec![ListEntryKey::object("a")], false, None)
.expect("local EOF is valid");
assert_eq!(merger.next_fetch(), None);
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(outcome.picks.len(), 1);
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
@@ -805,7 +740,6 @@ mod tests {
source: Some("source-1".to_string()),
source_done: false,
last_key: Some("a".to_string()),
no_progress: None,
};
let mut merger = ListThroughMerger::new(1, Some(&resume));
merger.disable_source();
@@ -817,7 +751,7 @@ mod tests {
Some("local-2".to_string()),
)
.expect("local cursor advances");
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("truncated page carries a token");
assert_eq!(token.source.as_deref(), Some("source-1"), "the source cursor must not move");
@@ -896,7 +830,7 @@ mod tests {
.expect("opaque cursor advances regardless of sort order");
}
assert!(merger.next_fetch().is_none(), "two source fetches exhaust the request budget");
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert!(outcome.picks.is_empty());
assert!(outcome.is_truncated);
let token = outcome.next_token.expect("empty progressing page has a cursor");
@@ -906,7 +840,7 @@ mod tests {
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], false, None)
.expect("source EOF");
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(
outcome.picks,
vec![MergePick {
@@ -953,7 +887,7 @@ mod tests {
Err(ListPageError::Repeated)
);
merger.disable_source();
let outcome = merger.finish(false).expect("valid merge outcome");
let outcome = merger.finish();
assert_eq!(
outcome.picks,
vec![MergePick {
@@ -1045,8 +979,8 @@ mod tests {
let encoded = token.encode();
assert_eq!(decode_continuation_token(&encoded), Ok(ListThroughCursor::Merged(Box::new(token))));
let bumped = encoded.replace("\"v\":1", "\"v\":3");
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(3)));
let bumped = encoded.replace("\"v\":1", "\"v\":2");
assert_eq!(decode_continuation_token(&bumped), Err(ListThroughTokenError::UnsupportedVersion(2)));
let extra = encoded.replace("{", "{\"x\":1,");
assert_eq!(decode_continuation_token(&extra), Err(ListThroughTokenError::Malformed));
@@ -1058,257 +992,6 @@ mod tests {
assert_eq!(decode_continuation_token(no_version), Err(ListThroughTokenError::Malformed));
}
fn progress_token(count: Option<u8>, local_done: bool, source_done: bool) -> ListThroughToken {
let mut token = ListThroughToken::new(
SideCursor {
token: None,
done: local_done,
},
SideCursor {
token: Some("A".into()),
done: source_done,
},
Some("last-key".into()),
);
if let Some(count) = count {
token.v = LIST_THROUGH_PROGRESS_TOKEN_VERSION;
token.no_progress = Some(count);
}
token
}
fn push_empty_pages(merger: &mut ListThroughMerger, side: MergeSide) {
for _ in 0..MAX_LIST_FETCHES_PER_SIDE {
let fetch = merger.next_fetch().expect("empty truncated side must be fetched");
assert_eq!(fetch.side, side);
let next = format!("{}:next", fetch.token.unwrap_or_default());
merger
.push_page(side, vec![], true, Some(next))
.expect("opaque cursor advances");
}
}
#[test]
fn progress_tokens_preserve_v1_bytes_and_validate_v2_counts() {
let token = progress_token(None, true, false);
assert_eq!(
token.encode(),
r#"{"t":"odm-list","v":1,"local":null,"local_done":true,"source":"A","source_done":false,"last_key":"last-key"}"#
);
for count in 1..MAX_LIST_NO_PROGRESS_PAGES {
let token = progress_token(Some(count), true, false);
assert_eq!(decode_continuation_token(&token.encode()), Ok(ListThroughCursor::Merged(Box::new(token))));
}
for version in [1, 2] {
for value in ["null", "0", "16", "-1", "1.5", "256", "18446744073709551616", "\"1\""] {
let encoded = format!(r#"{{"t":"odm-list","v":{version},"no_progress":{value}}}"#);
assert_eq!(decode_continuation_token(&encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
}
}
for encoded in [
r#"{"t":"odm-list","v":1,"no_progress":1}"#,
r#"{"t":"odm-list","v":2}"#,
r#"{"t":"odm-list","v":2,"no_progress":1,"extra":true}"#,
] {
assert_eq!(decode_continuation_token(encoded), Err(ListThroughTokenError::Malformed), "{encoded}");
}
}
#[test]
fn reader_only_nodes_do_not_start_a_budget_but_mixed_readers_preserve_one() {
let mut token = progress_token(None, true, false);
for _ in 0..MAX_LIST_NO_PROGRESS_PAGES {
let mut merger = ListThroughMerger::new(2, Some(&token));
push_empty_pages(&mut merger, MergeSide::Source);
token = merger
.finish(false)
.expect("reader-only v1 behavior")
.next_token
.expect("truncated cursor");
assert_eq!(token.v, 1);
assert_eq!(token.no_progress, None);
}
for count in 1..=MAX_LIST_NO_PROGRESS_PAGES {
let mut merger = ListThroughMerger::new(2, Some(&token));
push_empty_pages(&mut merger, MergeSide::Source);
assert!(merger.next_fetch().is_none(), "the per-request two-fetch limit stays intact");
let outcome = merger.finish(count % 2 == 1);
if count == MAX_LIST_NO_PROGRESS_PAGES {
assert_eq!(outcome, Err(ListPageError::NoProgress(MergeSide::Source)));
break;
}
token = outcome.expect("budget not exhausted").next_token.expect("truncated cursor");
assert_eq!(token.no_progress, Some(count));
let ListThroughCursor::Merged(decoded) = decode_continuation_token(&token.encode()).expect("round-trip v2") else {
panic!("merged cursor expected");
};
token = *decoded;
}
}
#[test]
fn objects_and_common_prefixes_reset_a_budget_at_the_boundary() {
for entry in [ListEntryKey::object("result"), ListEntryKey::prefix("result/")] {
for issue_tokens in [false, true] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Source, vec![], true, Some("B".into()))
.expect("empty advancing page");
merger
.push_page(MergeSide::Source, vec![entry.clone()], true, Some("C".into()))
.expect("real progress");
let outcome = merger
.finish(issue_tokens)
.expect("real progress does not exhaust the budget");
assert_eq!(
outcome.picks,
vec![MergePick {
side: MergeSide::Source,
index: 0
}]
);
let next = outcome.next_token.expect("source remains truncated");
assert_eq!(next.last_key.as_deref(), Some(entry.name.as_str()));
assert_eq!(next.v, 1);
assert_eq!(next.no_progress, None);
assert!(!next.encode().contains("no_progress"));
}
}
}
#[test]
fn only_a_new_eof_transition_resets_the_empty_page_budget() {
for finished_side in [MergeSide::Local, MergeSide::Source] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
if finished_side == MergeSide::Local {
merger
.push_page(MergeSide::Local, vec![], false, None)
.expect("new local EOF");
push_empty_pages(&mut merger, MergeSide::Source);
} else {
push_empty_pages(&mut merger, MergeSide::Local);
merger
.push_page(MergeSide::Source, vec![], false, None)
.expect("new source EOF");
}
let next = merger
.finish(false)
.expect("new EOF is progress")
.next_token
.expect("other side truncated");
assert_eq!(next.no_progress, None);
assert_eq!(next.v, 1);
assert_eq!(next.local_done, finished_side == MergeSide::Local);
assert_eq!(next.source_done, finished_side == MergeSide::Source);
let mut merger = ListThroughMerger::new(2, Some(&next));
let remaining = if finished_side == MergeSide::Local {
MergeSide::Source
} else {
MergeSide::Local
};
push_empty_pages(&mut merger, remaining);
let next = merger
.finish(true)
.expect("a new budget starts")
.next_token
.expect("truncated");
assert_eq!(next.no_progress, Some(1), "an already-done side cannot reset every page");
}
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger.push_page(MergeSide::Source, vec![], false, None).expect("final EOF");
let outcome = merger.finish(false).expect("EOF succeeds at the budget boundary");
assert!(!outcome.is_truncated);
assert!(outcome.next_token.is_none());
}
#[test]
fn filtered_duplicates_cannot_reset_the_no_progress_budget() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
for next in ["B", "C"] {
let entries = [ListEntryKey::object("last-key"), ListEntryKey::object("earlier")]
.into_iter()
.filter(|entry| merger.accepts(&entry.name))
.collect::<Vec<_>>();
assert!(entries.is_empty(), "both provider entries were already consumed");
merger
.push_page(MergeSide::Source, entries, true, Some(next.into()))
.expect("advancing cursor");
}
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source)));
}
#[test]
fn no_progress_is_attributed_to_local_when_source_cannot_unblock_it() {
for source_mode in ["disabled", "done", "empty", "data"] {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, source_mode == "done");
let mut merger = ListThroughMerger::new(2, Some(&resume));
if source_mode == "disabled" {
merger.disable_source();
}
push_empty_pages(&mut merger, MergeSide::Local);
match source_mode {
"empty" => push_empty_pages(&mut merger, MergeSide::Source),
"data" => merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("source")], false, None)
.expect("source data"),
_ => {}
}
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Local)), "{source_mode}");
}
}
#[test]
fn source_budget_failure_remerges_local_objects_and_prefixes_without_refetching() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), false, false);
let mut merger = ListThroughMerger::new(2, Some(&resume));
merger
.push_page(MergeSide::Local, vec![ListEntryKey::object("local")], true, Some("L1".into()))
.expect("local object");
merger
.push_page(MergeSide::Local, vec![ListEntryKey::prefix("prefix/")], true, Some("L2".into()))
.expect("local prefix");
push_empty_pages(&mut merger, MergeSide::Source);
assert_eq!(merger.finish(false), Err(ListPageError::NoProgress(MergeSide::Source)));
merger.disable_source();
assert!(merger.next_fetch().is_none(), "fallback does not perform another fetch");
let outcome = merger.finish(false).expect("local data makes progress");
assert_eq!(
outcome.picks,
vec![
MergePick {
side: MergeSide::Local,
index: 0
},
MergePick {
side: MergeSide::Local,
index: 1
}
]
);
let token = outcome.next_token.expect("remaining local page");
assert_eq!(token.local.as_deref(), Some("L2"));
assert_eq!(token.source.as_deref(), Some("A"));
assert_eq!(token.last_key.as_deref(), Some("prefix/"));
assert_eq!(token.no_progress, None);
assert_eq!(token.v, 1);
}
#[test]
fn a_zero_sized_merge_preserves_an_existing_budget() {
let resume = progress_token(Some(MAX_LIST_NO_PROGRESS_PAGES - 1), true, false);
let mut merger = ListThroughMerger::new(0, Some(&resume));
merger
.push_page(MergeSide::Source, vec![ListEntryKey::object("result")], true, Some("B".into()))
.expect("source page");
let outcome = merger.finish(false).expect("a zero-sized request cannot consume entries");
assert!(outcome.picks.is_empty());
assert_eq!(outcome.next_token.expect("unconsumed source").no_progress, resume.no_progress);
}
#[test]
fn a_plain_local_marker_stays_local() {
assert_eq!(
@@ -40,10 +40,9 @@ pub use config::{
SourceCredentials, SourceErrorPolicy, SourceTimeout, TlsConfig, ValidationContext,
};
pub use list_through::{
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger,
ListThroughToken, ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MAX_LIST_NO_PROGRESS_PAGES, MergeOutcome, MergePick,
MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter,
decode_continuation_token, source_list_plan,
FetchRequest, LIST_THROUGH_TOKEN_VERSION, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken,
ListThroughTokenError, MAX_LIST_FETCHES_PER_SIDE, MergeOutcome, MergePick, MergeSide, SOURCE_LIST_MAX_RATE_WAIT,
SOURCE_LIST_RATE_PER_SEC, SourceListPlan, SourceListRateLimiter, decode_continuation_token, source_list_plan,
};
pub use negative_cache::{NEGATIVE_CACHE_MAX_ENTRIES, NegativeCache};
pub use pull::{
File diff suppressed because it is too large Load Diff
@@ -37,7 +37,7 @@ use crate::services::tier::{
use bytes::Bytes;
use http::StatusCode;
use rustfs_s3_client::credentials::{Credentials, SignatureType, Static, Value};
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, TransitionCore};
use rustfs_s3_client::transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore};
use rustfs_s3_client::{
admin_handler_utils::AdminError,
api_error_response::to_error_response,
@@ -320,27 +320,6 @@ pub(crate) fn endpoint_authority(url: &url::Url) -> Result<String, std::io::Erro
}
}
fn transition_timeout_from_env(env_key: &str, default_secs: u64) -> Duration {
Duration::from_secs(rustfs_utils::get_env_u64(env_key, default_secs))
}
pub(crate) fn transition_client_timeouts_from_env() -> TransitionClientTimeouts {
TransitionClientTimeouts::new(
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS,
),
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS,
),
transition_timeout_from_env(
rustfs_config::ENV_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
),
)
}
/// Build the [`WarmBackendS3`] shared by the S3-compatible warm backend providers.
///
/// Credential, bucket, and endpoint validation run in this order because the
@@ -371,7 +350,6 @@ pub(crate) async fn new_s3_compatible_warm_backend(
signer_type: SignatureType::SignatureV4,
..Default::default()
}));
let timeouts = transition_client_timeouts_from_env();
let opts = Options {
creds,
secure: u.scheme() == "https",
@@ -384,7 +362,7 @@ pub(crate) async fn new_s3_compatible_warm_backend(
// Run the SSRF guard after the host-presence check so a host-less endpoint
// keeps this constructor's stable error text.
(params.validate_endpoint)(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?;
let client = TransitionClient::new_with_timeouts(&endpoint, opts, params.provider_tag, timeouts).await?;
let client = TransitionClient::new(&endpoint, opts, params.provider_tag).await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
@@ -26,7 +26,7 @@ use crate::services::tier::{
tier_config::TierS3,
warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options, endpoint_authority, transition_client_timeouts_from_env,
build_transition_put_options, endpoint_authority,
},
};
use http::HeaderMap;
@@ -139,7 +139,6 @@ impl WarmBackendS3 {
} else {
return Err(std::io::Error::other("insufficient parameters for S3 backend authentication"));
}
let timeouts = transition_client_timeouts_from_env();
let opts = Options {
creds,
secure: u.scheme() == "https",
@@ -148,7 +147,7 @@ impl WarmBackendS3 {
..Default::default()
};
let endpoint = endpoint_authority(&u)?;
let client = TransitionClient::new_with_timeouts(&endpoint, opts, tier_type, timeouts).await?;
let client = TransitionClient::new(&endpoint, opts, tier_type).await?;
let client = Arc::new(client);
let core = TransitionCore(Arc::clone(&client));
+4 -84
View File
@@ -2452,9 +2452,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let write_quorum = fi.write_quorum(self.default_write_quorum());
let read_quorum = fi.read_quorum(self.default_read_quorum());
// Release the registry guard before recovery and cleanup read it again:
// a queued topology writer would otherwise deadlock those nested reads.
let disks = self.get_disks_internal().await;
let disks = self.disks.read().await;
let disks = disks.clone();
// let disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
let part_path = format!("{}/{}/", upload_id_path, fi.data_dir.unwrap_or(Uuid::nil()));
self.recover_part_transactions(&part_path, read_quorum, write_quorum)
@@ -6742,87 +6743,6 @@ mod tests {
.await;
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_multipart_releases_disk_snapshot_before_cleanup() {
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-topology-lock-bucket";
let object = "object";
let body = vec![0x65; 4096];
make_bucket_on_all(&disk_stores, bucket).await;
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &body, &ObjectOptions::default()).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
for dir in &temp_dirs {
assert!(
dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
"the test must create real upload staging on every disk"
);
}
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterObjectPublication);
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
// Hold a separate read gate so the real writer queues even when completion
// correctly releases its snapshot guard. Polling Pending proves admission
// to Tokio's write-preferring queue before the cleanup attempts another read.
let read_gate = set_disks.disks.read().await;
let writer = set_disks.disks.write();
tokio::pin!(writer);
assert!(matches!(
futures::poll!(tokio::task::unconstrained(writer.as_mut())),
std::task::Poll::Pending
));
assert!(
set_disks.disks.try_read().is_err(),
"the pending writer must already block new readers before the cleanup resumes"
);
drop(read_gate);
barrier.release();
let writer_guard = tokio::time::timeout(Duration::from_secs(5), writer)
.await
.expect("a queued topology writer must not deadlock with multipart cleanup's disk snapshot");
// A reconnect can publish the same handles; this test isolates admission
// order without changing the disks that contain the committed object.
drop(writer_guard);
tokio::time::timeout(Duration::from_secs(10), complete)
.await
.expect("multipart cleanup must finish after the topology writer releases")
.expect("completion task should not panic")
.expect("completion should preserve the successful object commit");
let mut reader = tokio::time::timeout(
Duration::from_secs(10),
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
)
.await
.expect("GET should finish after completion")
.expect("the completed object should remain readable");
let mut observed_body = Vec::new();
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut observed_body))
.await
.expect("the completed object body should finish streaming")
.expect("the completed object body should be readable");
assert_eq!(observed_body, body);
assert!(matches!(
set_disks.check_upload_id_exists(bucket, object, &upload_id, false).await,
Err(StorageError::InvalidUploadID(..))
));
for dir in &temp_dirs {
assert!(
!dir.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_id_path).exists(),
"successful completion must remove its upload staging from every disk"
);
}
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_releases_object_lock_before_cleanup_and_keeps_upload_lock() {
+44 -460
View File
@@ -43,10 +43,6 @@ const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer";
const ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT: &str = "Expiration cannot specify both Days and Date";
const ERR_LIFECYCLE_MULTIPLE_TRANSITIONS: &str = "Only one Transition action per lifecycle rule is supported";
const ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS: &str =
"Only one NoncurrentVersionTransition action per lifecycle rule is supported";
const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str =
"'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer";
const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
@@ -365,12 +361,6 @@ impl Lifecycle for BucketLifecycleConfiguration {
{
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS));
}
if expiration.days.is_some() && expiration.date.is_some() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT,
));
}
if let Some(expiration_date) = &expiration.date {
let date = OffsetDateTime::from(expiration_date.clone());
if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
@@ -404,20 +394,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
}
if let Some(transitions) = &r.transitions {
if transitions.len() > 1 {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, ERR_LIFECYCLE_MULTIPLE_TRANSITIONS));
}
for transition in transitions {
TransitionOps::validate(transition)?;
}
}
if let Some(noncurrent_transitions) = &r.noncurrent_version_transitions {
if noncurrent_transitions.len() > 1 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS,
));
}
for transition in noncurrent_transitions {
NoncurrentVersionTransitionOps::validate(transition)?;
}
@@ -492,8 +473,6 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
async fn eval(&self, obj: &ObjectOpts) -> Event {
// A single-object lookup cannot prove how many newer historical versions
// survive. Count-dependent actions wait for the complete-group evaluator.
self.eval_inner(obj, OffsetDateTime::now_utc(), 0).await
}
@@ -557,8 +536,23 @@ impl Lifecycle for BucketLifecycleConfiguration {
return Event::default();
};
if let Some(event) = obj.restored_copy_expiry(now) {
events.push(event);
if let Some(restore_expires) = obj.restore_expires
&& restore_expires.unix_timestamp() != 0
&& now.unix_timestamp() > restore_expires.unix_timestamp()
{
let mut action = IlmAction::DeleteRestoredAction;
if !obj.is_latest {
action = IlmAction::DeleteRestoredVersionAction;
}
events.push(Event {
action,
due: Some(now),
rule_id: "".into(),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
storage_class: "".into(),
});
}
if let Some(ref lc_rules) = self.filter_rules(obj).await {
@@ -617,12 +611,17 @@ impl Lifecycle for BucketLifecycleConfiguration {
continue;
}
if !obj.is_latest
&& let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
&& let Some(retain_newer_noncurrent_versions) = noncurrent_version_expiration.newer_noncurrent_versions
&& newer_noncurrent_versions < usize::try_from(retain_newer_noncurrent_versions).unwrap_or(usize::MAX)
{
continue;
}
if !obj.is_latest
&& let Some(ref noncurrent_version_expiration) = rule.noncurrent_version_expiration
&& let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
&& noncurrent_version_expiration
.newer_noncurrent_versions
.is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
{
if let Some(successor_mod_time) = obj.successor_mod_time {
let expected_expiry = expected_expiry_time(successor_mod_time, noncurrent_days);
@@ -652,11 +651,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
&& let Some(noncurrent_version_transition) = rule
.noncurrent_version_transitions
.as_ref()
.filter(|transitions| transitions.len() == 1)
.and_then(|transitions| transitions.first())
&& noncurrent_version_transition
.newer_noncurrent_versions
.is_none_or(|retain| usize::try_from(retain).is_ok_and(|retain| newer_noncurrent_versions >= retain))
&& let Some(storage_class) = noncurrent_version_transition.storage_class.as_ref()
&& !storage_class.as_str().is_empty()
&& !obj.delete_marker
@@ -740,11 +735,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
if obj.transition_status != TRANSITION_COMPLETE
&& let Some(transition) = rule
.transitions
.as_ref()
.filter(|transitions| transitions.len() == 1)
.and_then(|transitions| transitions.first())
&& let Some(transition) = rule.transitions.as_ref().and_then(|transitions| transitions.first())
&& let Some(storage_class) = transition.storage_class.as_ref()
&& !storage_class.as_str().is_empty()
{
@@ -767,15 +758,18 @@ impl Lifecycle for BucketLifecycleConfiguration {
}
if !events.is_empty() {
// Eligible expiration takes precedence over transition, even when a
// failed transition has an earlier deadline. Within each action class,
// prefer the earliest deadline using a deterministic total order.
// Select the winning event using a strict total order (MinIO semantics):
// the earliest `due` wins, and ties break toward delete-type actions. A
// missing `due` is treated as UNIX_EPOCH. This replaces a hand-written
// `sort_by` comparator that was not a strict weak ordering (it could return
// `Ordering::Less` for both `(a, b)` and `(b, a)`), which panics on the
// repository toolchain and did not deterministically pick the earliest event.
let event = events
.iter()
.min_by_key(|event| {
(
ilm_action_priority_rank(&event.action),
event.due.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp(),
ilm_action_priority_rank(&event.action),
)
})
.cloned()
@@ -1048,27 +1042,6 @@ impl ObjectOpts {
pub fn expired_object_deletemarker(&self) -> bool {
self.delete_marker && self.is_latest && self.num_versions == 1
}
pub(crate) fn restored_copy_expiry(&self, now: OffsetDateTime) -> Option<Event> {
let restore_expires = self.restore_expires?;
// Restore metadata alone does not prove that a durable remote copy exists.
if self.transition_status != TRANSITION_COMPLETE
|| restore_expires.unix_timestamp() == 0
|| now.unix_timestamp() <= restore_expires.unix_timestamp()
{
return None;
}
let action = if self.is_latest {
IlmAction::DeleteRestoredAction
} else {
IlmAction::DeleteRestoredVersionAction
};
expiration_action_has_valid_target(action, self.version_id, self.is_latest, self.delete_marker).then(|| Event {
action,
due: Some(now),
..Default::default()
})
}
}
/// Returns whether an expiry action has enough identity to target the object
@@ -1091,8 +1064,11 @@ pub fn expiration_action_has_valid_target(
}
}
/// Eligible logical expiration takes precedence over transition and restore-copy
/// cleanup. Deadlines break ties within an action class.
/// Total-order rank for lifecycle actions used to break `due` ties.
///
/// Delete-type actions rank before every other action so that, when two events
/// share the same `due`, a delete wins (MinIO semantics). The concrete numeric
/// values only matter relative to each other.
fn ilm_action_priority_rank(action: &IlmAction) -> u8 {
match action {
IlmAction::DeleteAllVersionsAction
@@ -4183,392 +4159,6 @@ mod tests {
assert_eq!(event.action, IlmAction::NoneAction);
}
mod adversarial_regressions {
use super::*;
use s3s::dto::NoncurrentVersionExpiration;
fn run(test: impl std::future::Future<Output = ()>) {
with_default_ilm_process_time(|| {
tokio::runtime::Builder::new_current_thread()
.build()
.expect("lifecycle regression runtime should build")
.block_on(test);
});
}
fn noncurrent_object() -> ObjectOpts {
ObjectOpts {
name: "logs/object".to_string(),
mod_time: Some(datetime!(2020-01-01 00:00:00 UTC)),
successor_mod_time: Some(datetime!(2020-01-02 00:00:00 UTC)),
version_id: Some(Uuid::from_u128(1)),
size: 1024 * 1024,
..Default::default()
}
}
#[test]
#[serial]
fn noncurrent_transition_retains_the_requested_newer_versions() {
run(async {
let mut rule = enabled_rule(None, None, Some("retain-two-hot-versions"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = Arc::new(BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
});
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid noncurrent transition policy");
let objects = (0..4)
.map(|index| ObjectOpts {
mod_time: Some(datetime!(2020-01-05 00:00:00 UTC) - Duration::days(index)),
successor_mod_time: (index > 0).then_some(datetime!(2020-01-06 00:00:00 UTC) - Duration::days(index)),
version_id: Some(Uuid::from_u128(u128::try_from(index + 1).expect("small version index"))),
is_latest: index == 0,
num_versions: 4,
..noncurrent_object()
})
.collect::<Vec<_>>();
let actions = crate::Evaluator::new(lc)
.eval(&objects)
.await
.expect("complete version chain should evaluate")
.into_iter()
.map(|event| event.action)
.collect::<Vec<_>>();
assert_eq!(
actions,
[
IlmAction::NoneAction,
IlmAction::NoneAction,
IlmAction::NoneAction,
IlmAction::TransitionVersionAction
],
"the two newest noncurrent versions must remain in their current storage class"
);
});
}
#[test]
#[serial]
fn noncurrent_transition_checks_count_age_and_single_object_context() {
run(async {
let mut rule = enabled_rule(None, None, Some("retain-two"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(3),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid counted transition");
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
for (newer, expected) in [
(0, IlmAction::NoneAction),
(1, IlmAction::NoneAction),
(2, IlmAction::TransitionVersionAction),
(3, IlmAction::TransitionVersionAction),
] {
assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
}
assert_eq!(
lc.eval_inner(&object, datetime!(2020-01-04 00:00:00 UTC), 2).await.action,
IlmAction::NoneAction,
"the retention count does not replace the age condition"
);
assert_eq!(
lc.eval(&object).await.action,
IlmAction::NoneAction,
"a single-object lookup must not assume a complete version history"
);
for retain in [None, Some(0), Some(-1), Some(i32::MAX)] {
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition exists")[0]
.newer_noncurrent_versions = retain;
let expected = if matches!(retain, None | Some(0)) {
IlmAction::TransitionVersionAction
} else {
IlmAction::NoneAction
};
assert_eq!(lc.eval_inner(&object, now, 2).await.action, expected, "retention: {retain:?}");
}
});
}
#[test]
#[serial]
fn noncurrent_expiration_and_transition_have_independent_retention_counts() {
run(async {
let mut rule = enabled_rule(None, None, Some("independent-counts"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(90),
newer_noncurrent_versions: Some(4),
});
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(30),
newer_noncurrent_versions: Some(2),
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid independent retention limits");
let object = noncurrent_object();
let now = datetime!(2020-05-01 00:00:00 UTC);
for (newer, expected) in [
(1, IlmAction::NoneAction),
(2, IlmAction::TransitionVersionAction),
(3, IlmAction::TransitionVersionAction),
(4, IlmAction::DeleteVersionAction),
] {
assert_eq!(lc.eval_inner(&object, now, newer).await.action, expected, "newer count: {newer}");
}
});
}
#[test]
#[serial]
fn expiration_retention_does_not_skip_an_independent_transition() {
run(async {
let mut rule = enabled_rule(None, None, Some("transition-then-expire"));
rule.filter = Some(LifecycleRuleFilter::default());
rule.noncurrent_version_transitions = Some(vec![NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
let transition_only = lc.eval_inner(&object, now, 0).await;
assert_eq!(transition_only.action, IlmAction::TransitionVersionAction);
lc.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(90),
newer_noncurrent_versions: Some(2),
});
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid combined policy");
let combined = lc.eval_inner(&object, now, 0).await;
assert_eq!(combined.action, transition_only.action, "retention limits expiration, not transition");
assert_eq!(combined.storage_class, transition_only.storage_class);
});
}
#[test]
#[serial]
fn current_transition_rejects_multiple_stages_in_any_order() {
run(async {
let mut rule = enabled_rule(None, None, Some("two-current-transitions"));
rule.transitions = Some(vec![
Transition {
date: Some(datetime!(2020-03-01 00:00:00 UTC).into()),
days: None,
storage_class: Some(TransitionStorageClass::from_static("COLD")),
},
Transition {
date: Some(datetime!(2020-01-03 00:00:00 UTC).into()),
days: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
},
]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = ObjectOpts {
is_latest: true,
..noncurrent_object()
};
let now = datetime!(2020-01-10 00:00:00 UTC);
for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
lc.rules[0].status = ExpirationStatus::from_static(status);
for _ in 0..2 {
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("multiple transition stages must be rejected");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_TRANSITIONS);
assert_eq!(
lc.eval_inner(&object, now, 0).await.action,
IlmAction::NoneAction,
"legacy multi-stage configurations must not silently execute their first stage"
);
lc.rules[0]
.transitions
.as_mut()
.expect("transition array is present")
.reverse();
}
}
lc.rules[0]
.transitions
.as_mut()
.expect("transition array is present")
.remove(0);
lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("one stage is supported");
let event = lc.eval_inner(&object, now, 0).await;
assert_eq!(event.action, IlmAction::TransitionAction);
assert_eq!(event.storage_class, "WARM");
});
}
#[test]
#[serial]
fn noncurrent_transition_rejects_multiple_stages_in_any_order() {
run(async {
let mut rule = enabled_rule(None, None, Some("two-noncurrent-transitions"));
rule.noncurrent_version_transitions = Some(vec![
NoncurrentVersionTransition {
noncurrent_days: Some(30),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("COLD")),
},
NoncurrentVersionTransition {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
},
]);
let mut lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
let object = noncurrent_object();
let now = datetime!(2020-01-10 00:00:00 UTC);
for status in [ExpirationStatus::ENABLED, ExpirationStatus::DISABLED] {
lc.rules[0].status = ExpirationStatus::from_static(status);
for _ in 0..2 {
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("multiple noncurrent transition stages must be rejected");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_MULTIPLE_NONCURRENT_TRANSITIONS);
assert_eq!(
lc.eval_inner(&object, now, 0).await.action,
IlmAction::NoneAction,
"legacy multi-stage configurations must not silently execute their first stage"
);
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition array is present")
.reverse();
}
}
lc.rules[0]
.noncurrent_version_transitions
.as_mut()
.expect("transition array is present")
.remove(0);
lc.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::ENABLED);
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("one stage is supported");
let event = lc.eval_inner(&object, now, 0).await;
assert_eq!(event.action, IlmAction::TransitionVersionAction);
assert_eq!(event.storage_class, "WARM");
});
}
#[test]
#[serial]
fn expiration_rejects_simultaneous_days_and_date() {
run(async {
let mut lc = BucketLifecycleConfiguration {
rules: vec![enabled_rule(
Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
None,
Some("ambiguous-expiry"),
)],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("a single Days expiration is valid");
lc.rules[0].expiration.as_mut().expect("expiration is present").date =
Some(datetime!(2099-01-01 00:00:00 UTC).into());
let err = lc
.validate(&ObjectLockConfiguration::default())
.await
.expect_err("Days and Date are mutually exclusive; accepting both silently overrides Days");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert_eq!(err.to_string(), ERR_LIFECYCLE_EXPIRATION_DAYS_DATE_CONFLICT);
});
}
#[test]
#[serial]
fn overdue_transition_does_not_starve_permanent_expiration() {
run(async {
let mut rule = enabled_rule(
Some(LifecycleExpiration {
days: Some(90),
..Default::default()
}),
None,
Some("archive-then-delete"),
);
rule.transitions = Some(vec![Transition {
days: Some(30),
date: None,
storage_class: Some(TransitionStorageClass::from_static("WARM")),
}]);
let lc = BucketLifecycleConfiguration {
rules: vec![rule],
expiry_updated_at: None,
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("valid transition and expiration policy");
let object = ObjectOpts {
is_latest: true,
version_id: None,
transition_status: TRANSITION_PENDING.to_string(),
..noncurrent_object()
};
let before_expiration = lc.eval_inner(&object, datetime!(2020-02-15 00:00:00 UTC), 0).await;
assert_eq!(before_expiration.action, IlmAction::TransitionAction);
let overdue = lc.eval_inner(&object, datetime!(2020-05-01 00:00:00 UTC), 0).await;
assert_eq!(
overdue.action,
IlmAction::DeleteAction,
"an unavailable tier must not prevent permanent expiration indefinitely"
);
});
}
}
/// Property-based tests for the rule evaluator (backlog#1148 ilm-14,
/// follow-up to backlog#1030 / rustfs#4455).
///
@@ -4579,7 +4169,7 @@ mod tests {
///
/// * `eval_inner` never panics and is deterministic for a fixed input;
/// * the winning event matches an independently recomputed candidate set:
/// eligible expiration wins over transition, then earliest `due` wins (the
/// earliest `due` wins, ties break toward delete-class actions (the
/// `min_by_key` selection that replaced the rustfs#4455 comparator);
/// * `expected_expiry_time` is monotonically non-decreasing in `days` and
/// always lands on the processing boundary, both at production defaults
@@ -4868,8 +4458,8 @@ mod tests {
/// consider for a live current version under `selection`-shaped rules
/// (expiration and first-transition only, no filters): expiration
/// fires when `now >= due`, transition when `now > due` and the object
/// has not already transitioned. Eligible expiration wins over transition;
/// the earliest deadline wins within the selected action class.
/// has not already transitioned. Selection semantics under test:
/// earliest due wins, ties prefer delete-class.
fn oracle_candidates(lc: &BucketLifecycleConfiguration, obj: &ObjectOpts, now: OffsetDateTime) -> Vec<Candidate> {
let mod_time = obj.mod_time.expect("selection strategy always sets mod_time");
let mut candidates = Vec::new();
@@ -4958,8 +4548,8 @@ mod tests {
/// Differential test of winner selection (the rustfs#4455 fix):
/// for a live current version under randomized expiration and
/// transition rules, `eval_inner`'s winner must carry the
/// earliest expiration from the independently recomputed candidate
/// set, or the earliest transition when no expiration is eligible,
/// minimum `(due, rank)` of the independently recomputed
/// candidate set — earliest due wins, ties prefer delete-class —
/// and must be `NoneAction` exactly when that set is empty.
#[test]
#[serial]
@@ -4988,13 +4578,7 @@ mod tests {
// Oracle and evaluator must observe the same (pinned) time env.
let (event, expected) = with_production_time_env(|| {
let candidates = oracle_candidates(&lc, &obj, now);
let expected = candidates
.iter()
.filter(|(_, rank)| *rank == 0)
.min()
.copied()
.or_else(|| candidates.into_iter().min());
let expected = oracle_candidates(&lc, &obj, now).into_iter().min();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
+7 -93
View File
@@ -116,10 +116,13 @@ impl Evaluator {
break 'top_loop;
}
}
// Restore expiry removes only the temporary local copy; the
// retained logical version and its remote data remain intact.
IlmAction::DeleteAction | IlmAction::DeleteVersionAction if self.is_object_locked(obj) => {
event = obj.restored_copy_expiry(now).unwrap_or_default();
IlmAction::DeleteAction
| IlmAction::DeleteRestoredAction
| IlmAction::DeleteVersionAction
| IlmAction::DeleteRestoredVersionAction
if self.is_object_locked(obj) =>
{
event = Event::default();
}
_ => {}
}
@@ -203,95 +206,6 @@ mod tests {
use super::*;
use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
#[tokio::test]
async fn adversarial_restore_expiry_survives_legal_hold() {
let mut policy = (*latest_expiration_lifecycle()).clone();
policy.rules[0].status = ExpirationStatus::from_static(ExpirationStatus::DISABLED);
let policy = Arc::new(policy);
policy
.validate(&lock_enabled_without_default_retention())
.await
.expect("valid disabled lifecycle rule");
let mut objects = [true, false].map(|is_latest| ObjectOpts {
is_latest,
num_versions: 2,
mod_time: Some(
OffsetDateTime::from_unix_timestamp(if is_latest { 1_200_000 } else { 1_000_000 })
.expect("fixed version timestamp"),
),
successor_mod_time: (!is_latest)
.then(|| OffsetDateTime::from_unix_timestamp(1_200_000).expect("fixed successor timestamp")),
transition_status: crate::TRANSITION_COMPLETE.to_string(),
restore_expires: Some(OffsetDateTime::from_unix_timestamp(2_000_000).expect("fixed expired restore timestamp")),
..current_object_opts(ReplicationStatusType::Completed)
});
let evaluator = Evaluator::new(policy).with_lock_retention(Some(lock_enabled_without_default_retention()));
let expected = [IlmAction::DeleteRestoredAction, IlmAction::DeleteRestoredVersionAction];
let unlocked = evaluator
.eval(&objects)
.await
.expect("unlocked restored versions should evaluate");
assert_eq!(unlocked.iter().map(|event| event.action).collect::<Vec<_>>(), expected);
for object in &mut objects {
object
.user_defined
.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
}
let locked = evaluator
.eval(&objects)
.await
.expect("locked restored versions should evaluate");
assert_eq!(
locked.iter().map(|event| event.action).collect::<Vec<_>>(),
expected,
"expiring a restored local copy preserves the retained logical version and remote object"
);
let mut expiring_policy = (*latest_expiration_lifecycle()).clone();
expiring_policy.rules[0].noncurrent_version_expiration = Some(NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
});
let expiring_evaluator =
Evaluator::new(Arc::new(expiring_policy)).with_lock_retention(Some(lock_enabled_without_default_retention()));
let locked = expiring_evaluator
.eval(&objects)
.await
.expect("locked expired versions should evaluate");
assert_eq!(
locked.iter().map(|event| event.action).collect::<Vec<_>>(),
expected,
"blocked logical expiration must still allow an eligible restore-copy cleanup"
);
for status in [ReplicationStatusType::Pending, ReplicationStatusType::Failed] {
for object in &mut objects {
object.replication_status = status.clone();
}
for evaluator in [&evaluator, &expiring_evaluator] {
let events = evaluator.eval(&objects).await.expect("pending replication should evaluate");
assert!(events.iter().all(|event| event.action == IlmAction::NoneAction));
}
}
for object in &mut objects {
object.replication_status = ReplicationStatusType::Completed;
}
for transition_status in ["", crate::TRANSITION_PENDING, "unknown"] {
for object in &mut objects {
object.transition_status = transition_status.to_string();
}
for evaluator in [&evaluator, &expiring_evaluator] {
let events = evaluator.eval(&objects).await.expect("incomplete transition should evaluate");
assert!(
events.iter().all(|event| event.action == IlmAction::NoneAction),
"restore metadata cannot authorize cleanup without a completed transition"
);
}
}
}
fn expired_marker_lifecycle() -> Arc<BucketLifecycleConfiguration> {
Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
+48 -204
View File
@@ -120,10 +120,18 @@ impl TransitionClient {
let h = resp.headers().clone();
let mut body = resp.into_body();
let body_vec = if let Some(limit) = max_response_bytes {
self.collect_response_body(resp.into_body(), limit).await?
collect_response_body(body, limit).await?
} else {
self.collect_response_body_unbounded(resp.into_body()).await?
let mut body_vec = Vec::new();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
body_vec
};
Ok((object_stat, h, BufReader::new(Cursor::new(body_vec))))
}
@@ -135,7 +143,7 @@ mod bounded_response_tests {
use crate::{
api_get_options::GetObjectOptions,
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts, collect_response_body},
transition_api::{BucketLookupType, Options, TransitionClient, collect_response_body},
};
use http_body_util::Full;
use hyper::body::Bytes;
@@ -167,31 +175,7 @@ mod bounded_response_tests {
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
fn test_options() -> Options {
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
}
}
async fn client_for_endpoint(endpoint: &str, timeouts: TransitionClientTimeouts) -> TransitionClient {
TransitionClient::new_with_timeouts(endpoint, test_options(), "", timeouts)
.await
.expect("fixture client should build")
}
async fn bounded_get_fixture_with_timeouts(
body: &'static [u8],
timeouts: TransitionClientTimeouts,
) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
@@ -225,14 +209,27 @@ mod bounded_response_tests {
stream.write_all(body).await.expect("fixture should write response body");
request
});
let client = client_for_endpoint(&endpoint, timeouts).await;
let client = TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
)
.await
.expect("fixture client should build");
Some((client, request))
}
async fn bounded_get_fixture(body: &'static [u8]) -> Option<(TransitionClient, tokio::task::JoinHandle<String>)> {
bounded_get_fixture_with_timeouts(body, TransitionClientTimeouts::default()).await
}
#[tokio::test]
async fn real_transport_accepts_the_exact_closed_range_length() {
let Some((client, request)) = bounded_get_fixture(b"RustFS!").await else {
@@ -295,7 +292,24 @@ mod bounded_response_tests {
.local_addr()
.expect("listener local address should be available")
.to_string();
let client = client_for_endpoint(&endpoint, TransitionClientTimeouts::default()).await;
let client = TransitionClient::new(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
)
.await
.expect("fixture client should build");
let mut opts = GetObjectOptions::default();
opts.headers
.insert("range".to_string(), "bytes=0-18446744073709551615".to_string());
@@ -312,176 +326,6 @@ mod bounded_response_tests {
.is_err()
);
}
#[tokio::test]
async fn connection_refused_returns_without_waiting_for_the_request_timeout() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
drop(listener);
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(5), Duration::from_secs(1)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let result = tokio::time::timeout(Duration::from_secs(2), client.get_object_inner("bucket", "probe", &opts))
.await
.expect("connection refused should return before the broader request timeout");
assert!(result.is_err(), "connection refused must fail instead of hanging");
}
#[tokio::test]
async fn response_header_stall_returns_timed_out() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_millis(50), Duration::from_secs(1)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let err = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect_err("response header stalls must be bounded");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
#[tokio::test]
async fn response_body_idle_stall_returns_timed_out() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
stream
.write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 7\r\nConnection: close\r\n\r\nRu")
.await
.expect("fixture should write the first body chunk");
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_millis(50)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let err = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect_err("body stalls after partial progress must be bounded");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
#[tokio::test]
async fn response_body_idle_timer_resets_on_progress() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one GET");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
stream
.write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 7\r\nConnection: close\r\n\r\n")
.await
.expect("fixture should write response headers");
for byte in b"RustFS!" {
stream.write_all(&[*byte]).await.expect("fixture should write body progress");
tokio::time::sleep(Duration::from_millis(20)).await;
}
});
let client = client_for_endpoint(
&endpoint,
TransitionClientTimeouts::new(Duration::from_millis(10), Duration::from_secs(1), Duration::from_millis(100)),
)
.await;
let mut opts = GetObjectOptions::default();
opts.set_range(0, 6).expect("the probe range should be valid");
let (_, _, mut reader) = client
.get_object_inner("bucket", "probe", &opts)
.await
.expect("continuous body progress must not be killed by the idle timer");
let mut body = Vec::new();
reader
.read_to_end(&mut body)
.await
.expect("bounded response should be readable");
assert_eq!(body, b"RustFS!");
fixture.await.expect("fixture should join");
}
}
#[derive(Default)]
+10 -82
View File
@@ -27,6 +27,7 @@ use crate::{
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, collect_response_body},
};
use http::{HeaderMap, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
@@ -123,9 +124,14 @@ impl TransitionClient {
}
//let mut list_bucket_result = ListBucketV2Result::default();
let body_vec = self
.collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let mut list_bucket_result = match quick_xml::de::from_str::<ListBucketV2Result>(&String::from_utf8_lossy(&body_vec)) {
Ok(result) => result,
Err(err) => {
@@ -208,9 +214,7 @@ impl TransitionClient {
let resp_status = resp.status();
let headers = resp.headers().clone();
let body = self
.collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let body = collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE).await?;
if resp_status != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(
resp_status,
@@ -424,30 +428,6 @@ fn decode_s3_name(name: &str, encoding_type: &str) -> Result<String, std::io::Er
#[cfg(test)]
mod tests {
use super::*;
use crate::{
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClientTimeouts},
};
use std::time::Duration;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
};
fn timeout_test_options() -> Options {
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
}
}
#[test]
fn list_versions_xml_preserves_versions_and_delete_markers() {
@@ -545,56 +525,4 @@ mod tests {
assert_eq!(parsed.common_prefixes.len(), 1);
assert_eq!(parsed.common_prefixes[0].prefix, "subdir/");
}
#[tokio::test]
async fn list_objects_v2_body_stall_returns_timed_out() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one list request");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 512\r\nConnection: close\r\n\r\n<ListBucketResult><Name>warm")
.await
.expect("fixture should write a partial list response");
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = TransitionClient::new_with_timeouts(
&endpoint,
timeout_test_options(),
"",
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_millis(50)),
)
.await
.expect("fixture client should build");
client
.bucket_loc_cache
.lock()
.expect("location cache should lock")
.set("bucket", "us-east-1");
let err = client
.list_objects_v2_query("bucket", "", "", false, false, "", "", 1, HeaderMap::new())
.await
.expect_err("a stalled ListObjectsV2 body must be bounded");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
}
@@ -18,6 +18,7 @@
#![allow(clippy::all)]
use http::{HeaderMap, HeaderName, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Bytes;
use s3s::S3ErrorCode;
use std::collections::HashMap;
@@ -246,9 +247,14 @@ impl TransitionClient {
// Parse the CreateMultipartUpload response for the UploadId. Returning a
// default (empty) result here made every multipart transition fail at the
// first UploadPart with "UploadID cannot be empty" (rustfs/rustfs#4811).
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::other(e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let initiate_multipart_upload_result =
quick_xml::de::from_str::<InitiateMultipartUploadResult>(&String::from_utf8_lossy(&body_vec))
.map_err(|e| std::io::Error::other(format!("failed to parse CreateMultipartUpload response: {e}")))?;
+9 -3
View File
@@ -19,6 +19,7 @@
#![allow(clippy::all)]
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use rustfs_utils::HashAlgorithm;
@@ -350,9 +351,14 @@ impl TransitionClient {
)
.await?;
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
process_remove_multi_objects_response(
ReaderImpl::Body(Bytes::from(body_vec)),
bucket_name,
+11 -72
View File
@@ -19,6 +19,7 @@
#![allow(clippy::all)]
use http::{HeaderMap, HeaderValue, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
@@ -118,9 +119,14 @@ impl TransitionClient {
let resp_status = resp.status();
let h = resp.headers().clone();
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let resperr = http_resp_to_error_response(resp_status, &h, body_vec, bucket_name, "");
warn!("bucket exists, resperr: {:?}", resperr);
@@ -164,13 +170,11 @@ impl TransitionClient {
let resp_status = resp.status();
let h = resp.headers().clone();
let body_vec = self
.collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let body_vec = collect_response_body(resp.into_body(), rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE).await?;
parse_bucket_versioning_response(resp_status, &h, body_vec, bucket_name)
}
Err(err) => Err(err),
Err(err) => Err(std::io::Error::other(err)),
}
}
@@ -270,14 +274,8 @@ impl TransitionClient {
#[cfg(test)]
mod tests {
use super::parse_bucket_versioning_response;
use crate::{
credentials::{Credentials, SignatureType, Static, Value},
transition_api::{BucketLookupType, Options, TransitionClient, TransitionClientTimeouts},
};
use http::{HeaderMap, StatusCode};
use s3s::dto::BucketVersioningStatus;
use std::time::Duration;
use tokio::{io::AsyncReadExt, net::TcpListener};
#[test]
fn parses_bucket_versioning_statuses_mfa_delete_and_unversioned_state() {
@@ -340,63 +338,4 @@ mod tests {
assert_eq!(strict_err.kind(), std::io::ErrorKind::InvalidData);
}
}
#[tokio::test]
async fn get_bucket_versioning_preserves_request_timeout_kind() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let fixture = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("fixture should accept one versioning request");
let mut request = Vec::new();
let mut buffer = [0; 1024];
loop {
let read = stream.read(&mut buffer).await.expect("fixture should read request headers");
assert_ne!(read, 0, "connection closed before request headers were received");
request.extend_from_slice(&buffer[..read]);
if request.windows(4).any(|window| window == b"\r\n\r\n") {
break;
}
}
tokio::time::sleep(Duration::from_millis(200)).await;
});
let client = TransitionClient::new_with_timeouts(
&endpoint,
Options {
creds: Credentials::new(Static(Value {
access_key_id: "access-key".to_string(),
secret_access_key: "secret-key".to_string(),
signer_type: SignatureType::SignatureV4,
..Default::default()
})),
region: "us-east-1".to_string(),
bucket_lookup: BucketLookupType::BucketLookupPath,
max_retries: 1,
..Default::default()
},
"",
TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_millis(50), Duration::from_secs(1)),
)
.await
.expect("fixture client should build");
client
.bucket_loc_cache
.lock()
.expect("location cache should lock")
.set("bucket", "us-east-1");
let err = client
.get_bucket_versioning("bucket")
.await
.expect_err("a stalled versioning request must time out");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
fixture.await.expect("fixture should join");
}
}
+10 -5
View File
@@ -26,6 +26,7 @@ use crate::{
transition_api::{CreateBucketConfiguration, LocationConstraint, TransitionClient},
};
use http::Request;
use http_body_util::BodyExt;
use hyper::StatusCode;
use hyper::body::Body;
use hyper::body::Bytes;
@@ -85,7 +86,7 @@ impl TransitionClient {
let req = self.get_bucket_location_request(bucket_name)?;
let mut resp = self.doit(req).await?;
location = process_bucket_location_response(self, resp, bucket_name, &self.tier_type).await?;
location = process_bucket_location_response(resp, bucket_name, &self.tier_type).await?;
{
if let Ok(mut bucket_loc_cache) = self.bucket_loc_cache.lock() {
bucket_loc_cache.set(bucket_name, &location);
@@ -197,7 +198,6 @@ impl TransitionClient {
}
async fn process_bucket_location_response(
client: &TransitionClient,
mut resp: http::Response<Incoming>,
bucket_name: &str,
tier_type: &str,
@@ -237,9 +237,14 @@ async fn process_bucket_location_response(
}
//}
let body_vec = client
.collect_response_body(resp.into_body(), MAX_S3_CLIENT_RESPONSE_SIZE)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let mut location = "".to_string();
if tier_type == "huaweicloud" {
if let Ok(body_str) = String::from_utf8(body_vec) {
+41 -328
View File
@@ -41,7 +41,7 @@ use http::{
request::{Builder, Request},
};
use http_body::Body;
use http_body_util::BodyExt;
use http_body_util::{BodyExt, LengthLimitError, Limited};
use hyper::body::Bytes;
use hyper::body::Incoming;
use hyper_rustls::{ConfigBuilderExt, HttpsConnector};
@@ -67,12 +67,10 @@ use s3s::dto::Owner;
use s3s::dto::ReplicationStatus;
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use std::error::Error as StdError;
use std::io::Cursor;
use std::pin::Pin;
use std::sync::atomic::{AtomicI32, Ordering};
use std::task::{Context, Poll};
use std::time::Duration as StdDuration;
use std::{
collections::HashMap,
sync::{Arc, Mutex},
@@ -81,108 +79,28 @@ use time::Duration;
use time::OffsetDateTime;
use tokio::io::BufReader;
use tokio::io::{AsyncRead, AsyncReadExt};
use tracing::{debug, error, trace, warn};
use tracing::{debug, error, warn};
use url::{Url, form_urlencoded};
use uuid::Uuid;
const C_USER_AGENT: &str = "RustFS (linux; x86)";
pub const MAX_S3_ERROR_RESPONSE_SIZE: usize = 64 * 1024;
const EVENT_TIER_REMOTE_TRANSPORT: &str = "tier_remote_transport";
const LOG_COMPONENT_S3_CLIENT: &str = "s3_client";
const LOG_SUBSYSTEM_TIER: &str = "tier";
const SUCCESS_STATUS: [StatusCode; 3] = [StatusCode::OK, StatusCode::NO_CONTENT, StatusCode::PARTIAL_CONTENT];
fn response_body_exceeds_limit_error() -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit")
}
fn remote_tier_timeout_error(message: &'static str) -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::TimedOut, message)
}
fn source_chain_has_io_kind(error: &(dyn StdError + 'static), kind: std::io::ErrorKind) -> bool {
let mut current = Some(error);
while let Some(error) = current {
if error
.downcast_ref::<std::io::Error>()
.is_some_and(|io_error| io_error.kind() == kind)
{
return true;
}
current = error.source();
}
false
}
fn transition_transport_error(err: hyper_util::client::legacy::Error) -> std::io::Error {
if source_chain_has_io_kind(&err, std::io::ErrorKind::TimedOut) {
return remote_tier_timeout_error("remote tier connection timed out");
}
std::io::Error::other(err)
}
async fn next_response_body_data<B>(
mut body: Pin<&mut B>,
idle_timeout: Option<StdDuration>,
) -> Result<Option<Bytes>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
let next_nonempty_data = async {
loop {
let Some(frame) = std::future::poll_fn(|cx| body.as_mut().poll_frame(cx)).await else {
return Ok(None);
};
let frame = frame.map_err(std::io::Error::other)?;
let Ok(data) = frame.into_data() else {
continue;
};
if !data.is_empty() {
return Ok(Some(data));
}
}
};
if let Some(idle_timeout) = idle_timeout {
tokio::time::timeout(idle_timeout, next_nonempty_data)
.await
.map_err(|_| remote_tier_timeout_error("remote tier response body stalled"))?
} else {
next_nonempty_data.await
}
}
async fn collect_response_body_inner<B>(
body: B,
limit: Option<usize>,
idle_timeout: Option<StdDuration>,
) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
let mut body_vec = Vec::new();
let mut body = std::pin::pin!(body);
while let Some(data) = next_response_body_data(body.as_mut(), idle_timeout).await? {
let Some(new_len) = body_vec.len().checked_add(data.len()) else {
return Err(response_body_exceeds_limit_error());
};
if limit.is_some_and(|limit| new_len > limit) {
return Err(response_body_exceeds_limit_error());
}
body_vec.extend_from_slice(&data);
}
Ok(body_vec)
}
pub async fn collect_response_body<B>(body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
{
collect_response_body_inner(body, Some(limit), None).await
let body = Limited::new(body, limit).collect().await.map_err(|err| {
if err.is::<LengthLimitError>() {
std::io::Error::new(std::io::ErrorKind::InvalidData, "remote tier response body exceeds limit")
} else {
std::io::Error::other(err)
}
})?;
Ok(body.to_bytes().to_vec())
}
const C_UNKNOWN: i32 = -1;
@@ -278,62 +196,6 @@ pub struct TransitionClient {
pub trailing_header_support: bool,
pub max_retries: i64,
pub tier_type: String,
pub timeouts: TransitionClientTimeouts,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TransitionClientTimeouts {
pub connect_timeout: StdDuration,
pub request_timeout: StdDuration,
pub response_body_idle_timeout: StdDuration,
}
impl TransitionClientTimeouts {
pub const fn new(
connect_timeout: StdDuration,
request_timeout: StdDuration,
response_body_idle_timeout: StdDuration,
) -> Self {
Self {
connect_timeout,
request_timeout,
response_body_idle_timeout,
}
}
fn validate(self) -> Result<Self, std::io::Error> {
if self.connect_timeout.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"remote tier connect timeout must be greater than zero",
));
}
if self.request_timeout.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"remote tier request timeout must be greater than zero",
));
}
if self.response_body_idle_timeout.is_zero() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"remote tier response body idle timeout must be greater than zero",
));
}
Ok(self)
}
}
impl Default for TransitionClientTimeouts {
fn default() -> Self {
Self {
connect_timeout: StdDuration::from_secs(rustfs_config::DEFAULT_TIER_REMOTE_CONNECT_TIMEOUT_SECS),
request_timeout: StdDuration::from_secs(rustfs_config::DEFAULT_TIER_REMOTE_REQUEST_TIMEOUT_SECS),
response_body_idle_timeout: StdDuration::from_secs(
rustfs_config::DEFAULT_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS,
),
}
}
}
#[derive(Debug, Default)]
@@ -426,28 +288,12 @@ async fn build_tls_config() -> Result<rustls::ClientConfig, std::io::Error> {
impl TransitionClient {
pub async fn new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
Self::private_new(endpoint, opts, tier_type, TransitionClientTimeouts::default()).await
let client = Self::private_new(endpoint, opts, tier_type).await?;
Ok(client)
}
/// Builds a transition client with explicit transport timeout budgets.
///
/// [`Self::new`] keeps the historical constructor surface and uses the
/// production defaults from [`TransitionClientTimeouts::default`].
pub async fn new_with_timeouts(
endpoint: &str,
opts: Options,
tier_type: &str,
timeouts: TransitionClientTimeouts,
) -> Result<TransitionClient, std::io::Error> {
Self::private_new(endpoint, opts, tier_type, timeouts).await
}
async fn private_new(
endpoint: &str,
opts: Options,
tier_type: &str,
timeouts: TransitionClientTimeouts,
) -> Result<TransitionClient, std::io::Error> {
async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
if rustls::crypto::CryptoProvider::get_default().is_none() {
// No default provider is set yet; try to install aws-lc-rs.
// `install_default` can only fail if another thread races us and installs a provider
@@ -460,19 +306,15 @@ impl TransitionClient {
}
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
let timeouts = timeouts.validate()?;
let tls = build_tls_config().await?;
let mut http = HttpConnector::new();
http.enforce_http(false);
http.set_connect_timeout(Some(timeouts.connect_timeout));
let https = hyper_rustls::HttpsConnectorBuilder::new()
.with_tls_config(tls)
.https_or_http()
.enable_http1()
.enable_http2()
.wrap_connector(http);
.build();
let http_client = Client::builder(TokioExecutor::new()).build(https);
let mut client = TransitionClient {
@@ -495,7 +337,6 @@ impl TransitionClient {
trailing_header_support: opts.trailing_headers,
max_retries: opts.max_retries,
tier_type: tier_type.to_string(),
timeouts,
};
{
@@ -660,43 +501,29 @@ impl TransitionClient {
}
pub async fn doit(&self, req: Request<s3s::Body>) -> Result<Response<Incoming>, std::io::Error> {
let req_method;
let req_uri;
let resp;
let http_client = self.http_client.clone();
let req_method = req.method().clone();
let resp = tokio::time::timeout(self.timeouts.request_timeout, http_client.request(req)).await;
{
req_method = req.method().clone();
req_uri = req.uri().clone();
debug!("endpoint_url: {}", self.endpoint_url.as_str().to_string());
resp = http_client.request(req);
}
let resp = resp.await;
debug!("http_client url: {} {}", req_method, req_uri);
if let Err(err) = resp {
error!("http_client call error: {:?}", err);
return Err(std::io::Error::other(err));
}
let resp = match resp {
Ok(Ok(resp)) => resp,
Ok(Err(err)) => {
let err = transition_transport_error(err);
error!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
error_kind = ?err.kind(),
"remote tier request failed"
);
return Err(err);
}
Err(_) => {
warn!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
timeout_ms = self.timeouts.request_timeout.as_millis(),
"remote tier request timed out before response headers"
);
return Err(remote_tier_timeout_error("remote tier request timed out before response headers"));
}
Ok(r) => r,
Err(_) => return Err(std::io::Error::other("Unexpected error in response")),
};
trace!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
status = %resp.status(),
"remote tier response received"
);
debug!(status = %resp.status(), "remote tier response received");
//let b = resp.body_mut().store_all_unlimited().await.unwrap().to_vec();
//debug!("http_resp_body: {}", String::from_utf8(b).unwrap());
@@ -710,15 +537,7 @@ impl TransitionClient {
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_string();
warn!(
event = EVENT_TIER_REMOTE_TRANSPORT,
component = LOG_COMPONENT_S3_CLIENT,
subsystem = LOG_SUBSYSTEM_TIER,
method = %req_method,
status = %status,
request_id,
"remote tier request rejected"
);
warn!(status = %status, request_id, "remote tier request rejected");
}
Ok(resp)
}
@@ -762,9 +581,7 @@ impl TransitionClient {
let resp_status = resp.status();
let h = resp.headers().clone();
let body_vec = self
.collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE)
.await?;
let body_vec = collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE).await?;
let parsed_error =
http_resp_to_error_response(resp_status, &h, body_vec, &metadata.bucket_name, &metadata.object_name);
let routing_region = parsed_error.region;
@@ -818,22 +635,6 @@ impl TransitionClient {
Err(std::io::Error::other("remote tier request did not produce a response"))
}
pub async fn collect_response_body<B>(&self, body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
collect_response_body_inner(body, Some(limit), Some(self.timeouts.response_body_idle_timeout)).await
}
pub async fn collect_response_body_unbounded<B>(&self, body: B) -> Result<Vec<u8>, std::io::Error>
where
B: Body<Data = Bytes>,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
collect_response_body_inner(body, None, Some(self.timeouts.response_body_idle_timeout)).await
}
async fn new_request(
&self,
method: &http::Method,
@@ -1703,17 +1504,12 @@ pub struct CreateBucketConfiguration {
mod tests {
use super::{
MAX_S3_CLIENT_RESPONSE_SIZE, MAX_S3_ERROR_RESPONSE_SIZE, SignatureType, build_tls_config, collect_response_body,
collect_response_body_inner, signer_error_to_io_error, to_object_info_for_provider, validate_header_values,
with_rustls_init_guard,
signer_error_to_io_error, to_object_info_for_provider, validate_header_values, with_rustls_init_guard,
};
use crate::provider_versions::{BucketVersioningState, ProviderVersionCapabilities, RemoteVersion};
use futures::stream;
use http::{HeaderMap, HeaderValue, Request};
use http_body::Frame;
use http_body_util::{Full, StreamBody};
use http::{HeaderMap, HeaderValue};
use http_body_util::Full;
use hyper::body::Bytes;
use std::time::Duration as StdDuration;
use tokio::net::TcpListener;
use uuid::Uuid;
#[tokio::test]
@@ -1744,77 +1540,6 @@ mod tests {
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
#[tokio::test]
async fn empty_data_frames_do_not_reset_the_body_idle_timeout() {
let frames = stream::unfold((), |_| async {
tokio::time::sleep(StdDuration::from_millis(10)).await;
Some((Ok::<_, std::io::Error>(Frame::data(Bytes::new())), ()))
});
let body = StreamBody::new(Box::pin(frames));
let err = tokio::time::timeout(
StdDuration::from_millis(200),
collect_response_body_inner(body, Some(1), Some(StdDuration::from_millis(50))),
)
.await
.expect("the collector should enforce its own body idle timeout")
.expect_err("empty frames must not count as body progress");
assert_eq!(err.kind(), std::io::ErrorKind::TimedOut);
}
#[tokio::test]
async fn public_body_collector_accepts_non_unpin_bodies() {
let body = StreamBody::new(stream::once(async { Ok::<_, std::io::Error>(Frame::data(Bytes::from_static(b"ok"))) }));
let collected = collect_response_body(body, 2)
.await
.expect("the public collector should pin non-Unpin bodies internally");
assert_eq!(collected, b"ok");
}
#[tokio::test]
async fn https_endpoints_reach_the_transport_connector() {
let listener = match TcpListener::bind("127.0.0.1:0").await {
Ok(listener) => listener,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("test listener should bind: {err}"),
};
let endpoint = listener
.local_addr()
.expect("listener local address should be available")
.to_string();
let accepted = tokio::spawn(async move {
let (stream, _) = tokio::time::timeout(StdDuration::from_secs(1), listener.accept())
.await
.expect("HTTPS connector should reach the TCP listener")
.expect("fixture should accept the HTTPS connection");
drop(stream);
});
let client = super::TransitionClient::new_with_timeouts(
&endpoint,
super::Options {
secure: true,
..Default::default()
},
"",
super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::from_secs(1), StdDuration::from_secs(1)),
)
.await
.expect("fixture client should build");
let request = Request::builder()
.uri(format!("https://{endpoint}/"))
.body(s3s::Body::empty())
.expect("fixture request should build");
client
.doit(request)
.await
.expect_err("the fixture closes before completing the TLS handshake");
accepted.await.expect("fixture should join");
}
#[test]
fn rustls_guard_converts_panics_to_io_errors() {
let err = with_rustls_init_guard(|| -> Result<(), std::io::Error> { panic!("missing provider") })
@@ -1848,18 +1573,6 @@ mod tests {
assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
}
#[test]
fn transition_timeouts_reject_zero_budgets() {
for timeouts in [
super::TransitionClientTimeouts::new(StdDuration::ZERO, StdDuration::from_secs(1), StdDuration::from_secs(1)),
super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::ZERO, StdDuration::from_secs(1)),
super::TransitionClientTimeouts::new(StdDuration::from_secs(1), StdDuration::from_secs(1), StdDuration::ZERO),
] {
let err = timeouts.validate().expect_err("zero timeout budgets must fail closed");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
}
}
#[test]
fn validate_header_values_returns_header_name_for_non_utf8_values() {
let mut headers = HeaderMap::new();
-10
View File
@@ -7,16 +7,6 @@ On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** t
The module is on by default (rustfs/backlog#2163); set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off (`rustfs/src/module_switches.rs`). With the switch off, the runtime never intervenes on a read and the admin `PUT` route refuses with `OnDemandMigrationDisabled`. Reads of the configuration and of the status endpoint keep working while the switch is off, so a disabled deployment can still be inspected. The switch only decides whether the module may act at all: a bucket with no `on-demand-migration.json` is never resolved by the runtime and makes no source call, so turning the module on changes nothing for buckets you have not configured.
## List continuation token rollout
`RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false`; unset or invalid boolean values also keep it off. It controls only whether a v1 listing may first issue a v2 continuation token after an empty truncated merged page. Every node with this reader support accepts existing v2 tokens and continues their budget even with the switch off. Ordinary pages that consume an object or common prefix retain the original v1 token shape.
Leave the switch off while deploying v2 reader support to every node that can receive a continuation request, including nodes behind other load-balancer routes. Then set it to `true` in each node's environment and restart those nodes to enable issuance. A v1-only binary rejects v2 with `400 InvalidArgument` before the source-error policy runs; neither `not_found` nor turning off list-through makes that old reader compatible. With issuance still off, a new v1 chain retains the existing limitation: an empty source cursor cycle spanning requests can continue indefinitely. The default rollout does not claim to fix that chain until issuance is enabled.
An active v2 budget rejects the sixteenth consecutive merged page that consumes no new object/common prefix and reaches no new end-of-list state. The first fifteen empty pages can be resumed; with the existing two-fetch-per-side limit, that interval costs at most 32 fetches per side, including the failing request. A key, common prefix, or a newly exhausted side on the sixteenth request succeeds and resets the budget. A side that was already exhausted does not reset it again. This is a resource bound, not proof of a cursor cycle: an unusually long but valid empty source-page chain also reaches the limit. Tokens are unsigned base64 JSON, so this budget applies to clients that continue with the returned token unchanged; replaying or editing a token can reset it, and it is not a malicious-client defense or a global request quota. The two-fetch-per-side request limit and existing source rate limiter still apply. A source failure follows `policy.source_error`: `propagate` returns `424 SourceUnavailable` with `invalid_pagination`; `not_found` returns the fetched local listing with `x-rustfs-on-demand-migration-list: local_only`. A blocking local-side failure returns `InternalError`, without silently discarding local entries.
For rollback, first turn issuance off on every node. Keep v2-capable readers available for outstanding v2 chains: switching issuance off does not erase their budgets, and tokens have no expiration that proves those chains have drained. Route those continuations to compatible readers or have clients explicitly restart their listings before restoring v1-only binaries. Restarting a listing is a new scan and can repeat entries. Do not roll back readers while assuming the issuance switch makes existing v2 tokens disappear.
## Positioning
| Capability | Direction | What it moves | Where the authoritative copy is | When to use it instead |
@@ -29,18 +29,6 @@
Both knobs are read by the RustFS process that owns the replication target, at client build time; restart the server after changing them.
### Remote tier transport timeouts
Remote tier S3-compatible clients use separate transport budgets. These settings do not change bucket or site replication clients.
| Variable | Default | Meaning |
| --- | --- | --- |
| `RUSTFS_TIER_REMOTE_CONNECT_TIMEOUT_SECS` | `10` | Maximum time to establish the remote tier TCP connection. |
| `RUSTFS_TIER_REMOTE_REQUEST_TIMEOUT_SECS` | `86400` | Maximum time for a remote tier request to reach response headers. The long default preserves large transition-upload headroom. |
| `RUSTFS_TIER_REMOTE_RESPONSE_BODY_IDLE_TIMEOUT_SECS` | `60` | Maximum time without a non-empty response-body chunk. Empty HTTP/2 frames do not count as progress. |
All three values must be positive integers. Zero fails tier client initialization instead of silently disabling the boundary. An invalid integer is logged and falls back to the default; very large values are accepted and provide a correspondingly long effective budget. The values are read when the tier client is built; recreate or reload the tier configuration after changing them.
## Before changing any of this
Follow the SOP in `docs/postmortems/2026-09-03-replication-checksum-default-regression.md`: inventory the target-side rules the current default satisfies, run the outbound target matrix, and document any new knob here in the same PR.
-10
View File
@@ -22,16 +22,6 @@
| `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` |
| Dual-key internal metadata helpers (`insert_bytes` / `get_bytes`) | `crates/utils/src/http/metadata_compat.rs` |
## Lifecycle rule limits and evaluation
Each lifecycle rule supports at most one `Transition` and one `NoncurrentVersionTransition`. A version can make one initial transition; chaining additional tiers after it reaches `complete` is not supported. Splitting stages across overlapping rules does not enable a transition chain. `PutBucketLifecycleConfiguration` rejects multiple entries in either transition array with `InvalidArgument`, including in disabled rules. Existing stored multi-entry arrays are not executed; replace each with a single intended destination. Independent expiration actions in the rule remain eligible.
`Expiration.Days` and `Expiration.Date` are mutually exclusive. A request containing both is rejected instead of silently selecting the date. When expiration and transition are both eligible, expiration takes precedence; a failed earlier transition does not keep an expired object indefinitely. Deadlines select the earliest action within the same action class.
Noncurrent expiration and transition have independent `NewerNoncurrentVersions` limits. A transition with a positive limit waits for a complete version-group evaluation to establish that enough newer noncurrent versions remain. Single-object evaluation, including the current manual transition and immediate-enqueue paths, conservatively defers these counted transitions to the lifecycle scanner. An unmet expiration retention limit does not suppress a separately eligible transition.
An expired restored local copy can be cleaned up under Object Lock because the retained logical version and remote data remain intact. Cleanup requires a completed transition and still waits for pending or failed replication. The storage layer revalidates the source identity and restore metadata before removing the local copy; restore headers alone do not authorize cleanup.
## Free-version recovery controls
The dedicated free-version recovery loop is enabled by default and is independent of the data scanner and heal switches. Setting `RUSTFS_SCANNER_ENABLED=false` does not stop this repair loop. Set `RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED=false` before process startup to disable only the dedicated persisted-marker walk. That setting does not disable lifecycle workers or prevent another scanner path from discovering a free version, and it can leave remote cleanup markers pending for longer, so use it as a break-glass pressure control rather than a cleanup mechanism.
+13 -316
View File
@@ -26,8 +26,8 @@ use super::storage_api::bucket_usecase::ECStore;
use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
use super::storage_api::bucket_usecase::StorageObjectOptions;
use super::storage_api::bucket_usecase::bucket::on_demand_migration::{
BucketOdmState, ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
MergeSide, OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
BucketOdmState, ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, MergeSide,
OnDemandMigrationSys, SOURCE_LIST_MAX_RATE_WAIT, SourceClient, SourceError, SourceErrorPolicy, SourceListPlan,
SourceListRequest, SourceObject, SourcePage, decode_continuation_token, source_list_plan,
};
use super::storage_api::bucket_usecase::bucket::versioning_sys::BucketVersioningSys;
@@ -51,9 +51,6 @@ type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
/// yet, so the only class RustFS can vouch for is the default one.
const SOURCE_STORAGE_CLASS: &str = "STANDARD";
/// Enable only after every node serving continuation requests can read v2.
const ENV_LIST_PROGRESS_TOKENS: &str = "RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS";
/// Concurrent local metadata probes when a versioned bucket has to check
/// source-only keys for a shadowing delete marker.
const DELETE_MARKER_PROBE_CONCURRENCY: usize = 32;
@@ -267,18 +264,7 @@ pub(crate) async fn merged_list_objects_v2(
buffers[usize::from(fetch.side == MergeSide::Source)].extend(kept.into_iter().map(Some));
}
let issue_progress_tokens = rustfs_utils::get_env_bool(ENV_LIST_PROGRESS_TOKENS, false);
let outcome = match merger.finish(issue_progress_tokens) {
Ok(outcome) => outcome,
Err(ListPageError::NoProgress(MergeSide::Source)) => {
degrade_or_fail(&mut merger, &mut degraded, policy.source_error, "invalid_pagination")?;
merger
.finish(issue_progress_tokens)
.map_err(|error| S3Error::with_message(S3ErrorCode::InternalError, error.to_string()))?
}
Err(error) => return Err(S3Error::with_message(S3ErrorCode::InternalError, error.to_string())),
};
drop(merger);
let outcome = merger.finish();
let mut objects = Vec::with_capacity(outcome.picks.len());
let mut prefixes = Vec::new();
let mut source_only_keys = Vec::new();
@@ -444,8 +430,7 @@ mod tests {
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
use crate::app::storage_api::bucket_usecase::bucket::on_demand_migration::{
FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig,
SourceCredentials, TlsConfig,
FilterConfig, OnDemandMigrationConfig, PathStyle, PolicyConfig, Provider, SourceConfig, SourceCredentials, TlsConfig,
};
use crate::app::storage_api::bucket_usecase::s3::{ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response};
use crate::app::storage_api::test::StoragePutObjReader;
@@ -463,7 +448,6 @@ mod tests {
source: Some("source-2".to_string()),
source_done: false,
last_key: Some("k".to_string()),
no_progress: None,
}
}
@@ -531,24 +515,6 @@ mod tests {
assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted));
}
#[test]
fn a_v2_token_keeps_the_local_cursor_when_list_through_is_turned_off() {
let mut resume = token(Some("local-2"), false);
resume.v = 2;
resume.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 1);
let encoded = resume.encode();
let decoded = decode_list_cursor(Some(&encoded)).expect("a v2 envelope decodes");
assert_eq!(decoded.as_ref(), Some(&resume));
assert!(matches!(
local_cursor(Some(&encoded), decoded.as_ref()),
LocalListCursor::Token(Some(local)) if local == "local-2"
));
resume.local_done = true;
let encoded = resume.encode();
let decoded = decode_list_cursor(Some(&encoded)).expect("v2 with local EOF decodes");
assert!(matches!(local_cursor(Some(&encoded), decoded.as_ref()), LocalListCursor::Exhausted));
}
#[test]
fn a_plain_local_token_is_passed_through_and_a_tampered_one_is_rejected() {
assert!(
@@ -585,30 +551,14 @@ mod tests {
/// Serves exactly the scripted S3 pages and joins every connection before
/// returning. A source retry or unexpected operation fails the test.
async fn scripted_list_source(pages: Vec<String>) -> (String, tokio_util::task::AbortOnDropHandle<Vec<String>>) {
let (endpoint, server, _) = list_source(pages.into_iter()).await;
(endpoint, server)
}
async fn list_source(
pages: impl Iterator<Item = String> + Send + 'static,
) -> (
String,
tokio_util::task::AbortOnDropHandle<Vec<String>>,
tokio_util::sync::CancellationToken,
) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind listing source");
let address = listener.local_addr().expect("listing source address");
let stop = tokio_util::sync::CancellationToken::new();
let server_stop = stop.clone();
let server = tokio::spawn(async move {
let mut requests = Vec::new();
for body in pages {
let (mut stream, _) = tokio::select! {
_ = server_stop.cancelled() => break,
accepted = listener.accept() => accepted.expect("accept source listing"),
};
let (mut stream, _) = listener.accept().await.expect("accept source listing");
let mut request = Vec::new();
let mut chunk = [0; 4096];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
@@ -638,7 +588,7 @@ mod tests {
}
requests
});
(format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server), stop)
(format!("http://{address}"), tokio_util::task::AbortOnDropHandle::new(server))
}
fn source_xml(next: Option<&str>, truncated: bool, key: Option<&str>) -> String {
@@ -666,12 +616,12 @@ mod tests {
}
}
async fn source_policy_input(
endpoint: String,
async fn source_policy_request(
pages: Vec<String>,
policy: SourceErrorPolicy,
resume_source: Option<&str>,
filter_prefix: Option<&str>,
) -> (ListThroughTestState, ListObjectsV2Input) {
) -> (S3Result<S3Response<ListObjectsV2Output>>, Vec<String>) {
let store = shared_gating_ecstore().await;
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
let bucket = format!("odm-list-{}", uuid::Uuid::new_v4().simple());
@@ -688,8 +638,9 @@ mod tests {
)
.await
.expect("seed real local listing");
let (endpoint, server) = scripted_list_source(pages).await;
let sys = OnDemandMigrationSys::get();
let state_guard = ListThroughTestState {
let _state_guard = ListThroughTestState {
bucket: bucket.clone(),
module_enabled: sys.is_module_enabled(),
};
@@ -734,7 +685,6 @@ mod tests {
source: Some(source.into()),
source_done: false,
last_key: None,
no_progress: None,
};
base64_simd::STANDARD.encode_to_string(token.encode().as_bytes())
});
@@ -751,10 +701,6 @@ mod tests {
request_payer: None,
start_after: None,
};
(state_guard, input)
}
async fn execute_source_list(input: ListObjectsV2Input) -> S3Result<S3Response<ListObjectsV2Output>> {
let request = S3Request {
input,
method: http::Method::GET,
@@ -766,23 +712,12 @@ mod tests {
service: None,
trailing_headers: None,
};
tokio::time::timeout(
let result = tokio::time::timeout(
Duration::from_secs(10),
DefaultBucketUsecase::from_global().execute_list_objects_v2(request),
)
.await
.expect("listing must complete within its bounded source budget")
}
async fn source_policy_request(
pages: Vec<String>,
policy: SourceErrorPolicy,
resume_source: Option<&str>,
filter_prefix: Option<&str>,
) -> (S3Result<S3Response<ListObjectsV2Output>>, Vec<String>) {
let (endpoint, server) = scripted_list_source(pages).await;
let (_state_guard, input) = source_policy_input(endpoint, policy, resume_source, filter_prefix).await;
let result = execute_source_list(input).await;
.expect("listing must complete within its bounded source budget");
let requests = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("source connections must finish")
@@ -907,244 +842,6 @@ mod tests {
});
}
#[test]
#[serial_test::serial]
fn list_through_cross_request_empty_cursor_cycle_obeys_policy() {
run_large_stack_test("list-through-cross-request-cursor-cycle", || async {
temp_env::async_with_vars(
[
(ENV_LIST_PROGRESS_TOKENS, Some("true")),
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
("HTTP_PROXY", None),
("HTTPS_PROXY", None),
("ALL_PROXY", None),
("http_proxy", None),
("https_proxy", None),
("all_proxy", None),
("NO_PROXY", Some("*")),
("no_proxy", Some("*")),
],
async {
for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] {
let pages = ["B", "C", "A"].map(|next| source_xml(Some(next), true, None));
let (endpoint, server, stop) = list_source(pages.into_iter().cycle()).await;
let (_state_guard, mut input) = source_policy_input(endpoint, policy, Some("A"), None).await;
let mut seen = std::collections::HashSet::from([input
.continuation_token
.clone()
.expect("the first request resumes source cursor A")]);
let mut client_requests = 0;
let mut empty_pages = 0;
let terminal = tokio::time::timeout(Duration::from_secs(30), async {
loop {
client_requests += 1;
let response = match execute_source_list(input.clone()).await {
Ok(response) => response,
Err(error) => break Err(error),
};
if response.headers.contains_key("x-rustfs-on-demand-migration-list") {
break Ok(response);
}
let output = response.output;
assert!(output.contents.as_ref().is_none_or(Vec::is_empty));
assert!(output.common_prefixes.as_ref().is_none_or(Vec::is_empty));
assert_eq!(output.key_count, Some(0));
assert_eq!(output.is_truncated, Some(true));
let next = output
.next_continuation_token
.expect("a truncated page must carry its cursor");
assert!(
seen.insert(next.clone()),
"a cross-request source cursor cycle must not return an identical empty merged token"
);
empty_pages += 1;
input.continuation_token = Some(next);
}
})
.await
.expect("a source cursor cycle must terminate within a bounded client pagination chain");
assert_eq!(empty_pages, usize::from(MAX_LIST_NO_PROGRESS_PAGES - 1));
assert_eq!(client_requests, usize::from(MAX_LIST_NO_PROGRESS_PAGES));
assert_source_policy_result(terminal, policy);
stop.cancel();
let requests = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("cyclic source server must stop")
.expect("cyclic source server must not panic");
assert_eq!(requests.len(), 2 * client_requests, "the sixteenth empty page exhausts the budget");
for (index, request) in requests.iter().enumerate() {
let source_cursor = ["A", "B", "C"][index % 3];
assert!(
request.contains(&format!("continuation-token={source_cursor}")),
"the real SDK must follow the returned source cursor: {request}"
);
}
}
},
)
.await;
});
}
#[test]
#[serial_test::serial]
fn list_through_default_rollout_continues_v2_without_issuing_it_from_v1() {
run_large_stack_test("list-through-reader-first-rollout", || async {
temp_env::async_with_vars(
[
(ENV_LIST_PROGRESS_TOKENS, None),
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
("HTTP_PROXY", None),
("HTTPS_PROXY", None),
("ALL_PROXY", None),
("http_proxy", None),
("https_proxy", None),
("all_proxy", None),
("NO_PROXY", Some("*")),
("no_proxy", Some("*")),
],
async {
for policy in [SourceErrorPolicy::Propagate, SourceErrorPolicy::NotFound] {
let pages = ["B", "C", "A"].map(|next| source_xml(Some(next), true, None));
let (endpoint, server, stop) = list_source(pages.into_iter().cycle()).await;
let (_state_guard, mut input) = source_policy_input(endpoint, policy, Some("A"), None).await;
let original = input.continuation_token.clone();
for _ in 0..3 {
let response = execute_source_list(input.clone()).await.expect("reader-only v1 behavior");
assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
assert_eq!(response.output.key_count, Some(0));
assert_eq!(response.output.is_truncated, Some(true));
let next = response.output.next_continuation_token.expect("resumable empty page");
let raw = base64_simd::STANDARD.decode_to_vec(&next).expect("base64 continuation token");
let decoded = std::str::from_utf8(&raw).expect("JSON token");
let token = decode_list_cursor(Some(decoded)).expect("v1 reader").expect("merged token");
assert_eq!(token.v, 1, "the default rollout cannot begin issuing v2");
assert_eq!(token.no_progress, None);
assert!(!decoded.contains("no_progress"), "ordinary v1 wire shape stays unchanged");
input.continuation_token = Some(next);
}
assert_eq!(input.continuation_token, original, "default rollout retains the known v1 limitation");
let raw = base64_simd::STANDARD
.decode_to_vec(input.continuation_token.as_ref().expect("v1 token"))
.expect("base64 continuation token");
let mut token = decode_list_cursor(Some(std::str::from_utf8(&raw).expect("JSON token")))
.expect("v1 reader")
.expect("merged token");
token.v = 2;
token.no_progress = Some(MAX_LIST_NO_PROGRESS_PAGES - 2);
input.continuation_token = Some(base64_simd::STANDARD.encode_to_string(token.encode().as_bytes()));
let response = execute_source_list(input.clone()).await.expect("reader-only node resumes v2");
assert_eq!(response.output.key_count, Some(0));
assert_eq!(response.output.is_truncated, Some(true));
let next = response.output.next_continuation_token.expect("last allowed empty cursor");
let raw = base64_simd::STANDARD.decode_to_vec(&next).expect("base64 continuation token");
let token = decode_list_cursor(Some(std::str::from_utf8(&raw).expect("JSON token")))
.expect("v2 reader")
.expect("merged token");
assert_eq!(token.v, 2);
assert_eq!(token.no_progress, Some(MAX_LIST_NO_PROGRESS_PAGES - 1));
input.continuation_token = Some(next);
assert_source_policy_result(execute_source_list(input).await, policy);
stop.cancel();
let requests = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("cyclic source server must stop")
.expect("source server must not panic");
assert_eq!(requests.len(), 10, "five handler requests each fetched two source pages");
for (index, request) in requests.iter().enumerate() {
let cursor = ["A", "B", "C"][index % 3];
assert!(request.contains(&format!("continuation-token={cursor}")), "{request}");
}
}
},
)
.await;
});
}
#[test]
#[serial_test::serial]
fn list_through_empty_advancing_pages_resume_across_handler_requests() {
run_large_stack_test("list-through-resumable-empty-pages", || async {
temp_env::async_with_vars(
[
(ENV_LIST_PROGRESS_TOKENS, Some("true")),
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", Some("true")),
("HTTP_PROXY", None),
("HTTPS_PROXY", None),
("ALL_PROXY", None),
("http_proxy", None),
("https_proxy", None),
("all_proxy", None),
("NO_PROXY", Some("*")),
("no_proxy", Some("*")),
],
async {
for filter_prefix in [None, Some("photos/2024/")] {
let source_key = filter_prefix.map_or("a-source", |_| "photos/2024/a-source");
let (endpoint, server) = scripted_list_source(vec![
source_xml(Some("A"), true, None),
source_xml(Some("B"), true, None),
source_xml(Some("C"), true, None),
source_xml(None, false, Some(source_key)),
])
.await;
let (_state_guard, mut input) =
source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, filter_prefix).await;
let first = execute_source_list(input.clone())
.await
.expect("valid empty pages must remain resumable");
assert!(!first.headers.contains_key("x-rustfs-on-demand-migration-list"));
assert!(first.output.contents.as_ref().is_none_or(Vec::is_empty));
assert!(first.output.common_prefixes.as_ref().is_none_or(Vec::is_empty));
assert_eq!(first.output.key_count, Some(0));
assert_eq!(first.output.is_truncated, Some(true));
input.continuation_token = Some(first.output.next_continuation_token.expect("empty advancing cursor"));
let second = execute_source_list(input)
.await
.expect("a progressing empty chain must reach its data");
assert!(!second.headers.contains_key("x-rustfs-on-demand-migration-list"));
let output = second.output;
let objects = output
.contents
.unwrap_or_default()
.into_iter()
.map(|object| object.key.expect("listed object key"))
.collect::<Vec<_>>();
let prefixes = output
.common_prefixes
.unwrap_or_default()
.into_iter()
.map(|prefix| prefix.prefix.expect("listed common prefix"))
.collect::<Vec<_>>();
if filter_prefix.is_some() {
assert_eq!(objects, vec!["z-local"]);
assert_eq!(prefixes, vec!["photos/"]);
} else {
assert_eq!(objects, vec!["a-source", "z-local"]);
assert!(prefixes.is_empty());
}
assert_eq!(output.key_count, Some(2));
assert_eq!(output.is_truncated, Some(false));
assert!(output.next_continuation_token.is_none());
let requests = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("finite source connections must finish")
.expect("finite source server must not panic");
assert_eq!(requests.len(), 4);
assert!(!requests[0].contains("continuation-token="));
for (request, cursor) in requests[1..].iter().zip(["A", "B", "C"]) {
assert!(request.contains(&format!("continuation-token={cursor}")), "{request}");
}
}
},
)
.await;
});
}
fn assert_source_policy_result(result: S3Result<S3Response<ListObjectsV2Output>>, policy: SourceErrorPolicy) {
match policy {
SourceErrorPolicy::Propagate => {
+4 -4
View File
@@ -634,8 +634,8 @@ pub(crate) mod bucket {
};
#[cfg(test)]
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, MAX_LIST_NO_PROGRESS_PAGES, OnDemandMigrationConfig,
PathStyle, Provider, SourceConfig, SourceCredentials, TlsConfig,
BREAKER_FAILURE_THRESHOLD, BreakerState, FilterConfig, OnDemandMigrationConfig, PathStyle, Provider, SourceConfig,
SourceCredentials, TlsConfig,
};
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
BucketOdmState, HeadPolicy, OdmLookup, OdmOp, OdmOutcome, OdmStateError, OnDemandMigrationSys, PolicyConfig,
@@ -643,8 +643,8 @@ pub(crate) mod bucket {
commit_inline, idle_guarded_body,
};
pub(crate) use crate::storage::storage_api::ecstore_bucket::on_demand_migration::{
ListEntryKey, ListPageError, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError,
MergeSide, SOURCE_LIST_MAX_RATE_WAIT, SourceListPlan, decode_continuation_token, source_list_plan,
ListEntryKey, ListThroughCursor, ListThroughMerger, ListThroughToken, ListThroughTokenError, MergeSide,
SOURCE_LIST_MAX_RATE_WAIT, SourceListPlan, decode_continuation_token, source_list_plan,
};
}
+6 -201
View File
@@ -5,9 +5,7 @@ from __future__ import annotations
import hashlib
import json
import os
import re
import subprocess
import sys
import tempfile
import tomllib
@@ -483,20 +481,18 @@ def yaml_block(lines: list[str], key: str, indent: int) -> list[str] | None:
return lines[start:end]
def workflow_step_block(
job_lines: list[str], value: str, key: str = "uses", indent: int = 6
) -> tuple[int, list[str]] | None:
def workflow_step_block(job_lines: list[str], action: str) -> tuple[int, list[str]] | None:
uses_index = next(
(
index
for index, line in enumerate(job_lines)
if (
line.split("#", 1)[0].strip() == f"- {key}: {value}"
and len(line) - len(line.lstrip()) == indent
line.split("#", 1)[0].strip() == f"- uses: {action}"
and len(line) - len(line.lstrip()) == 6
)
or (
line.split("#", 1)[0].strip() == f"{key}: {value}"
and len(line) - len(line.lstrip()) == indent + 2
line.split("#", 1)[0].strip() == f"uses: {action}"
and len(line) - len(line.lstrip()) == 8
)
),
None,
@@ -524,67 +520,6 @@ def workflow_step_block(
return start, job_lines[start:end]
def yaml_scalar_continues(lines: list[str], index: int, indent: int) -> bool:
following = next(
(line for line in lines[index + 1:] if line.strip() and not line.lstrip().startswith("#")), None
)
return following is not None and len(following) - len(following.lstrip()) > indent
def check_quick_checks(root: Path) -> list[str]:
errors: list[str] = []
bypass_key = r'''(?:if|continue-on-error|needs|"if"|"continue-on-error"|"needs"|'if'|'continue-on-error'|'needs')\s*:'''
for name in ("ci.yml", "ci-docs-only.yml"):
relative = f".github/workflows/{name}"
path = root / relative
job = yaml_block(path.read_text().splitlines(), "quick-checks", 2) if path.is_file() else None
if job is None:
errors.append(f"{relative}: missing Quick Checks job")
continue
conditions = [index for index, line in enumerate(job) if re.match(rf"^ {bypass_key}", line)]
expected = ["if: github.event_name != 'pull_request' || github.event.action != 'closed'"] if name == "ci.yml" else []
if [job[index].strip() for index in conditions] != expected or any(
yaml_scalar_continues(job, index, 4) for index in conditions
):
errors.append(f"{relative}: Quick Checks job must not add dependencies, bypass failures, or change its event condition")
checkout = workflow_step_block(job, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0")
action = workflow_step_block(job, "./.github/actions/quick-checks")
if checkout is None or action is None:
errors.append(f"{relative}: Quick Checks requires checkout and the shared quick-checks action")
continue
if checkout[0] >= action[0]:
errors.append(f"{relative}: checkout must run before shared Quick Checks")
if " persist-credentials: false" not in checkout[1]:
errors.append(f"{relative}: Quick Checks checkout must disable persisted credentials")
for step in (checkout, action):
if any(re.match(rf"^\s+(?:- )?{bypass_key}", line) for line in step[1]):
errors.append(f"{relative}: Quick Checks checkout and shared action must run without bypasses")
relative = ".github/actions/quick-checks/action.yml"
path = root / relative
runs = yaml_block(path.read_text().splitlines(), "runs", 0) if path.is_file() else None
if runs is None or " using: composite" not in runs:
errors.append(f"{relative}: missing composite action")
return errors
steps = yaml_block(runs, "steps", 2) or []
for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh"):
step = workflow_step_block(steps, command, key="run", indent=4)
if step is None:
errors.append(f"{relative}: missing direct execution of {command}")
continue
if " shell: bash" not in step[1] or any(
re.match(rf"^\s+(?:- )?{bypass_key}", line) for line in step[1]
):
errors.append(f"{relative}: {command} must use bash without a condition or continue-on-error")
run_index = next(
index for index, line in enumerate(step[1])
if line.split("#", 1)[0].rstrip() in (f" run: {command}", f" - run: {command}")
)
if yaml_scalar_continues(step[1], run_index, 6):
errors.append(f"{relative}: {command} must remain a single-line run scalar")
return errors
def alert_step_errors(
job_lines: list[str],
expected_action_if: str | None,
@@ -885,139 +820,10 @@ def validate(root: Path) -> list[str]:
errors.extend(check_workflow_readiness(root))
errors.extend(check_profile_definitions(root))
errors.extend(check_scheduled_alerts(root))
errors.extend(check_quick_checks(root))
return errors
class SelfTests(unittest.TestCase):
def test_quick_checks_rejects_caller_and_execution_bypasses(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
caller = (
"jobs:\n quick-checks:\n steps:\n"
" - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
" with:\n persist-credentials: false\n"
" - uses: ./.github/actions/quick-checks\n"
)
action = (
"runs:\n using: composite\n steps:\n"
" - uses: taiki-e/install-action@pinned\n"
" with:\n tool: actionlint@1.7.12\n"
" - name: Lint workflows\n shell: bash\n run: shellcheck --version && actionlint\n"
" - name: Error format ratchet\n shell: bash\n"
" run: ./scripts/check_error_other_format_ratchet.sh\n"
)
sources = {
".github/workflows/ci.yml": caller.replace(
" steps:", " if: github.event_name != 'pull_request' || github.event.action != 'closed'\n steps:"
),
".github/workflows/ci-docs-only.yml": caller,
".github/actions/quick-checks/action.yml": action,
}
for relative, source in sources.items():
path = root / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(source)
self.assertEqual(check_quick_checks(root), [])
for relative in (".github/workflows/ci.yml", ".github/workflows/ci-docs-only.yml"):
source = sources[relative]
mutations = {
"different action": source.replace("./.github/actions/quick-checks", "./.github/actions/other"),
"conditional call": source + " if: false\n",
"ignored call failure": source + " continue-on-error: true\n",
"conditional checkout": source.replace(" with:", " if: false\n with:"),
"ignored job failure": source.replace(" steps:", " continue-on-error: true\n steps:"),
"changed job condition": (
source.replace("github.event_name != 'pull_request' || github.event.action != 'closed'", "false")
if relative.endswith("/ci.yml") else source.replace(" steps:", " if: false\n steps:")
),
"persisted credentials": source.replace("persist-credentials: false", "persist-credentials: true"),
"late checkout": source.replace(" - uses: ./.github/actions/quick-checks\n", "").replace(
" steps:\n", " steps:\n - uses: ./.github/actions/quick-checks\n"
),
"missing job": source.replace(" quick-checks:", " other-checks:"),
}
for key in ("'if' : false", '"if": false', "'continue-on-error': true", '"continue-on-error" : true'):
mutations[f"quoted call {key}"] = source + f" {key}\n"
mutations[f"quoted checkout {key}"] = source.replace(" with:", f" {key}\n with:")
job_source = source.replace(
" if: github.event_name != 'pull_request' || github.event.action != 'closed'\n", ""
) if "if" in key else source
mutations[f"quoted job {key}"] = job_source.replace(" steps:", f" {key}\n steps:")
for dependency in ("needs: prerequisite", "needs: [prerequisite]", "needs:\n - prerequisite", "'needs' : [prerequisite]", '"needs": [prerequisite]'):
for condition in ("false", "true"):
prerequisite = f"\n prerequisite:\n if: {condition}\n runs-on: ubuntu-latest\n steps:\n - run: exit 1\n"
mutations[f"job dependency {dependency} if {condition}"] = source.replace(" steps:", f" {dependency}\n steps:") + prerequisite
if relative.endswith("/ci.yml"):
for separator in ("", "\n", " # continued condition\n"):
mutations[f"continued job condition {separator!r}"] = source.replace(
" steps:", f"{separator} && false\n steps:"
)
for case, mutated in mutations.items():
with self.subTest(path=relative, case=case):
(root / relative).write_text(mutated)
self.assertTrue(check_quick_checks(root))
(root / relative).write_text(source)
relative = ".github/actions/quick-checks/action.yml"
mutations = {
"not composite": action.replace("using: composite", "using: node24"),
"only installed actionlint": action.replace("run: shellcheck --version && actionlint", "run: echo actionlint"),
"missing shellcheck preflight": action.replace("shellcheck --version && ", ""),
"missing ratchet": action.replace("run: ./scripts/check_error_other_format_ratchet.sh", "run: echo skipped"),
"swallowed lint failure": action.replace("&& actionlint", "&& actionlint || true"),
"swallowed ratchet failure": action.replace("ratchet.sh", "ratchet.sh || true"),
"conditional lint": action.replace("run: shellcheck", "if: false\n run: shellcheck"),
"ignored ratchet failure": action.replace("run: ./scripts/", "continue-on-error: true\n run: ./scripts/"),
"non-failing shell": action.replace("shell: bash", "shell: bash {0}"),
"run text in step name": action.replace(
"name: Lint workflows", "name: |\n run: shellcheck --version && actionlint"
).replace("\n run: shellcheck --version && actionlint\n", "\n run: shellcheck --version && actionlint\n || true\n"),
}
for command in ("shellcheck --version && actionlint", "./scripts/check_error_other_format_ratchet.sh"):
for key in ("'if' : false", '"if": false', "'continue-on-error': true", '"continue-on-error" : true'):
mutations[f"quoted {command} {key}"] = action.replace(f"run: {command}", f"{key}\n run: {command}")
for separator in ("", "\n", " # continued command\n"):
mutations[f"continued {command} {separator!r}"] = action.replace(
f"run: {command}\n", f"run: {command}\n{separator} || true\n"
)
for case, mutated in mutations.items():
with self.subTest(case=case):
(root / relative).write_text(mutated)
self.assertTrue(check_quick_checks(root))
(root / relative).unlink()
self.assertTrue(check_quick_checks(root))
def test_quick_checks_commands_propagate_failure(self) -> None:
runs = yaml_block((ROOT / ".github/actions/quick-checks/action.yml").read_text().splitlines(), "runs", 0)
steps = yaml_block(runs or [], "steps", 2) or []
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "scripts").mkdir()
commands = ("shellcheck", "actionlint", "./scripts/check_error_other_format_ratchet.sh")
for failing in commands:
with self.subTest(command=failing):
run = "shellcheck --version && actionlint" if failing != commands[-1] else failing
step = workflow_step_block(steps, run, key="run", indent=4)
self.assertIsNotNone(step)
run_index = next(index for index, line in enumerate(step[1]) if line.startswith(" run:"))
self.assertFalse(yaml_scalar_continues(step[1], run_index, 6))
body = step[1][run_index].removeprefix(" run: ")
for command in commands:
shim = root / command
shim.write_text(f"#!/bin/sh\nexit {17 if command == failing else 0}\n")
shim.chmod(0o755)
result = subprocess.run(
["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", body],
cwd=root, env=dict(os.environ, PATH=f"{root}{os.pathsep}{os.environ['PATH']}"),
capture_output=True, text=True,
)
self.assertEqual(result.returncode, 17, result.stderr)
def test_validate_includes_quick_checks(self) -> None:
error = "Quick Checks wiring regression"
with mock.patch(__name__ + ".check_quick_checks", return_value=[error]):
self.assertIn(error, validate(ROOT))
def test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -1252,7 +1058,6 @@ class SelfTests(unittest.TestCase):
mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
mock.patch(__name__ + ".check_ilm_build_budget", return_value=[]),
mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]),
mock.patch(__name__ + ".check_quick_checks", return_value=[]),
):
self.assertEqual(len(validate(root)), 1)
@@ -1693,7 +1498,7 @@ def main() -> int:
for error in errors:
print(f"ERROR: {error}", file=sys.stderr)
return 1
print("OK: e2e modules, runner selection, fuzz matrices, profiles, scheduled alerts, and Quick Checks are wired")
print("OK: e2e modules, runner selection, fuzz matrices, profiles, and scheduled alerts are wired")
return 0